dynamodb v2 no indexes

moto/__init__.py

@@ -3,6 +3,7 @@ logging.getLogger('boto').setLevel(logging.CRITICAL)
 
 from .autoscaling import mock_autoscaling
 from .dynamodb import mock_dynamodb
+from .dynamodb2 import mock_dynamodb2
 from .ec2 import mock_ec2
 from .elb import mock_elb
 from .emr import mock_emr

moto/backends.py

@@ -1,5 +1,6 @@
 from moto.autoscaling import autoscaling_backend
 from moto.dynamodb import dynamodb_backend
+from moto.dynamodb2 import dynamodb_backend2
 from moto.ec2 import ec2_backend
 from moto.elb import elb_backend
 from moto.emr import emr_backend
@@ -13,6 +14,7 @@ from moto.route53 import route53_backend
 BACKENDS = {
     'autoscaling': autoscaling_backend,
     'dynamodb': dynamodb_backend,
+    'dynamodb2': dynamodb_backend2,
     'ec2': ec2_backend,
     'elb': elb_backend,
     'emr': emr_backend,

moto/dynamodb2/__init__.py

@@ -0,0 +1,2 @@
+from .models import dynamodb_backend2
+mock_dynamodb2 = dynamodb_backend2.decorator

moto/dynamodb2/comparisons.py

@@ -0,0 +1,20 @@
+# TODO add tests for all of these
+COMPARISON_FUNCS = {
+    'EQ': lambda item_value, test_value: item_value == test_value,
+    'NE': lambda item_value, test_value: item_value != test_value,
+    'LE': lambda item_value, test_value: item_value <= test_value,
+    'LT': lambda item_value, test_value: item_value < test_value,
+    'GE': lambda item_value, test_value: item_value >= test_value,
+    'GT': lambda item_value, test_value: item_value > test_value,
+    'NULL': lambda item_value: item_value is None,
+    'NOT_NULL': lambda item_value: item_value is not None,
+    'CONTAINS': lambda item_value, test_value: test_value in item_value,
+    'NOT_CONTAINS': lambda item_value, test_value: test_value not in item_value,
+    'BEGINS_WITH': lambda item_value, test_value: item_value.startswith(test_value),
+    'IN': lambda item_value, test_value: item_value in test_value,
+    'BETWEEN': lambda item_value, lower_test_value, upper_test_value: lower_test_value <= item_value <= upper_test_value,
+}
+
+
+def get_comparison_func(range_comparison):
+    return COMPARISON_FUNCS.get(range_comparison)

moto/dynamodb2/models.py

@@ -0,0 +1,313 @@
+from collections import defaultdict
+import datetime
+import json
+
+try:
+        from collections import OrderedDict
+except ImportError:
+        # python 2.6 or earlier, use backport
+        from ordereddict import OrderedDict
+
+
+from moto.core import BaseBackend
+from .comparisons import get_comparison_func
+from .utils import unix_time
+
+
+class DynamoJsonEncoder(json.JSONEncoder):
+    def default(self, obj):
+        if hasattr(obj, 'to_json'):
+            return obj.to_json()
+
+
+def dynamo_json_dump(dynamo_object):
+    return json.dumps(dynamo_object, cls=DynamoJsonEncoder)
+
+
+class DynamoType(object):
+    """
+    http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DataModel.html#DataModelDataTypes
+    """
+
+    def __init__(self, type_as_dict):
+        self.type = type_as_dict.keys()[0]
+        self.value = type_as_dict.values()[0]
+
+    def __hash__(self):
+        return hash((self.type, self.value))
+
+    def __eq__(self, other):
+        return (
+            self.type == other.type and
+            self.value == other.value
+        )
+
+    def __repr__(self):
+        return "DynamoType: {0}".format(self.to_json())
+
+    def to_json(self):
+        return {self.type: self.value}
+
+    def compare(self, range_comparison, range_objs):
+        """
+        Compares this type against comparison filters
+        """
+        range_values = [obj.value for obj in range_objs]
+        comparison_func = get_comparison_func(range_comparison)
+        return comparison_func(self.value, *range_values)
+
+class Item(object):
+    def __init__(self, hash_key, hash_key_type, range_key, range_key_type, attrs):
+        self.hash_key = hash_key
+        self.hash_key_type = hash_key_type
+        self.range_key = range_key
+        self.range_key_type = range_key_type
+
+        self.attrs = {}
+        for key, value in attrs.iteritems():
+            self.attrs[key] = DynamoType(value)
+
+    def __repr__(self):
+        return "Item: {0}".format(self.to_json())
+
+    def to_json(self):
+        attributes = {}
+        for attribute_key, attribute in self.attrs.iteritems():
+            attributes[attribute_key] = attribute.value
+
+        return {
+            "Attributes": attributes
+        }
+
+    def describe_attrs(self, attributes):
+        if attributes:
+            included = {}
+            for key, value in self.attrs.iteritems():
+                if key in attributes:
+                    included[key] = value
+        else:
+            included = self.attrs
+        return {
+            "Item": included
+        }
+
+class Table(object):
+
+    def __init__(self, table_name, schema=None, attr = None, throughput=None, indexes=None):
+        self.name = table_name
+        self.attr = attr
+        self.schema = schema
+        self.range_key_attr = None
+        self.hash_key_attr = None
+        self.range_key_type = None
+        self.hash_key_type = None
+        for elem in schema:
+            if elem["KeyType"] == "HASH":
+                self.hash_key_attr = elem["AttributeName"]
+                self.hash_key_type = elem["KeyType"]
+            else:
+                self.range_key_attr = elem["AttributeName"]
+                self.range_key_type = elem["KeyType"]
+        if throughput is None:
+             self.throughput = {u'WriteCapacityUnits': 10, u'ReadCapacityUnits': 10}
+        else:
+            self.throughput = throughput
+        self.throughput["NumberOfDecreasesToday"] = 0
+        self.indexes = indexes
+        self.created_at = datetime.datetime.now()
+        self.items = defaultdict(dict)
+        
+    @property
+    def describe(self):
+        results = {
+        'Table': {
+            'AttributeDefinitions': self.attr,
+            'ProvisionedThroughput': self.throughput, 
+            'TableSizeBytes': 0, 
+            'TableName': self.name, 
+            'TableStatus': 'ACTIVE', 
+            'KeySchema': self.schema, 
+            'ItemCount': len(self), 
+            'CreationDateTime': unix_time(self.created_at)
+            }
+        }
+        return results
+    
+    def __len__(self):
+        count = 0
+        for key, value in self.items.iteritems():
+            if self.has_range_key:
+                count += len(value)
+            else:
+                count += 1
+        return count
+    
+    def put_item(self, item_attrs):
+        hash_value = DynamoType(item_attrs.get(self.hash_key_attr))
+        if self.has_range_key:
+            range_value = DynamoType(item_attrs.get(self.range_key_attr))
+        else:
+            range_value = None
+
+        item = Item(hash_value, self.hash_key_type, range_value, self.range_key_type, item_attrs)
+
+        if range_value:
+            self.items[hash_value][range_value] = item
+        else:
+            self.items[hash_value] = item
+        return item
+    
+    def __nonzero__(self):
+        return True
+    
+    @property
+    def has_range_key(self):
+        return self.range_key_attr is not None
+    
+    def get_item(self, hash_key, range_key):
+        if self.has_range_key and not range_key:
+            raise ValueError("Table has a range key, but no range key was passed into get_item")
+        try:
+            if range_key:
+                return self.items[hash_key][range_key]
+            else:
+                return self.items[hash_key]
+        except KeyError:
+            return None
+        
+    def delete_item(self, hash_key, range_key):
+        try:
+            if range_key:
+                return self.items[hash_key].pop(range_key)
+            else:
+                return self.items.pop(hash_key)
+        except KeyError:
+            return None
+        
+    def query(self, hash_key, range_comparison, range_objs):
+        results = []
+        last_page = True  # Once pagination is implemented, change this
+
+        possible_results =  [ item for item in list(self.all_items()) if item.hash_key == hash_key] 
+        if range_comparison:
+            for result in possible_results:
+                if result.range_key.compare(range_comparison, range_objs):
+                    results.append(result)
+        else:
+            # If we're not filtering on range key, return all values
+            results = possible_results
+        return results, last_page
+
+    def all_items(self):
+        for hash_set in self.items.values():
+            if self.range_key_attr:
+                for item in hash_set.values():
+                    yield item
+            else:
+                yield hash_set
+                
+    def scan(self, filters):
+        results = []
+        scanned_count = 0
+        last_page = True  # Once pagination is implemented, change this
+
+        for result in self.all_items():
+            scanned_count += 1
+            passes_all_conditions = True
+            for attribute_name, (comparison_operator, comparison_objs) in filters.iteritems():
+                attribute = result.attrs.get(attribute_name)
+
+                if attribute:
+                    # Attribute found
+                    if not attribute.compare(comparison_operator, comparison_objs):
+                        passes_all_conditions = False
+                        break
+                elif comparison_operator == 'NULL':
+                    # Comparison is NULL and we don't have the attribute
+                    continue
+                else:
+                    # No attribute found and comparison is no NULL. This item fails
+                    passes_all_conditions = False
+                    break
+
+            if passes_all_conditions:
+                results.append(result)
+        return results, scanned_count, last_page
+    
+
+class DynamoDBBackend(BaseBackend):
+
+    def __init__(self):
+        self.tables = OrderedDict()
+
+    def create_table(self, name, **params):
+        table = Table(name, **params)
+        self.tables[name] = table
+        return table
+
+    def delete_table(self, name):
+        return self.tables.pop(name, None)
+
+    def update_table_throughput(self, name, throughput):
+        table = self.tables[name]
+        table.throughput = throughput
+        return table
+
+    def put_item(self, table_name, item_attrs):
+        table = self.tables.get(table_name)
+        if not table:
+            return None
+        return table.put_item(item_attrs)
+    
+    def get_table_keys_name(self, table_name):
+        table = self.tables.get(table_name)
+        if not table:
+            return None, None
+        else:
+            return table.hash_key_attr, table.range_key_attr
+         
+    def get_keys_value(self, table, keys):
+        if not table.hash_key_attr in keys or (table.has_range_key and not table.range_key_attr in keys):
+            raise ValueError("Table has a range key, but no range key was passed into get_item")        
+        hash_key = DynamoType(keys[table.hash_key_attr])    
+        range_key = DynamoType(keys[table.range_key_attr]) if table.has_range_key else None
+        return hash_key,range_key
+
+    def get_item(self, table_name, keys):
+        table = self.tables.get(table_name)
+        if not table:
+            return None
+        hash_key,range_key = self.get_keys_value(table,keys)
+        return table.get_item(hash_key, range_key)
+
+    def query(self, table_name, hash_key_dict, range_comparison, range_value_dicts):
+        table = self.tables.get(table_name)
+        if not table:
+            return None, None
+
+        hash_key = DynamoType(hash_key_dict)
+        range_values = [DynamoType(range_value) for range_value in range_value_dicts]
+
+        return table.query(hash_key, range_comparison, range_values)
+    
+    def scan(self, table_name, filters):
+        table = self.tables.get(table_name)
+        if not table:
+            return None, None, None
+
+        scan_filters = {}
+        for key, (comparison_operator, comparison_values) in filters.iteritems():
+            dynamo_types = [DynamoType(value) for value in comparison_values]
+            scan_filters[key] = (comparison_operator, dynamo_types)
+
+        return table.scan(scan_filters)
+    
+    def delete_item(self, table_name, keys):
+        table = self.tables.get(table_name)
+        if not table:
+            return None
+        hash_key, range_key = self.get_keys_value(table, keys)
+        return table.delete_item(hash_key, range_key)
+
+
+dynamodb_backend2 = DynamoDBBackend()

moto/dynamodb2/responses.py

@@ -0,0 +1,302 @@
+import json
+
+from moto.core.responses import BaseResponse
+from moto.core.utils import camelcase_to_underscores
+from .models import dynamodb_backend2, dynamo_json_dump
+
+
+GET_SESSION_TOKEN_RESULT = """
+<GetSessionTokenResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
+ <GetSessionTokenResult>
+ <Credentials>
+ <SessionToken>
+ AQoEXAMPLEH4aoAH0gNCAPyJxz4BlCFFxWNE1OPTgk5TthT+FvwqnKwRcOIfrRh3c/L
+ To6UDdyJwOOvEVPvLXCrrrUtdnniCEXAMPLE/IvU1dYUg2RVAJBanLiHb4IgRmpRV3z
+ rkuWJOgQs8IZZaIv2BXIa2R4OlgkBN9bkUDNCJiBeb/AXlzBBko7b15fjrBs2+cTQtp
+ Z3CYWFXG8C5zqx37wnOE49mRl/+OtkIKGO7fAE
+ </SessionToken>
+ <SecretAccessKey>
+ wJalrXUtnFEMI/K7MDENG/bPxRfiCYzEXAMPLEKEY
+ </SecretAccessKey>
+ <Expiration>2011-07-11T19:55:29.611Z</Expiration>
+ <AccessKeyId>AKIAIOSFODNN7EXAMPLE</AccessKeyId>
+ </Credentials>
+ </GetSessionTokenResult>
+ <ResponseMetadata>
+ <RequestId>58c5dbae-abef-11e0-8cfe-09039844ac7d</RequestId>
+ </ResponseMetadata>
+</GetSessionTokenResponse>"""
+
+
+def sts_handler():
+    return GET_SESSION_TOKEN_RESULT
+
+
+class DynamoHandler(BaseResponse):
+
+    def get_endpoint_name(self, headers):
+        """Parses request headers and extracts part od the X-Amz-Target
+        that corresponds to a method of DynamoHandler
+
+        ie: X-Amz-Target: DynamoDB_20111205.ListTables -> ListTables
+        """
+        # Headers are case-insensitive. Probably a better way to do this.
+        match = headers.get('x-amz-target') or headers.get('X-Amz-Target')
+        if match:
+            return match.split(".")[1]
+
+    def error(self, type_, status=400):
+        return status, self.response_headers, dynamo_json_dump({'__type': type_})
+
+    def call_action(self):
+        if 'GetSessionToken' in self.body:
+            return 200, self.response_headers, sts_handler()
+
+        self.body = json.loads(self.body or '{}')
+        endpoint = self.get_endpoint_name(self.headers)
+        if endpoint:
+            endpoint = camelcase_to_underscores(endpoint)
+            response = getattr(self, endpoint)()
+            if isinstance(response, basestring):
+                return 200, self.response_headers, response
+
+            else:
+                status_code, new_headers, response_content = response
+                self.response_headers.update(new_headers)
+                return status_code, self.response_headers, response_content
+        else:
+            return 404, self.response_headers, ""
+
+    def list_tables(self):
+        body = self.body
+        limit = body.get('Limit')
+        if body.get("ExclusiveStartTableName"):
+            last = body.get("ExclusiveStartTableName")
+            start = dynamodb_backend2.tables.keys().index(last) + 1
+        else:
+            start = 0
+        all_tables = dynamodb_backend2.tables.keys()
+        if limit:
+            tables = all_tables[start:start + limit]
+        else:
+            tables = all_tables[start:]
+        response = {"TableNames": tables}
+        if limit and len(all_tables) > start + limit:
+            response["LastEvaluatedTableName"] = tables[-1]
+        return dynamo_json_dump(response)
+
+    def create_table(self):
+        body = self.body
+        #get the table name
+        table_name = body['TableName']
+        #get the throughput
+        throughput = body["ProvisionedThroughput"]        
+        #getting the schema
+        key_schema = body['KeySchema']
+        #getting attribute definition
+        attr = body["AttributeDefinitions"]
+        #getting the indexes
+        table = dynamodb_backend2.create_table(table_name, 
+                   schema = key_schema,           
+                   throughput = throughput, 
+                   attr = attr)
+        return dynamo_json_dump(table.describe)        
+
+    def delete_table(self):
+        name = self.body['TableName']
+        table = dynamodb_backend2.delete_table(name)
+        if table is not None:
+            return dynamo_json_dump(table.describe)
+        else:
+            er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException'
+            return self.error(er)
+
+    def update_table(self):
+        name = self.body['TableName']
+        throughput = self.body["ProvisionedThroughput"]
+        table = dynamodb_backend2.update_table_throughput(name, throughput)
+        return dynamo_json_dump(table.describe)
+
+    def describe_table(self):
+        name = self.body['TableName']
+        try:
+            table = dynamodb_backend2.tables[name]
+        except KeyError:
+            er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException'
+            return self.error(er)
+        return dynamo_json_dump(table.describe)
+
+    def put_item(self):
+        name = self.body['TableName']
+        item = self.body['Item']
+        result = dynamodb_backend2.put_item(name, item)
+        
+        if result:
+            item_dict = result.to_json()
+            item_dict['ConsumedCapacityUnits'] = 1
+            return dynamo_json_dump(item_dict)
+        else:
+            er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException'
+            return self.error(er)
+
+    def batch_write_item(self):
+        table_batches = self.body['RequestItems']
+
+        for table_name, table_requests in table_batches.iteritems():
+            for table_request in table_requests:
+                request_type = table_request.keys()[0]
+                request = table_request.values()[0]
+                if request_type == 'PutRequest':
+                    item = request['Item']
+                    dynamodb_backend2.put_item(table_name, item)
+                elif request_type == 'DeleteRequest':
+                    keys = request['Key']
+                    item = dynamodb_backend2.delete_item(table_name, keys)
+
+        response = {
+            "Responses": {
+                "Thread": {
+                    "ConsumedCapacityUnits": 1.0
+                },
+                "Reply": {
+                    "ConsumedCapacityUnits": 1.0
+                }
+            },
+            "UnprocessedItems": {}
+        }
+
+        return dynamo_json_dump(response)
+    def get_item(self):
+        name = self.body['TableName']
+        key = self.body['Key']
+        try:
+            item = dynamodb_backend2.get_item(name, key)
+        except ValueError:
+            er = 'com.amazon.coral.validate#ValidationException'
+            return self.error(er, status=400)
+        if item:
+            item_dict = item.describe_attrs(attributes = None)
+            item_dict['ConsumedCapacityUnits'] = 0.5
+            return dynamo_json_dump(item_dict)
+        else:
+            # Item not found
+            er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException'
+            return self.error(er, status=404)
+
+    def batch_get_item(self):
+        table_batches = self.body['RequestItems']
+
+        results = { 
+            "ConsumedCapacity":[],
+            "Responses": {                
+            },
+            "UnprocessedKeys": {
+            }
+        }
+
+        for table_name, table_request in table_batches.iteritems():
+            items = []
+            keys = table_request['Keys']
+            attributes_to_get = table_request.get('AttributesToGet')
+            results["Responses"][table_name]=[]
+            for key in keys:
+                item = dynamodb_backend2.get_item(table_name, key)
+                if item:
+                    item_describe = item.describe_attrs(attributes_to_get)
+                    results["Responses"][table_name].append(item_describe["Item"])
+
+            results["ConsumedCapacity"].append({
+                "CapacityUnits": len(keys),
+                "TableName": table_name
+            })
+        return dynamo_json_dump(results)
+
+    def query(self):
+        name = self.body['TableName']
+        keys = self.body['KeyConditions']
+        hash_key_name, range_key_name = dynamodb_backend2.get_table_keys_name(name)
+        if hash_key_name is None:
+            er = "'com.amazonaws.dynamodb.v20120810#ResourceNotFoundException"  
+            return self.error(er)
+        hash_key = keys[hash_key_name]['AttributeValueList'][0]
+        if len(keys) == 1:
+            range_comparison = None
+            range_values = []
+        else:
+            if range_key_name == None:
+                er = "com.amazon.coral.validate#ValidationException"  
+                return self.error(er)
+            else:
+                range_condition = keys[range_key_name]
+                if range_condition:
+                    range_comparison = range_condition['ComparisonOperator']
+                    range_values = range_condition['AttributeValueList']
+                else:
+                    range_comparison = None
+                    range_values = []
+        items, last_page = dynamodb_backend2.query(name, hash_key, range_comparison, range_values)
+        if items is None:
+            er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException'
+            return self.error(er) 
+        
+        result = {
+            "Count": len(items),
+            "Items": [item.attrs for item in items],
+            "ConsumedCapacityUnits": 1,
+        }
+
+        # Implement this when we do pagination
+        # if not last_page:
+        #     result["LastEvaluatedKey"] = {
+        #         "HashKeyElement": items[-1].hash_key,
+        #         "RangeKeyElement": items[-1].range_key,
+        #     }
+        return dynamo_json_dump(result)
+
+    def scan(self):
+        name = self.body['TableName']
+
+        filters = {}
+        scan_filters = self.body.get('ScanFilter', {})
+        for attribute_name, scan_filter in scan_filters.iteritems():
+            # Keys are attribute names. Values are tuples of (comparison, comparison_value)
+            comparison_operator = scan_filter["ComparisonOperator"]
+            comparison_values = scan_filter.get("AttributeValueList", [])
+            filters[attribute_name] = (comparison_operator, comparison_values)
+
+        items, scanned_count, last_page = dynamodb_backend2.scan(name, filters)
+
+        if items is None:
+            er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException'
+            return self.error(er)
+
+        result = {
+            "Count": len(items),
+            "Items": [item.attrs for item in items],
+            "ConsumedCapacityUnits": 1,
+            "ScannedCount": scanned_count
+        }
+
+        # Implement this when we do pagination
+        # if not last_page:
+        #     result["LastEvaluatedKey"] = {
+        #         "HashKeyElement": items[-1].hash_key,
+        #         "RangeKeyElement": items[-1].range_key,
+        #     }
+        return dynamo_json_dump(result)
+
+    def delete_item(self):
+        name = self.body['TableName']
+        keys = self.body['Key']
+        return_values = self.body.get('ReturnValues', '')
+        item = dynamodb_backend2.delete_item(name, keys)
+        if item:
+            if return_values == 'ALL_OLD':
+                item_dict = item.to_json()
+            else:
+                item_dict = {'Attributes': []}
+            item_dict['ConsumedCapacityUnits'] = 0.5
+            return dynamo_json_dump(item_dict)
+        else:
+            er = 'com.amazonaws.dynamodb.v20120810#ConditionalCheckFailedException'
+            return self.error(er)

moto/dynamodb2/urls.py

@@ -0,0 +1,10 @@
+from .responses import DynamoHandler
+
+url_bases = [
+    "https?://dynamodb.(.+).amazonaws.com",
+    "https?://sts.amazonaws.com",
+]
+
+url_paths = {
+    "{0}/": DynamoHandler().dispatch,
+}

moto/dynamodb2/utils.py

@@ -0,0 +1,5 @@
+import calendar
+
+
+def unix_time(dt):
+    return calendar.timegm(dt.timetuple())