From 4dec187d80a59cbe0d1a8b78d58dab20131516f7 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sat, 5 Oct 2019 15:20:43 +0100 Subject: [PATCH 1/9] #1834 - Check item size in DynamoDB --- moto/dynamodb2/responses.py | 4 ++++ tests/test_dynamodb2/test_dynamodb.py | 21 +++++++++++++++++++++ 2 files changed, 25 insertions(+) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index d07beefd..e7f5f282 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -225,6 +225,10 @@ class DynamoHandler(BaseResponse): er = 'com.amazonaws.dynamodb.v20111205#ValidationException' return self.error(er, 'Return values set to invalid value') + if len(str(item).encode('utf-8')) > 405000: + er = 'com.amazonaws.dynamodb.v20111205#ValidationException' + return self.error(er, 'Item size has exceeded the maximum allowed size') + if has_empty_keys_or_values(item): return get_empty_str_error() diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index b0952f10..4fd0fb47 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -2278,6 +2278,27 @@ def test_index_with_unknown_attributes_should_fail(): ex.exception.response['Error']['Message'].should.contain(expected_exception) +# https://github.com/spulec/moto/issues/1874 +@mock_dynamodb2 +def test_item_size_is_under_400KB(): + dynamodb = boto3.client('dynamodb', region_name='us-east-1') + + dynamodb.create_table( + TableName='moto-test', + KeySchema=[{'AttributeName': 'id', 'KeyType': 'HASH'}], + AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}], + ProvisionedThroughput={'ReadCapacityUnits': 1, 'WriteCapacityUnits': 1} + ) + + with assert_raises(ClientError) as ex: + large_item = 'x' * 410 * 1000 + dynamodb.put_item( + TableName='moto-test', + Item={'id': {'S': 'foo'}, 'item': {'S': large_item}}) + ex.exception.response['Error']['Code'].should.equal('ValidationException') + ex.exception.response['Error']['Message'].should.equal('Item size has exceeded the maximum allowed size') + + def _create_user_table(): client = boto3.client('dynamodb', region_name='us-east-1') client.create_table( From dc89b47b40273fb25069f303fe57528f3323b00c Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sun, 6 Oct 2019 14:08:58 +0100 Subject: [PATCH 2/9] #1874 - Count item size based on contents of actual dictionary --- moto/dynamodb2/exceptions.py | 5 +++ moto/dynamodb2/models.py | 57 ++++++++++++++++++++++++++- moto/dynamodb2/responses.py | 12 +++--- tests/test_dynamodb2/test_dynamodb.py | 15 +++++-- 4 files changed, 78 insertions(+), 11 deletions(-) diff --git a/moto/dynamodb2/exceptions.py b/moto/dynamodb2/exceptions.py index 9df97329..88bb732a 100644 --- a/moto/dynamodb2/exceptions.py +++ b/moto/dynamodb2/exceptions.py @@ -1,2 +1,7 @@ class InvalidIndexNameError(ValueError): pass + + +class ItemSizeTooLarge(Exception): + message = 'Item size has exceeded the maximum allowed size' + pass diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 4ef4461c..b97cd94e 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -16,7 +16,7 @@ from moto.core.exceptions import JsonRESTError from .comparisons import get_comparison_func from .comparisons import get_filter_expression from .comparisons import get_expected -from .exceptions import InvalidIndexNameError +from .exceptions import InvalidIndexNameError, ItemSizeTooLarge class DynamoJsonEncoder(json.JSONEncoder): @@ -30,6 +30,10 @@ def dynamo_json_dump(dynamo_object): return json.dumps(dynamo_object, cls=DynamoJsonEncoder) +def bytesize(val): + return len(str(val).encode('utf-8')) + + class DynamoType(object): """ http://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DataModel.html#DataModelDataTypes @@ -99,6 +103,22 @@ class DynamoType(object): return None + def size(self): + if self.is_number(): + value_size = len(str(self.value)) + elif self.is_set(): + sub_type = self.type[0] + value_size = sum([DynamoType({sub_type: v}).size() for v in self.value]) + elif self.is_list(): + value_size = sum([DynamoType(v).size() for v in self.value]) + elif self.is_map(): + value_size = sum([bytesize(k) + DynamoType(v).size() for k, v in self.value.items()]) + elif type(self.value) == bool: + value_size = 1 + else: + value_size = bytesize(self.value) + return value_size + def to_json(self): return {self.type: self.value} @@ -126,6 +146,39 @@ class DynamoType(object): return self.type == other.type +# https://github.com/spulec/moto/issues/1874 +# Ensure that the total size of an item does not exceed 400kb +class LimitedSizeDict(dict): + def __init__(self, *args, **kwargs): + self.update(*args, **kwargs) + + def __setitem__(self, key, value): + current_item_size = sum([item.size() if type(item) == DynamoType else bytesize(str(item)) for item in (self.keys() + self.values())]) + new_item_size = bytesize(key) + (value.size() if type(value) == DynamoType else bytesize(str(value))) + # Official limit is set to 400000 (400KB) + # Manual testing confirms that the actual limit is between 409 and 410KB + # We'll set the limit to something in between to be safe + if (current_item_size + new_item_size) > 405000: + raise ItemSizeTooLarge + super(LimitedSizeDict, self).__setitem__(key, value) + + def update(self, *args, **kwargs): + if args: + if len(args) > 1: + raise TypeError("update expected at most 1 arguments, " + "got %d" % len(args)) + other = dict(args[0]) + for key in other: + self[key] = other[key] + for key in kwargs: + self[key] = kwargs[key] + + def setdefault(self, key, value=None): + if key not in self: + self[key] = value + return self[key] + + class Item(BaseModel): def __init__(self, hash_key, hash_key_type, range_key, range_key_type, attrs): @@ -134,7 +187,7 @@ class Item(BaseModel): self.range_key = range_key self.range_key_type = range_key_type - self.attrs = {} + self.attrs = LimitedSizeDict() for key, value in attrs.items(): self.attrs[key] = DynamoType(value) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index e7f5f282..c04cf0ed 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -6,7 +6,7 @@ import re from moto.core.responses import BaseResponse from moto.core.utils import camelcase_to_underscores, amzn_request_id -from .exceptions import InvalidIndexNameError +from .exceptions import InvalidIndexNameError, ItemSizeTooLarge from .models import dynamodb_backends, dynamo_json_dump @@ -225,10 +225,6 @@ class DynamoHandler(BaseResponse): er = 'com.amazonaws.dynamodb.v20111205#ValidationException' return self.error(er, 'Return values set to invalid value') - if len(str(item).encode('utf-8')) > 405000: - er = 'com.amazonaws.dynamodb.v20111205#ValidationException' - return self.error(er, 'Item size has exceeded the maximum allowed size') - if has_empty_keys_or_values(item): return get_empty_str_error() @@ -259,6 +255,9 @@ class DynamoHandler(BaseResponse): name, item, expected, condition_expression, expression_attribute_names, expression_attribute_values, overwrite) + except ItemSizeTooLarge: + er = 'com.amazonaws.dynamodb.v20111205#ValidationException' + return self.error(er, ItemSizeTooLarge.message) except ValueError: er = 'com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException' return self.error(er, 'A condition specified in the operation could not be evaluated.') @@ -648,6 +647,9 @@ class DynamoHandler(BaseResponse): name, key, update_expression, attribute_updates, expression_attribute_names, expression_attribute_values, expected, condition_expression ) + except ItemSizeTooLarge: + er = 'com.amazonaws.dynamodb.v20111205#ValidationException' + return self.error(er, ItemSizeTooLarge.message) except ValueError: er = 'com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException' return self.error(er, 'A condition specified in the operation could not be evaluated.') diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 4fd0fb47..bedd6812 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -2282,6 +2282,7 @@ def test_index_with_unknown_attributes_should_fail(): @mock_dynamodb2 def test_item_size_is_under_400KB(): dynamodb = boto3.client('dynamodb', region_name='us-east-1') + res = boto3.resource('dynamodb') dynamodb.create_table( TableName='moto-test', @@ -2289,12 +2290,18 @@ def test_item_size_is_under_400KB(): AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}], ProvisionedThroughput={'ReadCapacityUnits': 1, 'WriteCapacityUnits': 1} ) + table = res.Table('moto-test') + large_item = 'x' * 410 * 1000 with assert_raises(ClientError) as ex: - large_item = 'x' * 410 * 1000 - dynamodb.put_item( - TableName='moto-test', - Item={'id': {'S': 'foo'}, 'item': {'S': large_item}}) + dynamodb.put_item(TableName='moto-test', Item={'id': {'S': 'foo'}, 'item': {'S': large_item}}) + with assert_raises(ClientError) as ex: + table.put_item(Item={'id': 'bar', 'item': large_item}) + with assert_raises(ClientError) as ex: + dynamodb.update_item(TableName='moto-test', + Key={'id': {'S': 'foo2'}}, + UpdateExpression='set item=:Item', + ExpressionAttributeValues={':Item': {'S': large_item}}) ex.exception.response['Error']['Code'].should.equal('ValidationException') ex.exception.response['Error']['Message'].should.equal('Item size has exceeded the maximum allowed size') From a389fdcd95be945e6cc876723422b5b3b0b3273a Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sun, 6 Oct 2019 15:24:37 +0100 Subject: [PATCH 3/9] #1874 - Adjust for Python2/3 incompatibilities --- moto/dynamodb2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index b97cd94e..9b0e74a1 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -153,7 +153,7 @@ class LimitedSizeDict(dict): self.update(*args, **kwargs) def __setitem__(self, key, value): - current_item_size = sum([item.size() if type(item) == DynamoType else bytesize(str(item)) for item in (self.keys() + self.values())]) + current_item_size = sum([item.size() if type(item) == DynamoType else bytesize(str(item)) for item in (list(self.keys()) + list(self.values()))]) new_item_size = bytesize(key) + (value.size() if type(value) == DynamoType else bytesize(str(value))) # Official limit is set to 400000 (400KB) # Manual testing confirms that the actual limit is between 409 and 410KB From c9348bc45ca6d251149404c5cd95150bbed5e0c8 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Tue, 8 Oct 2019 15:51:46 +0100 Subject: [PATCH 4/9] #1874 - Add tests for more complex items --- tests/test_dynamodb2/test_dynamodb.py | 28 +++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index bedd6812..d3997a59 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -2293,15 +2293,27 @@ def test_item_size_is_under_400KB(): table = res.Table('moto-test') large_item = 'x' * 410 * 1000 + assert_failure_due_to_item_size(func=dynamodb.put_item, + TableName='moto-test', + Item={'id': {'S': 'foo'}, 'item': {'S': large_item}}) + assert_failure_due_to_item_size(func=table.put_item, + Item={'id': 'bar', 'item': large_item}) + assert_failure_due_to_item_size(func=dynamodb.update_item, + TableName='moto-test', + Key={'id': {'S': 'foo2'}}, + UpdateExpression='set item=:Item', + ExpressionAttributeValues={':Item': {'S': large_item}}) + # Assert op fails when updating a nested item + assert_failure_due_to_item_size(func=table.put_item, + Item={'id': 'bar', 'itemlist': [{'item': large_item}]}) + assert_failure_due_to_item_size(func=dynamodb.put_item, + TableName='moto-test', + Item={'id': {'S': 'foo'}, 'itemlist': {'L': [{'M': {'item1': {'S': large_item}}}]}}) + + +def assert_failure_due_to_item_size(func, **kwargs): with assert_raises(ClientError) as ex: - dynamodb.put_item(TableName='moto-test', Item={'id': {'S': 'foo'}, 'item': {'S': large_item}}) - with assert_raises(ClientError) as ex: - table.put_item(Item={'id': 'bar', 'item': large_item}) - with assert_raises(ClientError) as ex: - dynamodb.update_item(TableName='moto-test', - Key={'id': {'S': 'foo2'}}, - UpdateExpression='set item=:Item', - ExpressionAttributeValues={':Item': {'S': large_item}}) + func(**kwargs) ex.exception.response['Error']['Code'].should.equal('ValidationException') ex.exception.response['Error']['Message'].should.equal('Item size has exceeded the maximum allowed size') From 41af98c98b6c372c1c158f42611dccdf4dcf6c7e Mon Sep 17 00:00:00 2001 From: Seth Black Date: Tue, 8 Oct 2019 15:59:03 -0500 Subject: [PATCH 5/9] added UpdateFunctionCode and UpdateFunctionConfiguration and associated test cases --- moto/awslambda/models.py | 65 +++++++++++++++++ moto/awslambda/responses.py | 36 ++++++++++ moto/awslambda/urls.py | 4 +- tests/test_awslambda/test_lambda.py | 106 ++++++++++++++++++++++++++++ 4 files changed, 210 insertions(+), 1 deletion(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index acc7a525..f2400ec3 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -273,6 +273,71 @@ class LambdaFunction(BaseModel): "Configuration": self.get_configuration(), } + def update_configuration(self, config_updates): + for key, value in config_updates.items(): + if key == "Description": + self.description = value + elif key == "Handler": + self.handler = value + elif key == "MemorySize": + self.memory_size = value + elif key == "Role": + self.role = value + elif key == "Runtime": + self.run_time = value + elif key == "Timeout": + self.timeout = value + elif key == "VpcConfig": + self.vpc_config = value + + return self.get_configuration() + + def update_function_code(self, spec): + if 'DryRun' in spec and spec['DryRun']: + return self.get_configuration() + + if 'Publish' in spec and spec['Publish']: + self.set_version(self.version + 1) + + if 'ZipFile' in spec: + # using the "hackery" from __init__" because it seems to work + # TODOs and FIXMEs included, because they'll need to be fixed + # in both places now + try: + to_unzip_code = base64.b64decode( + bytes(spec['ZipFile'], 'utf-8')) + except Exception: + to_unzip_code = base64.b64decode(spec['ZipFile']) + + self.code_bytes = to_unzip_code + self.code_size = len(to_unzip_code) + self.code_sha_256 = hashlib.sha256(to_unzip_code).hexdigest() + + # TODO: we should be putting this in a lambda bucket + self.code['UUID'] = str(uuid.uuid4()) + self.code['S3Key'] = '{}-{}'.format(self.function_name, self.code['UUID']) + else: + key = None + try: + # FIXME: does not validate bucket region + key = s3_backend.get_key(spec['S3Bucket'], spec['S3Key']) + except MissingBucket: + if do_validate_s3(): + raise ValueError( + "InvalidParameterValueException", + "Error occurred while GetObject. S3 Error Code: NoSuchBucket. S3 Error Message: The specified bucket does not exist") + except MissingKey: + if do_validate_s3(): + raise ValueError( + "InvalidParameterValueException", + "Error occurred while GetObject. S3 Error Code: NoSuchKey. S3 Error Message: The specified key does not exist.") + if key: + self.code_bytes = key.value + self.code_size = key.size + self.code_sha_256 = hashlib.sha256(key.value).hexdigest() + + return self.get_configuration() + @staticmethod def convert(s): try: diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index 1e7feb0d..e22fee15 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -122,6 +122,18 @@ class LambdaResponse(BaseResponse): if request.method == 'POST': return self._add_policy(request, full_url, headers) + def configuration(self, request, full_url, headers): + if request.method == 'PUT': + return self._put_configuration(request, full_url) + else: + raise ValueError("Cannot handle request") + + def code(self, request, full_url, headers): + if request.method == 'PUT': + return self._put_code(request, full_url, headers) + else: + raise ValueError("Cannot handle request") + def _add_policy(self, request, full_url, headers): path = request.path if hasattr(request, 'path') else path_url(request.url) function_name = path.split('/')[-2] @@ -308,3 +320,27 @@ class LambdaResponse(BaseResponse): return 204, {}, "{}" else: return 404, {}, "{}" + + def _put_configuration(self, request, full_url): + function_name = self._get_param('FunctionName', None) + qualifier = self._get_param('Qualifier', None) + + fn = self.lambda_backend.get_function(function_name, qualifier) + + if fn: + config = fn.update_configuration(json.loads(request.body)) + return 200, {}, json.dumps(config) + else: + return 404, {}, "{}" + + def _put_code(self, request, full_url, headers): + function_name = self._get_param('FunctionName', None) + qualifier = self._get_param('Qualifier', None) + + fn = self.lambda_backend.get_function(function_name, qualifier) + + if fn: + config = fn.update_function_code(json.loads(request.body)) + return 200, {}, json.dumps(config) + else: + return 404, {}, "{}" diff --git a/moto/awslambda/urls.py b/moto/awslambda/urls.py index fb2c6ee7..a306feff 100644 --- a/moto/awslambda/urls.py +++ b/moto/awslambda/urls.py @@ -16,5 +16,7 @@ url_paths = { r'{0}/(?P[^/]+)/functions/(?P[\w_-]+)/invocations/?$': response.invoke, r'{0}/(?P[^/]+)/functions/(?P[\w_-]+)/invoke-async/?$': response.invoke_async, r'{0}/(?P[^/]+)/tags/(?P.+)': response.tag, - r'{0}/(?P[^/]+)/functions/(?P[\w_-]+)/policy/?$': response.policy + r'{0}/(?P[^/]+)/functions/(?P[\w_-]+)/policy/?$': response.policy, + r'{0}/(?P[^/]+)/functions/(?P[\w_-]+)/configuration/?$': response.configuration, + r'{0}/(?P[^/]+)/functions/(?P[\w_-]+)/code/?$': response.code } diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 9467b080..0a6df7cc 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -1245,3 +1245,109 @@ def test_delete_event_source_mapping(): assert response['State'] == 'Deleting' conn.get_event_source_mapping.when.called_with(UUID=response['UUID'])\ .should.throw(botocore.client.ClientError) + + +@mock_lambda +@mock_s3 +def test_update_configuration(): + s3_conn = boto3.client('s3', 'us-west-2') + s3_conn.create_bucket(Bucket='test-bucket') + + zip_content = get_test_zip_file2() + s3_conn.put_object(Bucket='test-bucket', Key='test.zip', Body=zip_content) + conn = boto3.client('lambda', 'us-west-2') + + fxn = conn.create_function( + FunctionName='testFunction', + Runtime='python2.7', + Role='test-iam-role', + Handler='lambda_function.lambda_handler', + Code={ + 'S3Bucket': 'test-bucket', + 'S3Key': 'test.zip', + }, + Description='test lambda function', + Timeout=3, + MemorySize=128, + Publish=True, + ) + + assert fxn['Description'] == 'test lambda function' + assert fxn['Handler'] == 'lambda_function.lambda_handler' + assert fxn['MemorySize'] == 128 + assert fxn['Runtime'] == 'python2.7' + assert fxn['Timeout'] == 3 + + updated_config = conn.update_function_configuration( + FunctionName='testFunction', + Description='updated test lambda function', + Handler='lambda_function.new_lambda_handler', + Runtime='python3.6', + Timeout=7 + ) + + assert updated_config['ResponseMetadata']['HTTPStatusCode'] == 200 + assert updated_config['Description'] == 'updated test lambda function' + assert updated_config['Handler'] == 'lambda_function.new_lambda_handler' + assert updated_config['MemorySize'] == 128 + assert updated_config['Runtime'] == 'python3.6' + assert updated_config['Timeout'] == 7 + + +@mock_lambda +@freeze_time('2015-01-01 00:00:00') +def test_update_function(): + conn = boto3.client('lambda', 'us-west-2') + + zip_content_one = get_test_zip_file1() + + fxn = conn.create_function( + FunctionName='testFunction', + Runtime='python2.7', + Role='test-iam-role', + Handler='lambda_function.lambda_handler', + Code={ + 'ZipFile': zip_content_one, + }, + Description='test lambda function', + Timeout=3, + MemorySize=128, + Publish=True, + ) + + zip_content_two = get_test_zip_file2() + + conn.update_function_code( + FunctionName='testFunction', + ZipFile=zip_content_two, + Publish=True + ) + + response = conn.get_function( + FunctionName='testFunction' + ) + response['Configuration'].pop('LastModified') + + response['ResponseMetadata']['HTTPStatusCode'].should.equal(200) + assert len(response['Code']) == 2 + assert response['Code']['RepositoryType'] == 'S3' + assert response['Code']['Location'].startswith('s3://awslambda-{0}-tasks.s3-{0}.amazonaws.com'.format(_lambda_region)) + response['Configuration'].should.equal( + { + "CodeSha256": hashlib.sha256(zip_content_two).hexdigest(), + "CodeSize": len(zip_content_two), + "Description": "test lambda function", + "FunctionArn": 'arn:aws:lambda:{}:123456789012:function:testFunction:2'.format(_lambda_region), + "FunctionName": "testFunction", + "Handler": "lambda_function.lambda_handler", + "MemorySize": 128, + "Role": "test-iam-role", + "Runtime": "python2.7", + "Timeout": 3, + "Version": '$LATEST', + "VpcConfig": { + "SecurityGroupIds": [], + "SubnetIds": [], + } + }, + ) From e759b565da499bf04e90e9914fc372c2da3449f8 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Wed, 9 Oct 2019 10:02:40 +0100 Subject: [PATCH 6/9] #1874 - Change tests to use resource/client when appropriate --- tests/test_dynamodb2/test_dynamodb.py | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 46d5f8f3..097d101e 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -2298,8 +2298,8 @@ def test_index_with_unknown_attributes_should_fail(): # https://github.com/spulec/moto/issues/1874 @mock_dynamodb2 def test_item_size_is_under_400KB(): - dynamodb = boto3.client('dynamodb', region_name='us-east-1') - res = boto3.resource('dynamodb') + dynamodb = boto3.resource('dynamodb') + client = boto3.client('dynamodb') dynamodb.create_table( TableName='moto-test', @@ -2307,15 +2307,14 @@ def test_item_size_is_under_400KB(): AttributeDefinitions=[{'AttributeName': 'id', 'AttributeType': 'S'}], ProvisionedThroughput={'ReadCapacityUnits': 1, 'WriteCapacityUnits': 1} ) - table = res.Table('moto-test') + table = dynamodb.Table('moto-test') large_item = 'x' * 410 * 1000 - assert_failure_due_to_item_size(func=dynamodb.put_item, + assert_failure_due_to_item_size(func=client.put_item, TableName='moto-test', Item={'id': {'S': 'foo'}, 'item': {'S': large_item}}) - assert_failure_due_to_item_size(func=table.put_item, - Item={'id': 'bar', 'item': large_item}) - assert_failure_due_to_item_size(func=dynamodb.update_item, + assert_failure_due_to_item_size(func=table.put_item, Item = {'id': 'bar', 'item': large_item}) + assert_failure_due_to_item_size(func=client.update_item, TableName='moto-test', Key={'id': {'S': 'foo2'}}, UpdateExpression='set item=:Item', @@ -2323,7 +2322,7 @@ def test_item_size_is_under_400KB(): # Assert op fails when updating a nested item assert_failure_due_to_item_size(func=table.put_item, Item={'id': 'bar', 'itemlist': [{'item': large_item}]}) - assert_failure_due_to_item_size(func=dynamodb.put_item, + assert_failure_due_to_item_size(func=client.put_item, TableName='moto-test', Item={'id': {'S': 'foo'}, 'itemlist': {'L': [{'M': {'item1': {'S': large_item}}}]}}) From 20dc8ae5c4b44439d302809bd8c4a74179bb35d1 Mon Sep 17 00:00:00 2001 From: Seth Black Date: Wed, 9 Oct 2019 15:15:10 -0500 Subject: [PATCH 7/9] getting tests working in server mode --- moto/awslambda/models.py | 20 +++++++++++--------- moto/awslambda/responses.py | 18 ++++++++++-------- tests/test_awslambda/test_lambda.py | 3 +-- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index f2400ec3..a1643cd0 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -292,22 +292,24 @@ class LambdaFunction(BaseModel): return self.get_configuration() - def update_function_code(self, spec): - if 'DryRun' in spec and spec['DryRun']: + def update_function_code(self, updated_spec): + if 'DryRun' in updated_spec and updated_spec['DryRun']: return self.get_configuration() - if 'Publish' in spec and spec['Publish']: + if 'Publish' in updated_spec and updated_spec['Publish']: self.set_version(self.version + 1) - if 'ZipFile' in spec: - # using the "hackery" from __init__" because it seems to work + if 'ZipFile' in updated_spec: + self.code['ZipFile'] = updated_spec['ZipFile'] + + # using the "hackery" from __init__ because it seems to work # TODOs and FIXMEs included, because they'll need to be fixed # in both places now try: to_unzip_code = base64.b64decode( - bytes(spec['ZipFile'], 'utf-8')) + bytes(updated_spec['ZipFile'], 'utf-8')) except Exception: - to_unzip_code = base64.b64decode(spec['ZipFile']) + to_unzip_code = base64.b64decode(updated_spec['ZipFile']) self.code_bytes = to_unzip_code self.code_size = len(to_unzip_code) @@ -316,11 +318,11 @@ class LambdaFunction(BaseModel): # TODO: we should be putting this in a lambda bucket self.code['UUID'] = str(uuid.uuid4()) self.code['S3Key'] = '{}-{}'.format(self.function_name, self.code['UUID']) - else: + elif 'S3Bucket' in updated_spec and 'S3Key' in updated_spec: key = None try: # FIXME: does not validate bucket region - key = s3_backend.get_key(spec['S3Bucket'], spec['S3Key']) + key = s3_backend.get_key(updated_spec['S3Bucket'], updated_spec['S3Key']) except MissingBucket: if do_validate_s3(): raise ValueError( diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index e22fee15..2041aa66 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -123,14 +123,16 @@ class LambdaResponse(BaseResponse): return self._add_policy(request, full_url, headers) def configuration(self, request, full_url, headers): + self.setup_class(request, full_url, headers) if request.method == 'PUT': - return self._put_configuration(request, full_url) + return self._put_configuration(request) else: raise ValueError("Cannot handle request") def code(self, request, full_url, headers): + self.setup_class(request, full_url, headers) if request.method == 'PUT': - return self._put_code(request, full_url, headers) + return self._put_code() else: raise ValueError("Cannot handle request") @@ -321,26 +323,26 @@ class LambdaResponse(BaseResponse): else: return 404, {}, "{}" - def _put_configuration(self, request, full_url): - function_name = self._get_param('FunctionName', None) + def _put_configuration(self, request): + function_name = self.path.rsplit('/', 2)[-2] qualifier = self._get_param('Qualifier', None) fn = self.lambda_backend.get_function(function_name, qualifier) if fn: - config = fn.update_configuration(json.loads(request.body)) + config = fn.update_configuration(self.json_body) return 200, {}, json.dumps(config) else: return 404, {}, "{}" - def _put_code(self, request, full_url, headers): - function_name = self._get_param('FunctionName', None) + def _put_code(self): + function_name = self.path.rsplit('/', 2)[-2] qualifier = self._get_param('Qualifier', None) fn = self.lambda_backend.get_function(function_name, qualifier) if fn: - config = fn.update_function_code(json.loads(request.body)) + config = fn.update_function_code(self.json_body) return 200, {}, json.dumps(config) else: return 404, {}, "{}" diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 0a6df7cc..20a806de 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -1295,7 +1295,6 @@ def test_update_configuration(): @mock_lambda -@freeze_time('2015-01-01 00:00:00') def test_update_function(): conn = boto3.client('lambda', 'us-west-2') @@ -1317,7 +1316,7 @@ def test_update_function(): zip_content_two = get_test_zip_file2() - conn.update_function_code( + fxn_updated = conn.update_function_code( FunctionName='testFunction', ZipFile=zip_content_two, Publish=True From dff24cb032fc11d80b50034423613b5739b9b492 Mon Sep 17 00:00:00 2001 From: Seth Black Date: Wed, 9 Oct 2019 16:20:49 -0500 Subject: [PATCH 8/9] bringing up test percentage --- moto/awslambda/models.py | 3 -- moto/awslambda/responses.py | 3 ++ tests/test_awslambda/test_lambda.py | 81 ++++++++++++++++++++++++++--- 3 files changed, 77 insertions(+), 10 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index a1643cd0..bd85ded9 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -296,9 +296,6 @@ class LambdaFunction(BaseModel): if 'DryRun' in updated_spec and updated_spec['DryRun']: return self.get_configuration() - if 'Publish' in updated_spec and updated_spec['Publish']: - self.set_version(self.version + 1) - if 'ZipFile' in updated_spec: self.code['ZipFile'] = updated_spec['ZipFile'] diff --git a/moto/awslambda/responses.py b/moto/awslambda/responses.py index 2041aa66..83a1cefc 100644 --- a/moto/awslambda/responses.py +++ b/moto/awslambda/responses.py @@ -342,6 +342,9 @@ class LambdaResponse(BaseResponse): fn = self.lambda_backend.get_function(function_name, qualifier) if fn: + if self.json_body.get('Publish', False): + fn = self.lambda_backend.publish_function(function_name) + config = fn.update_function_code(self.json_body) return 200, {}, json.dumps(config) else: diff --git a/tests/test_awslambda/test_lambda.py b/tests/test_awslambda/test_lambda.py index 20a806de..d6ee3f7f 100644 --- a/tests/test_awslambda/test_lambda.py +++ b/tests/test_awslambda/test_lambda.py @@ -1295,13 +1295,13 @@ def test_update_configuration(): @mock_lambda -def test_update_function(): +def test_update_function_zip(): conn = boto3.client('lambda', 'us-west-2') zip_content_one = get_test_zip_file1() fxn = conn.create_function( - FunctionName='testFunction', + FunctionName='testFunctionZip', Runtime='python2.7', Role='test-iam-role', Handler='lambda_function.lambda_handler', @@ -1317,13 +1317,14 @@ def test_update_function(): zip_content_two = get_test_zip_file2() fxn_updated = conn.update_function_code( - FunctionName='testFunction', + FunctionName='testFunctionZip', ZipFile=zip_content_two, Publish=True ) response = conn.get_function( - FunctionName='testFunction' + FunctionName='testFunctionZip', + Qualifier='2' ) response['Configuration'].pop('LastModified') @@ -1336,14 +1337,80 @@ def test_update_function(): "CodeSha256": hashlib.sha256(zip_content_two).hexdigest(), "CodeSize": len(zip_content_two), "Description": "test lambda function", - "FunctionArn": 'arn:aws:lambda:{}:123456789012:function:testFunction:2'.format(_lambda_region), - "FunctionName": "testFunction", + "FunctionArn": 'arn:aws:lambda:{}:123456789012:function:testFunctionZip:2'.format(_lambda_region), + "FunctionName": "testFunctionZip", "Handler": "lambda_function.lambda_handler", "MemorySize": 128, "Role": "test-iam-role", "Runtime": "python2.7", "Timeout": 3, - "Version": '$LATEST', + "Version": '2', + "VpcConfig": { + "SecurityGroupIds": [], + "SubnetIds": [], + } + }, + ) + +@mock_lambda +@mock_s3 +def test_update_function_s3(): + s3_conn = boto3.client('s3', 'us-west-2') + s3_conn.create_bucket(Bucket='test-bucket') + + zip_content = get_test_zip_file1() + s3_conn.put_object(Bucket='test-bucket', Key='test.zip', Body=zip_content) + + conn = boto3.client('lambda', 'us-west-2') + + fxn = conn.create_function( + FunctionName='testFunctionS3', + Runtime='python2.7', + Role='test-iam-role', + Handler='lambda_function.lambda_handler', + Code={ + 'S3Bucket': 'test-bucket', + 'S3Key': 'test.zip', + }, + Description='test lambda function', + Timeout=3, + MemorySize=128, + Publish=True, + ) + + zip_content_two = get_test_zip_file2() + s3_conn.put_object(Bucket='test-bucket', Key='test2.zip', Body=zip_content_two) + + fxn_updated = conn.update_function_code( + FunctionName='testFunctionS3', + S3Bucket='test-bucket', + S3Key='test2.zip', + Publish=True + ) + + response = conn.get_function( + FunctionName='testFunctionS3', + Qualifier='2' + ) + response['Configuration'].pop('LastModified') + + response['ResponseMetadata']['HTTPStatusCode'].should.equal(200) + assert len(response['Code']) == 2 + assert response['Code']['RepositoryType'] == 'S3' + assert response['Code']['Location'].startswith('s3://awslambda-{0}-tasks.s3-{0}.amazonaws.com'.format(_lambda_region)) + response['Configuration'].should.equal( + { + "CodeSha256": hashlib.sha256(zip_content_two).hexdigest(), + "CodeSize": len(zip_content_two), + "Description": "test lambda function", + "FunctionArn": 'arn:aws:lambda:{}:123456789012:function:testFunctionS3:2'.format(_lambda_region), + "FunctionName": "testFunctionS3", + "Handler": "lambda_function.lambda_handler", + "MemorySize": 128, + "Role": "test-iam-role", + "Runtime": "python2.7", + "Timeout": 3, + "Version": '2', "VpcConfig": { "SecurityGroupIds": [], "SubnetIds": [], From ed6d780dab3df6ed003183fddb6bd218b556a3b8 Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Thu, 10 Oct 2019 09:14:22 +0100 Subject: [PATCH 9/9] Remove dead code --- moto/awslambda/models.py | 8 --- moto/dynamodb2/comparisons.py | 15 ------ moto/dynamodb2/models.py | 6 +-- moto/rds/models.py | 91 ----------------------------------- 4 files changed, 3 insertions(+), 117 deletions(-) diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index acc7a525..ca045670 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -280,14 +280,6 @@ class LambdaFunction(BaseModel): except Exception: return s - @staticmethod - def is_json(test_str): - try: - response = json.loads(test_str) - except Exception: - response = test_str - return response - def _invoke_lambda(self, code, event=None, context=None): # TODO: context not yet implemented if event is None: diff --git a/moto/dynamodb2/comparisons.py b/moto/dynamodb2/comparisons.py index dbc0bd57..b08a5c19 100644 --- a/moto/dynamodb2/comparisons.py +++ b/moto/dynamodb2/comparisons.py @@ -799,21 +799,6 @@ class ConditionExpressionParser: else: # pragma: no cover raise ValueError("Unknown expression node kind %r" % node.kind) - def _print_debug(self, nodes): # pragma: no cover - print('ROOT') - for node in nodes: - self._print_node_recursive(node, depth=1) - - def _print_node_recursive(self, node, depth=0): # pragma: no cover - if len(node.children) > 0: - print(' ' * depth, node.nonterminal, node.kind) - for child in node.children: - self._print_node_recursive(child, depth=depth + 1) - else: - print(' ' * depth, node.nonterminal, node.kind, node.value) - - - def _assert(self, condition, message, nodes): if not condition: raise ValueError(message + " " + " ".join([t.text for t in nodes])) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index b2ff7ffc..28e2a350 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -824,7 +824,7 @@ class Table(BaseModel): exclusive_start_key, index_name) return results, scanned_count, last_evaluated_key - def _trim_results(self, results, limit, exclusive_start_key, scaned_index=None): + def _trim_results(self, results, limit, exclusive_start_key, scanned_index=None): if exclusive_start_key is not None: hash_key = DynamoType(exclusive_start_key.get(self.hash_key_attr)) range_key = exclusive_start_key.get(self.range_key_attr) @@ -844,10 +844,10 @@ class Table(BaseModel): if results[-1].range_key is not None: last_evaluated_key[self.range_key_attr] = results[-1].range_key - if scaned_index: + if scanned_index: all_indexes = self.all_indexes() indexes_by_name = dict((i['IndexName'], i) for i in all_indexes) - idx = indexes_by_name[scaned_index] + idx = indexes_by_name[scanned_index] idx_col_list = [i['AttributeName'] for i in idx['KeySchema']] for col in idx_col_list: last_evaluated_key[col] = results[-1].attrs[col] diff --git a/moto/rds/models.py b/moto/rds/models.py index feecefe0..592516b3 100644 --- a/moto/rds/models.py +++ b/moto/rds/models.py @@ -1,7 +1,5 @@ from __future__ import unicode_literals -import datetime - import boto.rds from jinja2 import Template @@ -14,95 +12,6 @@ from moto.rds2.models import rds2_backends class Database(BaseModel): - def __init__(self, **kwargs): - self.status = "available" - - self.is_replica = False - self.replicas = [] - - self.region = kwargs.get('region') - self.engine = kwargs.get("engine") - self.engine_version = kwargs.get("engine_version") - if self.engine_version is None: - self.engine_version = "5.6.21" - self.iops = kwargs.get("iops") - self.storage_encrypted = kwargs.get("storage_encrypted", False) - if self.storage_encrypted: - self.kms_key_id = kwargs.get("kms_key_id", "default_kms_key_id") - else: - self.kms_key_id = kwargs.get("kms_key_id") - self.storage_type = kwargs.get("storage_type") - self.master_username = kwargs.get('master_username') - self.master_password = kwargs.get('master_password') - self.auto_minor_version_upgrade = kwargs.get( - 'auto_minor_version_upgrade') - if self.auto_minor_version_upgrade is None: - self.auto_minor_version_upgrade = True - self.allocated_storage = kwargs.get('allocated_storage') - self.db_instance_identifier = kwargs.get('db_instance_identifier') - self.source_db_identifier = kwargs.get("source_db_identifier") - self.db_instance_class = kwargs.get('db_instance_class') - self.port = kwargs.get('port') - self.db_name = kwargs.get("db_name") - self.publicly_accessible = kwargs.get("publicly_accessible") - if self.publicly_accessible is None: - self.publicly_accessible = True - - self.copy_tags_to_snapshot = kwargs.get("copy_tags_to_snapshot") - if self.copy_tags_to_snapshot is None: - self.copy_tags_to_snapshot = False - - self.backup_retention_period = kwargs.get("backup_retention_period") - if self.backup_retention_period is None: - self.backup_retention_period = 1 - - self.availability_zone = kwargs.get("availability_zone") - self.multi_az = kwargs.get("multi_az") - self.db_subnet_group_name = kwargs.get("db_subnet_group_name") - self.instance_create_time = str(datetime.datetime.utcnow()) - if self.db_subnet_group_name: - self.db_subnet_group = rds_backends[ - self.region].describe_subnet_groups(self.db_subnet_group_name)[0] - else: - self.db_subnet_group = [] - - self.security_groups = kwargs.get('security_groups', []) - - # PreferredBackupWindow - # PreferredMaintenanceWindow - # backup_retention_period = self._get_param("BackupRetentionPeriod") - # OptionGroupName - # DBParameterGroupName - # VpcSecurityGroupIds.member.N - - @property - def db_instance_arn(self): - return "arn:aws:rds:{0}:1234567890:db:{1}".format( - self.region, self.db_instance_identifier) - - @property - def physical_resource_id(self): - return self.db_instance_identifier - - @property - def address(self): - return "{0}.aaaaaaaaaa.{1}.rds.amazonaws.com".format(self.db_instance_identifier, self.region) - - def add_replica(self, replica): - self.replicas.append(replica.db_instance_identifier) - - def remove_replica(self, replica): - self.replicas.remove(replica.db_instance_identifier) - - def set_as_replica(self): - self.is_replica = True - self.replicas = [] - - def update(self, db_kwargs): - for key, value in db_kwargs.items(): - if value is not None: - setattr(self, key, value) - def get_cfn_attribute(self, attribute_name): if attribute_name == 'Endpoint.Address': return self.address