From d6384fcb35dd1f3e546b6049cb5d7e9fad9243e1 Mon Sep 17 00:00:00 2001 From: Matthew Davis <7035647+mdavis-xyz@users.noreply.github.com> Date: Fri, 30 Apr 2021 22:47:47 +1000 Subject: [PATCH] Enforce dynamodb key size limit [solves #3866] (#3888) * add tests for dynamodb max key size correct too-large error for ddb key * remove unnecessary requires_boto_gte decorator from ddb tests * remove literal emoji from ddb test * implement dynamodb key limits, WIP * correct direction of dynamodb range key length check * fix tests for dynamodb max key size check * catch ddb validation errors and rethrow properly * finish ddb key size limit fixes * fix linting * handle unicode in v2.7 tests * fix encoding issue in py2.7 for ddb * linting * Python2/3 compatability Co-authored-by: Bert Blommers --- .gitignore | 3 +- moto/core/responses.py | 6 +- moto/core/utils.py | 2 +- moto/dynamodb2/exceptions.py | 22 ++ moto/dynamodb2/limits.py | 5 + moto/dynamodb2/models/__init__.py | 13 +- moto/dynamodb2/models/utilities.py | 2 +- moto/dynamodb2/responses.py | 5 +- tests/test_dynamodb2/test_dynamodb.py | 391 ++++++++++++++++++++++---- 9 files changed, 390 insertions(+), 59 deletions(-) create mode 100644 moto/dynamodb2/limits.py diff --git a/.gitignore b/.gitignore index 04480a29..b67fe913 100644 --- a/.gitignore +++ b/.gitignore @@ -23,4 +23,5 @@ tests/file.tmp .mypy_cache/ *.tmp .venv/ -htmlcov/ \ No newline at end of file +htmlcov/ +.~c9_* \ No newline at end of file diff --git a/moto/core/responses.py b/moto/core/responses.py index 68493c87..041623a7 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -149,7 +149,7 @@ class ActionAuthenticatorMixin(object): if settings.TEST_SERVER_MODE: response = requests.post( "http://localhost:5000/moto-api/reset-auth", - data=str(initial_no_auth_action_count).encode(), + data=str(initial_no_auth_action_count).encode("utf-8"), ) original_initial_no_auth_action_count = response.json()[ "PREVIOUS_INITIAL_NO_AUTH_ACTION_COUNT" @@ -167,7 +167,9 @@ class ActionAuthenticatorMixin(object): if settings.TEST_SERVER_MODE: requests.post( "http://localhost:5000/moto-api/reset-auth", - data=str(original_initial_no_auth_action_count).encode(), + data=str(original_initial_no_auth_action_count).encode( + "utf-8" + ), ) else: ActionAuthenticatorMixin.request_count = original_request_count diff --git a/moto/core/utils.py b/moto/core/utils.py index 66eeaeb9..09bdbd37 100644 --- a/moto/core/utils.py +++ b/moto/core/utils.py @@ -230,7 +230,7 @@ def unix_time_millis(dt=None): def gen_amz_crc32(response, headerdict=None): if not isinstance(response, bytes): - response = response.encode() + response = response.encode("utf-8") crc = binascii.crc32(response) if six.PY2: diff --git a/moto/dynamodb2/exceptions.py b/moto/dynamodb2/exceptions.py index 01b98b35..bc661549 100644 --- a/moto/dynamodb2/exceptions.py +++ b/moto/dynamodb2/exceptions.py @@ -1,3 +1,6 @@ +from moto.dynamodb2.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH + + class InvalidIndexNameError(ValueError): pass @@ -133,6 +136,25 @@ class ItemSizeToUpdateTooLarge(MockValidationException): ) +class HashKeyTooLong(MockValidationException): + # deliberately no space between of and {lim} + key_too_large_msg = "One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of{lim} bytes".format( + lim=HASH_KEY_MAX_LENGTH + ) + + def __init__(self): + super(HashKeyTooLong, self).__init__(self.key_too_large_msg) + + +class RangeKeyTooLong(MockValidationException): + key_too_large_msg = "One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of {lim} bytes".format( + lim=RANGE_KEY_MAX_LENGTH + ) + + def __init__(self): + super(RangeKeyTooLong, self).__init__(self.key_too_large_msg) + + class IncorrectOperandType(InvalidUpdateExpression): inv_operand_msg = "Incorrect operand type for operator or function; operator or function: {f}, operand type: {t}" diff --git a/moto/dynamodb2/limits.py b/moto/dynamodb2/limits.py new file mode 100644 index 00000000..d7b38fe8 --- /dev/null +++ b/moto/dynamodb2/limits.py @@ -0,0 +1,5 @@ +# https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html#limits-partition-sort-keys +# measured in bytes +# <= not < +HASH_KEY_MAX_LENGTH = 2048 +RANGE_KEY_MAX_LENGTH = 1024 diff --git a/moto/dynamodb2/models/__init__.py b/moto/dynamodb2/models/__init__.py index 2772dacf..d83a526b 100644 --- a/moto/dynamodb2/models/__init__.py +++ b/moto/dynamodb2/models/__init__.py @@ -18,6 +18,8 @@ from moto.dynamodb2.exceptions import ( InvalidIndexNameError, ItemSizeTooLarge, ItemSizeToUpdateTooLarge, + HashKeyTooLong, + RangeKeyTooLong, ConditionalCheckFailed, TransactionCanceledException, EmptyKeyAttributeException, @@ -27,6 +29,7 @@ from moto.dynamodb2.models.dynamo_type import DynamoType from moto.dynamodb2.parsing.executors import UpdateExpressionExecutor from moto.dynamodb2.parsing.expressions import UpdateExpressionParser from moto.dynamodb2.parsing.validators import UpdateExpressionValidator +from moto.dynamodb2.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH class DynamoJsonEncoder(json.JSONEncoder): @@ -70,6 +73,11 @@ class Item(BaseModel): self.range_key = range_key self.range_key_type = range_key_type + if hash_key and hash_key.size() > HASH_KEY_MAX_LENGTH: + raise HashKeyTooLong + if range_key and (range_key.size() > RANGE_KEY_MAX_LENGTH): + raise RangeKeyTooLong + self.attrs = LimitedSizeDict() for key, value in attrs.items(): self.attrs[key] = DynamoType(value) @@ -1309,13 +1317,14 @@ class DynamoDBBackend(BaseBackend): item.validate_no_empty_key_values(attribute_updates, table.key_attributes) if update_expression: - validated_ast = UpdateExpressionValidator( + validator = UpdateExpressionValidator( update_expression_ast, expression_attribute_names=expression_attribute_names, expression_attribute_values=expression_attribute_values, item=item, table=table, - ).validate() + ) + validated_ast = validator.validate() try: UpdateExpressionExecutor( validated_ast, item, expression_attribute_names diff --git a/moto/dynamodb2/models/utilities.py b/moto/dynamodb2/models/utilities.py index 9dd6f1e9..839087dc 100644 --- a/moto/dynamodb2/models/utilities.py +++ b/moto/dynamodb2/models/utilities.py @@ -2,7 +2,7 @@ import re def bytesize(val): - return len(str(val).encode("utf-8")) + return len(val.encode("utf-8")) def attribute_is_list(attr): diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 8ca9417a..63857eae 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -11,7 +11,6 @@ from moto.core.responses import BaseResponse from moto.core.utils import camelcase_to_underscores, amz_crc32, amzn_request_id from .exceptions import ( InvalidIndexNameError, - ItemSizeTooLarge, MockValidationException, TransactionCanceledException, ) @@ -296,9 +295,9 @@ class DynamoHandler(BaseResponse): expression_attribute_values, overwrite, ) - except ItemSizeTooLarge: + except MockValidationException as mve: er = "com.amazonaws.dynamodb.v20111205#ValidationException" - return self.error(er, ItemSizeTooLarge.item_size_too_large_msg) + return self.error(er, mve.exception_msg) except KeyError as ke: er = "com.amazonaws.dynamodb.v20111205#ValidationException" return self.error(er, ke.args[0]) diff --git a/tests/test_dynamodb2/test_dynamodb.py b/tests/test_dynamodb2/test_dynamodb.py index 352f542b..122b32c1 100644 --- a/tests/test_dynamodb2/test_dynamodb.py +++ b/tests/test_dynamodb2/test_dynamodb.py @@ -16,6 +16,7 @@ from tests.helpers import requires_boto_gte import moto.dynamodb2.comparisons import moto.dynamodb2.models +from moto.dynamodb2.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH import pytest @@ -184,9 +185,8 @@ def test_list_not_found_table_tags(): assert exception.response["Error"]["Code"] == "ResourceNotFoundException" -@requires_boto_gte("2.9") @mock_dynamodb2 -def test_item_add_empty_string_in_key_exception(): +def test_item_add_empty_string_hash_key_exception(): name = "TestTable" conn = boto3.client( "dynamodb", @@ -220,9 +220,49 @@ def test_item_add_empty_string_in_key_exception(): ) -@requires_boto_gte("2.9") @mock_dynamodb2 -def test_item_add_empty_string_no_exception(): +def test_item_add_empty_string_range_key_exception(): + name = "TestTable" + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + conn.create_table( + TableName=name, + KeySchema=[ + {"AttributeName": "forum_name", "KeyType": "HASH"}, + {"AttributeName": "ReceivedTime", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "forum_name", "AttributeType": "S"}, + {"AttributeName": "ReceivedTime", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + with pytest.raises(ClientError) as ex: + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": "LOLCat Forum"}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "someone@somewhere.edu"}, + "ReceivedTime": {"S": ""}, + }, + ) + + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.value.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: An AttributeValue may not contain an empty string" + ) + + +@mock_dynamodb2 +def test_item_add_empty_string_attr_no_exception(): name = "TestTable" conn = boto3.client( "dynamodb", @@ -249,52 +289,8 @@ def test_item_add_empty_string_no_exception(): ) -@requires_boto_gte("2.9") @mock_dynamodb2 -def test_update_item_with_empty_string_in_key_exception(): - name = "TestTable" - conn = boto3.client( - "dynamodb", - region_name="us-west-2", - aws_access_key_id="ak", - aws_secret_access_key="sk", - ) - conn.create_table( - TableName=name, - KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], - AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], - ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, - ) - - conn.put_item( - TableName=name, - Item={ - "forum_name": {"S": "LOLCat Forum"}, - "subject": {"S": "Check this out!"}, - "Body": {"S": "http://url_to_lolcat.gif"}, - "SentBy": {"S": "test"}, - "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, - }, - ) - - with pytest.raises(ClientError) as ex: - conn.update_item( - TableName=name, - Key={"forum_name": {"S": "LOLCat Forum"}}, - UpdateExpression="set forum_name=:NewName", - ExpressionAttributeValues={":NewName": {"S": ""}}, - ) - - ex.value.response["Error"]["Code"].should.equal("ValidationException") - ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) - ex.value.response["Error"]["Message"].should.equal( - "One or more parameter values were invalid: An AttributeValue may not contain an empty string" - ) - - -@requires_boto_gte("2.9") -@mock_dynamodb2 -def test_update_item_with_empty_string_no_exception(): +def test_update_item_with_empty_string_attr_no_exception(): name = "TestTable" conn = boto3.client( "dynamodb", @@ -328,6 +324,303 @@ def test_update_item_with_empty_string_no_exception(): ) +@mock_dynamodb2 +def test_item_add_long_string_hash_key_exception(): + name = "TestTable" + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": "x" * HASH_KEY_MAX_LENGTH}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "test"}, + "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, + }, + ) + + with pytest.raises(ClientError) as ex: + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": "x" * (HASH_KEY_MAX_LENGTH + 1)}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "test"}, + "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, + }, + ) + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + # deliberately no space between "of" and "2048" + ex.value.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes" + ) + + +@mock_dynamodb2 +def test_item_add_long_string_nonascii_hash_key_exception(): + name = "TestTable" + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + emoji_b = b"\xf0\x9f\x98\x83" # smile emoji + emoji = emoji_b.decode("utf-8") # 1 character, but 4 bytes + short_enough = emoji * int(HASH_KEY_MAX_LENGTH / len(emoji.encode("utf-8"))) + too_long = "x" + short_enough + + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": short_enough}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "test"}, + "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, + }, + ) + + with pytest.raises(ClientError) as ex: + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": too_long}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "test"}, + "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, + }, + ) + + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + # deliberately no space between "of" and "2048" + ex.value.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes" + ) + + +@mock_dynamodb2 +def test_item_add_long_string_range_key_exception(): + name = "TestTable" + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + conn.create_table( + TableName=name, + KeySchema=[ + {"AttributeName": "forum_name", "KeyType": "HASH"}, + {"AttributeName": "ReceivedTime", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "forum_name", "AttributeType": "S"}, + {"AttributeName": "ReceivedTime", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": "LOLCat Forum"}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "someone@somewhere.edu"}, + "ReceivedTime": {"S": "x" * RANGE_KEY_MAX_LENGTH}, + }, + ) + + with pytest.raises(ClientError) as ex: + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": "LOLCat Forum"}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "someone@somewhere.edu"}, + "ReceivedTime": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)}, + }, + ) + + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.value.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes" + ) + + +@mock_dynamodb2 +def test_item_add_long_string_range_key_exception(): + name = "TestTable" + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + conn.create_table( + TableName=name, + KeySchema=[ + {"AttributeName": "forum_name", "KeyType": "HASH"}, + {"AttributeName": "ReceivedTime", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "forum_name", "AttributeType": "S"}, + {"AttributeName": "ReceivedTime", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": "LOLCat Forum"}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "someone@somewhere.edu"}, + "ReceivedTime": {"S": "x" * RANGE_KEY_MAX_LENGTH}, + }, + ) + + with pytest.raises(ClientError) as ex: + conn.put_item( + TableName=name, + Item={ + "forum_name": {"S": "LOLCat Forum"}, + "subject": {"S": "Check this out!"}, + "Body": {"S": "http://url_to_lolcat.gif"}, + "SentBy": {"S": "someone@somewhere.edu"}, + "ReceivedTime": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)}, + }, + ) + + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.value.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes" + ) + + +@mock_dynamodb2 +def test_update_item_with_long_string_hash_key_exception(): + name = "TestTable" + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + conn.create_table( + TableName=name, + KeySchema=[{"AttributeName": "forum_name", "KeyType": "HASH"}], + AttributeDefinitions=[{"AttributeName": "forum_name", "AttributeType": "S"}], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + + conn.update_item( + TableName=name, + Key={ + "forum_name": {"S": "x" * HASH_KEY_MAX_LENGTH}, + "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, + }, + UpdateExpression="set body=:New", + ExpressionAttributeValues={":New": {"S": "hello"}}, + ) + + with pytest.raises(ClientError) as ex: + conn.update_item( + TableName=name, + Key={ + "forum_name": {"S": "x" * (HASH_KEY_MAX_LENGTH + 1)}, + "ReceivedTime": {"S": "12/9/2011 11:36:03 PM"}, + }, + UpdateExpression="set body=:New", + ExpressionAttributeValues={":New": {"S": "hello"}}, + ) + + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + # deliberately no space between "of" and "2048" + ex.value.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of2048 bytes" + ) + + +@mock_dynamodb2 +def test_update_item_with_long_string_range_key_exception(): + name = "TestTable" + conn = boto3.client( + "dynamodb", + region_name="us-west-2", + aws_access_key_id="ak", + aws_secret_access_key="sk", + ) + conn.create_table( + TableName=name, + KeySchema=[ + {"AttributeName": "forum_name", "KeyType": "HASH"}, + {"AttributeName": "ReceivedTime", "KeyType": "RANGE"}, + ], + AttributeDefinitions=[ + {"AttributeName": "forum_name", "AttributeType": "S"}, + {"AttributeName": "ReceivedTime", "AttributeType": "S"}, + ], + ProvisionedThroughput={"ReadCapacityUnits": 5, "WriteCapacityUnits": 5}, + ) + conn.update_item( + TableName=name, + Key={ + "forum_name": {"S": "Lolcat Forum"}, + "ReceivedTime": {"S": "x" * RANGE_KEY_MAX_LENGTH}, + }, + UpdateExpression="set body=:New", + ExpressionAttributeValues={":New": {"S": "hello"}}, + ) + + with pytest.raises(ClientError) as ex: + conn.update_item( + TableName=name, + Key={ + "forum_name": {"S": "Lolcat Forum"}, + "ReceivedTime": {"S": "x" * (RANGE_KEY_MAX_LENGTH + 1)}, + }, + UpdateExpression="set body=:New", + ExpressionAttributeValues={":New": {"S": "hello"}}, + ) + + ex.value.response["Error"]["Code"].should.equal("ValidationException") + ex.value.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + # deliberately no space between "of" and "2048" + ex.value.response["Error"]["Message"].should.equal( + "One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of 1024 bytes" + ) + + @requires_boto_gte("2.9") @mock_dynamodb2 def test_query_invalid_table():