catalin/moto
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Enforce dynamodb key size limit [solves #3866] (#3888)

Browse source 
* add tests for dynamodb max key size

correct too-large error for ddb key

* remove unnecessary requires_boto_gte decorator from ddb tests

* remove literal emoji from ddb test

* implement dynamodb key limits, WIP

* correct direction of dynamodb range key length check

* fix tests for dynamodb max key size check

* catch ddb validation errors and rethrow properly

* finish ddb key size limit fixes

* fix linting

* handle unicode in v2.7 tests

* fix encoding issue in py2.7 for ddb

* linting

* Python2/3 compatability

Co-authored-by: Bert Blommers <info@bertblommers.nl>
This commit is contained in:
Matthew Davis 2021-04-30 22:47:47 +10:00 • committed by GitHub
commit d6384fcb35
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 359 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

6
moto/core/responses.py
View file

@ -149,7 +149,7 @@ class ActionAuthenticatorMixin(object):
if settings.TEST_SERVER_MODE:
response = requests.post(
"http://localhost:5000/moto-api/reset-auth",
data=str(initial_no_auth_action_count).encode(),
data=str(initial_no_auth_action_count).encode("utf-8"),
)
original_initial_no_auth_action_count = response.json()[
"PREVIOUS_INITIAL_NO_AUTH_ACTION_COUNT"
@ -167,7 +167,9 @@ class ActionAuthenticatorMixin(object):
if settings.TEST_SERVER_MODE:
requests.post(
"http://localhost:5000/moto-api/reset-auth",
data=str(original_initial_no_auth_action_count).encode(),
data=str(original_initial_no_auth_action_count).encode(
"utf-8"
),
)
else:
ActionAuthenticatorMixin.request_count = original_request_count

2
moto/core/utils.py
View file

@ -230,7 +230,7 @@ def unix_time_millis(dt=None):
def gen_amz_crc32(response, headerdict=None):
if not isinstance(response, bytes):
response = response.encode()
response = response.encode("utf-8")
crc = binascii.crc32(response)
if six.PY2:

22
moto/dynamodb2/exceptions.py
View file

@ -1,3 +1,6 @@
from moto.dynamodb2.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH
class InvalidIndexNameError(ValueError):
pass
@ -133,6 +136,25 @@ class ItemSizeToUpdateTooLarge(MockValidationException):
)
class HashKeyTooLong(MockValidationException):
# deliberately no space between of and {lim}
key_too_large_msg = "One or more parameter values were invalid: Size of hashkey has exceeded the maximum size limit of{lim} bytes".format(
lim=HASH_KEY_MAX_LENGTH
)
def __init__(self):
super(HashKeyTooLong, self).__init__(self.key_too_large_msg)
class RangeKeyTooLong(MockValidationException):
key_too_large_msg = "One or more parameter values were invalid: Aggregated size of all range keys has exceeded the size limit of {lim} bytes".format(
lim=RANGE_KEY_MAX_LENGTH
)
def __init__(self):
super(RangeKeyTooLong, self).__init__(self.key_too_large_msg)
class IncorrectOperandType(InvalidUpdateExpression):
inv_operand_msg = "Incorrect operand type for operator or function; operator or function: {f}, operand type: {t}"

5
moto/dynamodb2/limits.py Normal file
View file

@ -0,0 +1,5 @@
# https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/Limits.html#limits-partition-sort-keys
# measured in bytes
# <= not <
HASH_KEY_MAX_LENGTH = 2048
RANGE_KEY_MAX_LENGTH = 1024

13
moto/dynamodb2/models/__init__.py
View file

@ -18,6 +18,8 @@ from moto.dynamodb2.exceptions import (
InvalidIndexNameError,
ItemSizeTooLarge,
ItemSizeToUpdateTooLarge,
HashKeyTooLong,
RangeKeyTooLong,
ConditionalCheckFailed,
TransactionCanceledException,
EmptyKeyAttributeException,
@ -27,6 +29,7 @@ from moto.dynamodb2.models.dynamo_type import DynamoType
from moto.dynamodb2.parsing.executors import UpdateExpressionExecutor
from moto.dynamodb2.parsing.expressions import UpdateExpressionParser
from moto.dynamodb2.parsing.validators import UpdateExpressionValidator
from moto.dynamodb2.limits import HASH_KEY_MAX_LENGTH, RANGE_KEY_MAX_LENGTH
class DynamoJsonEncoder(json.JSONEncoder):
@ -70,6 +73,11 @@ class Item(BaseModel):
self.range_key = range_key
self.range_key_type = range_key_type
if hash_key and hash_key.size() > HASH_KEY_MAX_LENGTH:
raise HashKeyTooLong
if range_key and (range_key.size() > RANGE_KEY_MAX_LENGTH):
raise RangeKeyTooLong
self.attrs = LimitedSizeDict()
for key, value in attrs.items():
self.attrs[key] = DynamoType(value)
@ -1309,13 +1317,14 @@ class DynamoDBBackend(BaseBackend):
item.validate_no_empty_key_values(attribute_updates, table.key_attributes)
if update_expression:
validated_ast = UpdateExpressionValidator(
validator = UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=expression_attribute_names,
expression_attribute_values=expression_attribute_values,
item=item,
table=table,
).validate()
)
validated_ast = validator.validate()
try:
UpdateExpressionExecutor(
validated_ast, item, expression_attribute_names

2
moto/dynamodb2/models/utilities.py
View file

@ -2,7 +2,7 @@ import re
def bytesize(val):
return len(str(val).encode("utf-8"))
return len(val.encode("utf-8"))
def attribute_is_list(attr):

5
moto/dynamodb2/responses.py
View file

@ -11,7 +11,6 @@ from moto.core.responses import BaseResponse
from moto.core.utils import camelcase_to_underscores, amz_crc32, amzn_request_id
from .exceptions import (
InvalidIndexNameError,
ItemSizeTooLarge,
MockValidationException,
TransactionCanceledException,
)
@ -296,9 +295,9 @@ class DynamoHandler(BaseResponse):
expression_attribute_values,
overwrite,
)
except ItemSizeTooLarge:
except MockValidationException as mve:
er = "com.amazonaws.dynamodb.v20111205#ValidationException"
return self.error(er, ItemSizeTooLarge.item_size_too_large_msg)
return self.error(er, mve.exception_msg)
except KeyError as ke:
er = "com.amazonaws.dynamodb.v20111205#ValidationException"
return self.error(er, ke.args[0])