Improve DDB expressions support4: Execution using AST
Part of structured approach for UpdateExpressions: 1) Expression gets parsed into a tokenlist (tokenized) 2) Tokenlist get transformed to expression tree (AST) 3) The AST gets validated (full semantic correctness) 4) AST gets processed to perform the update -> this commit This commit uses the AST to execute the UpdateExpression. All the existing tests pass. The only tests that have been updated are in test_dynamodb_table_with_range_key.py because they wrongly allow adding a set to a path that doesn't exist. This has been alligend to correspond to the behavior of AWS DynamoDB. This commit will resolve https://github.com/spulec/moto/issues/2806 Multiple tests have been implemented that verify this.
This commit is contained in:
pvbouwel
parent
6a41573eb8
commit
ec731ac901
8 changed files with 1200 additions and 269 deletions
|
|
@ -1,21 +1,17 @@
|
|||
from __future__ import unicode_literals, print_function
|
||||
|
||||
import re
|
||||
from decimal import Decimal
|
||||
|
||||
import six
|
||||
import boto
|
||||
import boto3
|
||||
from boto3.dynamodb.conditions import Attr, Key
|
||||
import re
|
||||
import requests
|
||||
import sure # noqa
|
||||
from moto import mock_dynamodb2, mock_dynamodb2_deprecated
|
||||
from moto.dynamodb2 import dynamodb_backend2, dynamodb_backends2
|
||||
from boto.exception import JSONResponseError
|
||||
from botocore.exceptions import ClientError, ParamValidationError
|
||||
from tests.helpers import requires_boto_gte
|
||||
import tests.backport_assert_raises
|
||||
|
||||
import moto.dynamodb2.comparisons
|
||||
import moto.dynamodb2.models
|
||||
|
|
@ -3221,6 +3217,25 @@ def test_remove_top_level_attribute():
|
|||
result.should.equal({"id": {"S": "foo"}})
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_remove_top_level_attribute_non_existent():
|
||||
"""
|
||||
Remove statements do not require attribute to exist they silently pass
|
||||
"""
|
||||
table_name = "test_remove"
|
||||
client = create_table_with_list(table_name)
|
||||
ddb_item = {"id": {"S": "foo"}, "item": {"S": "bar"}}
|
||||
client.put_item(TableName=table_name, Item=ddb_item)
|
||||
client.update_item(
|
||||
TableName=table_name,
|
||||
Key={"id": {"S": "foo"}},
|
||||
UpdateExpression="REMOVE non_existent_attribute",
|
||||
ExpressionAttributeNames={"#i": "item"},
|
||||
)
|
||||
result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"]
|
||||
result.should.equal(ddb_item)
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_remove_list_index__remove_existing_index():
|
||||
table_name = "test_list_index_access"
|
||||
|
|
@ -4331,3 +4346,251 @@ def test_list_tables_exclusive_start_table_name_empty():
|
|||
resp = client.list_tables(Limit=1, ExclusiveStartTableName="whatever")
|
||||
|
||||
len(resp["TableNames"]).should.equal(0)
|
||||
|
||||
|
||||
def assert_correct_client_error(
|
||||
client_error, code, message_template, message_values=None, braces=None
|
||||
):
|
||||
"""
|
||||
Assert whether a client_error is as expected. Allow for a list of values to be passed into the message
|
||||
|
||||
Args:
|
||||
client_error(ClientError): The ClientError exception that was raised
|
||||
code(str): The code for the error (e.g. ValidationException)
|
||||
message_template(str): Error message template. if message_values is not None then this template has a {values}
|
||||
as placeholder. For example:
|
||||
'Value provided in ExpressionAttributeValues unused in expressions: keys: {values}'
|
||||
message_values(list of str|None): The values that are passed in the error message
|
||||
braces(list of str|None): List of length 2 with opening and closing brace for the values. By default it will be
|
||||
surrounded by curly brackets
|
||||
"""
|
||||
braces = braces or ["{", "}"]
|
||||
assert client_error.response["Error"]["Code"] == code
|
||||
if message_values is not None:
|
||||
values_string = "{open_brace}(?P<values>.*){close_brace}".format(
|
||||
open_brace=braces[0], close_brace=braces[1]
|
||||
)
|
||||
re_msg = re.compile(message_template.format(values=values_string))
|
||||
match_result = re_msg.match(client_error.response["Error"]["Message"])
|
||||
assert match_result is not None
|
||||
values_string = match_result.groupdict()["values"]
|
||||
values = [key for key in values_string.split(", ")]
|
||||
assert len(message_values) == len(values)
|
||||
for value in message_values:
|
||||
assert value in values
|
||||
else:
|
||||
assert client_error.response["Error"]["Message"] == message_template
|
||||
|
||||
|
||||
def create_simple_table_and_return_client():
|
||||
dynamodb = boto3.client("dynamodb", region_name="eu-west-1")
|
||||
dynamodb.create_table(
|
||||
TableName="moto-test",
|
||||
KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
|
||||
AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"},],
|
||||
ProvisionedThroughput={"ReadCapacityUnits": 1, "WriteCapacityUnits": 1},
|
||||
)
|
||||
dynamodb.put_item(
|
||||
TableName="moto-test",
|
||||
Item={"id": {"S": "1"}, "myNum": {"N": "1"}, "MyStr": {"S": "1"},},
|
||||
)
|
||||
return dynamodb
|
||||
|
||||
|
||||
# https://github.com/spulec/moto/issues/2806
|
||||
# https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_UpdateItem.html
|
||||
# #DDB-UpdateItem-request-UpdateExpression
|
||||
@mock_dynamodb2
|
||||
def test_update_item_with_attribute_in_right_hand_side_and_operation():
|
||||
dynamodb = create_simple_table_and_return_client()
|
||||
|
||||
dynamodb.update_item(
|
||||
TableName="moto-test",
|
||||
Key={"id": {"S": "1"}},
|
||||
UpdateExpression="SET myNum = myNum+:val",
|
||||
ExpressionAttributeValues={":val": {"N": "3"}},
|
||||
)
|
||||
|
||||
result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}})
|
||||
assert result["Item"]["myNum"]["N"] == "4"
|
||||
|
||||
dynamodb.update_item(
|
||||
TableName="moto-test",
|
||||
Key={"id": {"S": "1"}},
|
||||
UpdateExpression="SET myNum = myNum - :val",
|
||||
ExpressionAttributeValues={":val": {"N": "1"}},
|
||||
)
|
||||
result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}})
|
||||
assert result["Item"]["myNum"]["N"] == "3"
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_non_existing_attribute_should_raise_exception():
|
||||
"""
|
||||
Does error message get correctly raised if attribute is referenced but it does not exist for the item.
|
||||
"""
|
||||
dynamodb = create_simple_table_and_return_client()
|
||||
|
||||
try:
|
||||
dynamodb.update_item(
|
||||
TableName="moto-test",
|
||||
Key={"id": {"S": "1"}},
|
||||
UpdateExpression="SET MyStr = no_attr + MyStr",
|
||||
)
|
||||
assert False, "Validation exception not thrown"
|
||||
except dynamodb.exceptions.ClientError as e:
|
||||
assert_correct_client_error(
|
||||
e,
|
||||
"ValidationException",
|
||||
"The provided expression refers to an attribute that does not exist in the item",
|
||||
)
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_update_expression_with_plus_in_attribute_name():
|
||||
"""
|
||||
Does error message get correctly raised if attribute contains a plus and is passed in without an AttributeName. And
|
||||
lhs & rhs are not attribute IDs by themselve.
|
||||
"""
|
||||
dynamodb = create_simple_table_and_return_client()
|
||||
|
||||
dynamodb.put_item(
|
||||
TableName="moto-test",
|
||||
Item={"id": {"S": "1"}, "my+Num": {"S": "1"}, "MyStr": {"S": "aaa"},},
|
||||
)
|
||||
try:
|
||||
dynamodb.update_item(
|
||||
TableName="moto-test",
|
||||
Key={"id": {"S": "1"}},
|
||||
UpdateExpression="SET MyStr = my+Num",
|
||||
)
|
||||
assert False, "Validation exception not thrown"
|
||||
except dynamodb.exceptions.ClientError as e:
|
||||
assert_correct_client_error(
|
||||
e,
|
||||
"ValidationException",
|
||||
"The provided expression refers to an attribute that does not exist in the item",
|
||||
)
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_update_expression_with_minus_in_attribute_name():
|
||||
"""
|
||||
Does error message get correctly raised if attribute contains a minus and is passed in without an AttributeName. And
|
||||
lhs & rhs are not attribute IDs by themselve.
|
||||
"""
|
||||
dynamodb = create_simple_table_and_return_client()
|
||||
|
||||
dynamodb.put_item(
|
||||
TableName="moto-test",
|
||||
Item={"id": {"S": "1"}, "my-Num": {"S": "1"}, "MyStr": {"S": "aaa"},},
|
||||
)
|
||||
try:
|
||||
dynamodb.update_item(
|
||||
TableName="moto-test",
|
||||
Key={"id": {"S": "1"}},
|
||||
UpdateExpression="SET MyStr = my-Num",
|
||||
)
|
||||
assert False, "Validation exception not thrown"
|
||||
except dynamodb.exceptions.ClientError as e:
|
||||
assert_correct_client_error(
|
||||
e,
|
||||
"ValidationException",
|
||||
"The provided expression refers to an attribute that does not exist in the item",
|
||||
)
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_update_expression_with_space_in_attribute_name():
|
||||
"""
|
||||
Does error message get correctly raised if attribute contains a space and is passed in without an AttributeName. And
|
||||
lhs & rhs are not attribute IDs by themselves.
|
||||
"""
|
||||
dynamodb = create_simple_table_and_return_client()
|
||||
|
||||
dynamodb.put_item(
|
||||
TableName="moto-test",
|
||||
Item={"id": {"S": "1"}, "my Num": {"S": "1"}, "MyStr": {"S": "aaa"},},
|
||||
)
|
||||
|
||||
try:
|
||||
dynamodb.update_item(
|
||||
TableName="moto-test",
|
||||
Key={"id": {"S": "1"}},
|
||||
UpdateExpression="SET MyStr = my Num",
|
||||
)
|
||||
assert False, "Validation exception not thrown"
|
||||
except dynamodb.exceptions.ClientError as e:
|
||||
assert_raise_syntax_error(e, "Num", "my Num")
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_summing_up_2_strings_raises_exception():
|
||||
"""
|
||||
Update set supports different DynamoDB types but some operations are not supported. For example summing up 2 strings
|
||||
raises an exception. It results in ClientError with code ValidationException:
|
||||
Saying An operand in the update expression has an incorrect data type
|
||||
"""
|
||||
dynamodb = create_simple_table_and_return_client()
|
||||
|
||||
try:
|
||||
dynamodb.update_item(
|
||||
TableName="moto-test",
|
||||
Key={"id": {"S": "1"}},
|
||||
UpdateExpression="SET MyStr = MyStr + MyStr",
|
||||
)
|
||||
assert False, "Validation exception not thrown"
|
||||
except dynamodb.exceptions.ClientError as e:
|
||||
assert_correct_client_error(
|
||||
e,
|
||||
"ValidationException",
|
||||
"An operand in the update expression has an incorrect data type",
|
||||
)
|
||||
|
||||
|
||||
# https://github.com/spulec/moto/issues/2806
|
||||
@mock_dynamodb2
|
||||
def test_update_item_with_attribute_in_right_hand_side():
|
||||
"""
|
||||
After tokenization and building expression make sure referenced attributes are replaced with their current value
|
||||
"""
|
||||
dynamodb = create_simple_table_and_return_client()
|
||||
|
||||
# Make sure there are 2 values
|
||||
dynamodb.put_item(
|
||||
TableName="moto-test",
|
||||
Item={"id": {"S": "1"}, "myVal1": {"S": "Value1"}, "myVal2": {"S": "Value2"}},
|
||||
)
|
||||
|
||||
dynamodb.update_item(
|
||||
TableName="moto-test",
|
||||
Key={"id": {"S": "1"}},
|
||||
UpdateExpression="SET myVal1 = myVal2",
|
||||
)
|
||||
|
||||
result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}})
|
||||
assert result["Item"]["myVal1"]["S"] == result["Item"]["myVal2"]["S"] == "Value2"
|
||||
|
||||
|
||||
@mock_dynamodb2
|
||||
def test_multiple_updates():
|
||||
dynamodb = create_simple_table_and_return_client()
|
||||
dynamodb.put_item(
|
||||
TableName="moto-test",
|
||||
Item={"id": {"S": "1"}, "myNum": {"N": "1"}, "path": {"N": "6"}},
|
||||
)
|
||||
dynamodb.update_item(
|
||||
TableName="moto-test",
|
||||
Key={"id": {"S": "1"}},
|
||||
UpdateExpression="SET myNum = #p + :val, newAttr = myNum",
|
||||
ExpressionAttributeValues={":val": {"N": "1"}},
|
||||
ExpressionAttributeNames={"#p": "path"},
|
||||
)
|
||||
result = dynamodb.get_item(TableName="moto-test", Key={"id": {"S": "1"}})["Item"]
|
||||
expected_result = {
|
||||
"myNum": {"N": "7"},
|
||||
"newAttr": {"N": "1"},
|
||||
"path": {"N": "6"},
|
||||
"id": {"S": "1"},
|
||||
}
|
||||
assert result == expected_result
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue