catalin/moto
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge branch 'master' into feature/dynamodb_item_limit

Browse source
This commit is contained in:
Steve Pulec 2020-04-25 18:38:08 -05:00 • committed by GitHub
commit 8595493aee
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
53 changed files with 4651 additions and 122 deletions
Download patch file Download diff file Expand all files Collapse all files

16
tests/test_apigateway/test_apigateway.py
View file

@ -69,6 +69,22 @@ def test_create_rest_api_with_tags():
response["tags"].should.equal({"MY_TAG1": "MY_VALUE1"})
@mock_apigateway
def test_create_rest_api_with_policy():
client = boto3.client("apigateway", region_name="us-west-2")
policy = '{"Version": "2012-10-17","Statement": []}'
response = client.create_rest_api(
name="my_api", description="this is my api", policy=policy
)
api_id = response["id"]
response = client.get_rest_api(restApiId=api_id)
assert "policy" in response
response["policy"].should.equal(policy)
@mock_apigateway
def test_create_rest_api_invalid_apikeysource():
client = boto3.client("apigateway", region_name="us-west-2")

14
tests/test_cloudformation/test_cloudformation_stack_integration.py
View file

@ -495,7 +495,7 @@ def test_autoscaling_group_with_elb():
"my-as-group": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Properties": {
"AvailabilityZones": ["us-east1"],
"AvailabilityZones": ["us-east-1a"],
"LaunchConfigurationName": {"Ref": "my-launch-config"},
"MinSize": "2",
"MaxSize": "2",
@ -522,7 +522,7 @@ def test_autoscaling_group_with_elb():
"my-elb": {
"Type": "AWS::ElasticLoadBalancing::LoadBalancer",
"Properties": {
"AvailabilityZones": ["us-east1"],
"AvailabilityZones": ["us-east-1a"],
"Listeners": [
{
"LoadBalancerPort": "80",
@ -545,10 +545,10 @@ def test_autoscaling_group_with_elb():
web_setup_template_json = json.dumps(web_setup_template)
conn = boto.cloudformation.connect_to_region("us-west-1")
conn = boto.cloudformation.connect_to_region("us-east-1")
conn.create_stack("web_stack", template_body=web_setup_template_json)
autoscale_conn = boto.ec2.autoscale.connect_to_region("us-west-1")
autoscale_conn = boto.ec2.autoscale.connect_to_region("us-east-1")
autoscale_group = autoscale_conn.get_all_groups()[0]
autoscale_group.launch_config_name.should.contain("my-launch-config")
autoscale_group.load_balancers[0].should.equal("my-elb")
@ -557,7 +557,7 @@ def test_autoscaling_group_with_elb():
autoscale_conn.get_all_launch_configurations().should.have.length_of(1)
# Confirm the ELB was actually created
elb_conn = boto.ec2.elb.connect_to_region("us-west-1")
elb_conn = boto.ec2.elb.connect_to_region("us-east-1")
elb_conn.get_all_load_balancers().should.have.length_of(1)
stack = conn.describe_stacks()[0]
@ -584,7 +584,7 @@ def test_autoscaling_group_with_elb():
elb_resource.physical_resource_id.should.contain("my-elb")
# confirm the instances were created with the right tags
ec2_conn = boto.ec2.connect_to_region("us-west-1")
ec2_conn = boto.ec2.connect_to_region("us-east-1")
reservations = ec2_conn.get_all_reservations()
len(reservations).should.equal(1)
reservation = reservations[0]
@ -604,7 +604,7 @@ def test_autoscaling_group_update():
"my-as-group": {
"Type": "AWS::AutoScaling::AutoScalingGroup",
"Properties": {
"AvailabilityZones": ["us-west-1"],
"AvailabilityZones": ["us-west-1a"],
"LaunchConfigurationName": {"Ref": "my-launch-config"},
"MinSize": "2",
"MaxSize": "2",

40
tests/test_cloudwatch/test_cloudwatch.py
View file

@ -1,9 +1,10 @@
import boto
from boto.ec2.cloudwatch.alarm import MetricAlarm
from boto.s3.key import Key
from datetime import datetime
import sure # noqa
from moto import mock_cloudwatch_deprecated
from moto import mock_cloudwatch_deprecated, mock_s3_deprecated
def alarm_fixture(name="tester", action=None):
@ -83,10 +84,11 @@ def test_put_metric_data():
)
metrics = conn.list_metrics()
metrics.should.have.length_of(1)
metric_names = [m for m in metrics if m.name == "metric"]
metric_names.should.have(1)
metric = metrics[0]
metric.namespace.should.equal("tester")
metric.name.should.equal("metric")
metric.name.should.equal("Metric:metric")
dict(metric.dimensions).should.equal({"InstanceId": ["i-0123456,i-0123457"]})
@ -153,3 +155,35 @@ def test_get_metric_statistics():
datapoint = datapoints[0]
datapoint.should.have.key("Minimum").which.should.equal(1.5)
datapoint.should.have.key("Timestamp").which.should.equal(metric_timestamp)
@mock_s3_deprecated
@mock_cloudwatch_deprecated
def test_cloudwatch_return_s3_metrics():
region = "us-east-1"
cw = boto.ec2.cloudwatch.connect_to_region(region)
s3 = boto.s3.connect_to_region(region)
bucket_name_1 = "test-bucket-1"
bucket_name_2 = "test-bucket-2"
bucket1 = s3.create_bucket(bucket_name=bucket_name_1)
key = Key(bucket1)
key.key = "the-key"
key.set_contents_from_string("foobar" * 4)
s3.create_bucket(bucket_name=bucket_name_2)
metrics_s3_bucket_1 = cw.list_metrics(dimensions={"BucketName": bucket_name_1})
# Verify that the OOTB S3 metrics are available for the created buckets
len(metrics_s3_bucket_1).should.be(2)
metric_names = [m.name for m in metrics_s3_bucket_1]
sorted(metric_names).should.equal(
["Metric:BucketSizeBytes", "Metric:NumberOfObjects"]
)
# Explicit clean up - the metrics for these buckets are messing with subsequent tests
key.delete()
s3.delete_bucket(bucket_name_1)
s3.delete_bucket(bucket_name_2)

16
tests/test_cloudwatch/test_cloudwatch_boto3.py
View file

@ -154,7 +154,7 @@ def test_put_metric_data_no_dimensions():
metrics.should.have.length_of(1)
metric = metrics[0]
metric["Namespace"].should.equal("tester")
metric["MetricName"].should.equal("metric")
metric["MetricName"].should.equal("Metric:metric")
@mock_cloudwatch
@ -182,7 +182,7 @@ def test_put_metric_data_with_statistics():
metrics.should.have.length_of(1)
metric = metrics[0]
metric["Namespace"].should.equal("tester")
metric["MetricName"].should.equal("statmetric")
metric["MetricName"].should.equal("Metric:statmetric")
# TODO: test statistics - https://github.com/spulec/moto/issues/1615
@ -233,8 +233,16 @@ def test_list_metrics():
# Verify format
res.should.equal(
[
{u"Namespace": "list_test_1/", u"Dimensions": [], u"MetricName": "metric1"},
{u"Namespace": "list_test_1/", u"Dimensions": [], u"MetricName": "metric1"},
{
u"Namespace": "list_test_1/",
u"Dimensions": [],
u"MetricName": "Metric:metric1",
},
{
u"Namespace": "list_test_1/",
u"Dimensions": [],
u"MetricName": "Metric:metric1",
},
]
)
# Verify unknown namespace still has no results

140
tests/test_dynamodb2/test_dynamodb.py
View file

@ -1,5 +1,6 @@
from __future__ import unicode_literals, print_function
import re
from decimal import Decimal
import six
@ -1453,6 +1454,13 @@ def test_filter_expression():
filter_expr.expr(row1).should.be(True)
filter_expr.expr(row2).should.be(False)
# lowercase AND test
filter_expr = moto.dynamodb2.comparisons.get_filter_expression(
"Id > :v0 and Subs < :v1", {}, {":v0": {"N": "5"}, ":v1": {"N": "7"}}
)
filter_expr.expr(row1).should.be(True)
filter_expr.expr(row2).should.be(False)
# OR test
filter_expr = moto.dynamodb2.comparisons.get_filter_expression(
"Id = :v0 OR Id=:v1", {}, {":v0": {"N": "5"}, ":v1": {"N": "8"}}
@ -2146,13 +2154,33 @@ def test_update_item_on_map():
# Nonexistent nested attributes are supported for existing top-level attributes.
table.update_item(
Key={"forum_name": "the-key", "subject": "123"},
UpdateExpression="SET body.#nested.#data = :tb, body.nested.#nonexistentnested.#data = :tb2",
UpdateExpression="SET body.#nested.#data = :tb",
ExpressionAttributeNames={"#nested": "nested", "#data": "data",},
ExpressionAttributeValues={":tb": "new_value"},
)
# Running this against AWS DDB gives an exception so make sure it also fails.:
with assert_raises(client.exceptions.ClientError):
# botocore.exceptions.ClientError: An error occurred (ValidationException) when calling the UpdateItem
# operation: The document path provided in the update expression is invalid for update
table.update_item(
Key={"forum_name": "the-key", "subject": "123"},
UpdateExpression="SET body.#nested.#nonexistentnested.#data = :tb2",
ExpressionAttributeNames={
"#nested": "nested",
"#nonexistentnested": "nonexistentnested",
"#data": "data",
},
ExpressionAttributeValues={":tb2": "other_value"},
)
table.update_item(
Key={"forum_name": "the-key", "subject": "123"},
UpdateExpression="SET body.#nested.#nonexistentnested = :tb2",
ExpressionAttributeNames={
"#nested": "nested",
"#nonexistentnested": "nonexistentnested",
"#data": "data",
},
ExpressionAttributeValues={":tb": "new_value", ":tb2": "other_value"},
ExpressionAttributeValues={":tb2": {"data": "other_value"}},
)
resp = table.scan()
@ -2160,8 +2188,8 @@ def test_update_item_on_map():
{"nested": {"data": "new_value", "nonexistentnested": {"data": "other_value"}}}
)
# Test nested value for a nonexistent attribute.
with assert_raises(client.exceptions.ConditionalCheckFailedException):
# Test nested value for a nonexistent attribute throws a ClientError.
with assert_raises(client.exceptions.ClientError):
table.update_item(
Key={"forum_name": "the-key", "subject": "123"},
UpdateExpression="SET nonexistent.#nested = :tb",
@ -2764,7 +2792,7 @@ def test_query_gsi_with_range_key():
res = dynamodb.query(
TableName="test",
IndexName="test_gsi",
KeyConditionExpression="gsi_hash_key = :gsi_hash_key AND gsi_range_key = :gsi_range_key",
KeyConditionExpression="gsi_hash_key = :gsi_hash_key and gsi_range_key = :gsi_range_key",
ExpressionAttributeValues={
":gsi_hash_key": {"S": "key1"},
":gsi_range_key": {"S": "range1"},
@ -3183,7 +3211,10 @@ def test_remove_top_level_attribute():
TableName=table_name, Item={"id": {"S": "foo"}, "item": {"S": "bar"}}
)
client.update_item(
TableName=table_name, Key={"id": {"S": "foo"}}, UpdateExpression="REMOVE item"
TableName=table_name,
Key={"id": {"S": "foo"}},
UpdateExpression="REMOVE #i",
ExpressionAttributeNames={"#i": "item"},
)
#
result = client.get_item(TableName=table_name, Key={"id": {"S": "foo"}})["Item"]
@ -3358,21 +3389,21 @@ def test_item_size_is_under_400KB():
assert_failure_due_to_item_size(
func=client.put_item,
TableName="moto-test",
Item={"id": {"S": "foo"}, "item": {"S": large_item}},
Item={"id": {"S": "foo"}, "cont": {"S": large_item}},
)
assert_failure_due_to_item_size(
func=table.put_item, Item={"id": "bar", "item": large_item}
func=table.put_item, Item={"id": "bar", "cont": large_item}
)
assert_failure_due_to_item_size(
assert_failure_due_to_item_size_to_update(
func=client.update_item,
TableName="moto-test",
Key={"id": {"S": "foo2"}},
UpdateExpression="set item=:Item",
UpdateExpression="set cont=:Item",
ExpressionAttributeValues={":Item": {"S": large_item}},
)
# Assert op fails when updating a nested item
assert_failure_due_to_item_size(
func=table.put_item, Item={"id": "bar", "itemlist": [{"item": large_item}]}
func=table.put_item, Item={"id": "bar", "itemlist": [{"cont": large_item}]}
)
assert_failure_due_to_item_size(
func=client.put_item,
@ -3393,6 +3424,15 @@ def assert_failure_due_to_item_size(func, **kwargs):
)
def assert_failure_due_to_item_size_to_update(func, **kwargs):
with assert_raises(ClientError) as ex:
func(**kwargs)
ex.exception.response["Error"]["Code"].should.equal("ValidationException")
ex.exception.response["Error"]["Message"].should.equal(
"Item size to update has exceeded the maximum allowed size"
)
@mock_dynamodb2
# https://docs.aws.amazon.com/amazondynamodb/latest/APIReference/API_Query.html#DDB-Query-request-KeyConditionExpression
def test_hash_key_cannot_use_begins_with_operations():
@ -4215,3 +4255,79 @@ def test_dynamodb_max_1mb_limit():
# We shouldn't get everything back - the total result set is well over 1MB
len(items).should.be.greater_than(response["Count"])
response["LastEvaluatedKey"].shouldnt.be(None)
def assert_raise_syntax_error(client_error, token, near):
"""
Assert whether a client_error is as expected Syntax error. Syntax error looks like: `syntax_error_template`
Args:
client_error(ClientError): The ClientError exception that was raised
token(str): The token that ws unexpected
near(str): The part in the expression that shows where the error occurs it generally has the preceding token the
optional separation and the problematic token.
"""
syntax_error_template = (
'Invalid UpdateExpression: Syntax error; token: "{token}", near: "{near}"'
)
expected_syntax_error = syntax_error_template.format(token=token, near=near)
assert client_error.response["Error"]["Code"] == "ValidationException"
assert expected_syntax_error == client_error.response["Error"]["Message"]
@mock_dynamodb2
def test_update_expression_with_numeric_literal_instead_of_value():
"""
DynamoDB requires literals to be passed in as values. If they are put literally in the expression a token error will
be raised
"""
dynamodb = boto3.client("dynamodb", region_name="eu-west-1")
dynamodb.create_table(
TableName="moto-test",
KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}],
)
try:
dynamodb.update_item(
TableName="moto-test",
Key={"id": {"S": "1"}},
UpdateExpression="SET MyStr = myNum + 1",
)
assert False, "Validation exception not thrown"
except dynamodb.exceptions.ClientError as e:
assert_raise_syntax_error(e, "1", "+ 1")
@mock_dynamodb2
def test_update_expression_with_multiple_set_clauses_must_be_comma_separated():
"""
An UpdateExpression can have multiple set clauses but if they are passed in without the separating comma.
"""
dynamodb = boto3.client("dynamodb", region_name="eu-west-1")
dynamodb.create_table(
TableName="moto-test",
KeySchema=[{"AttributeName": "id", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "id", "AttributeType": "S"}],
)
try:
dynamodb.update_item(
TableName="moto-test",
Key={"id": {"S": "1"}},
UpdateExpression="SET MyStr = myNum Mystr2 myNum2",
)
assert False, "Validation exception not thrown"
except dynamodb.exceptions.ClientError as e:
assert_raise_syntax_error(e, "Mystr2", "myNum Mystr2 myNum2")
@mock_dynamodb2
def test_list_tables_exclusive_start_table_name_empty():
client = boto3.client("dynamodb", region_name="us-east-1")
resp = client.list_tables(Limit=1, ExclusiveStartTableName="whatever")
len(resp["TableNames"]).should.equal(0)

259
tests/test_dynamodb2/test_dynamodb_expression_tokenizer.py Normal file
View file

@ -0,0 +1,259 @@
from moto.dynamodb2.exceptions import (
InvalidTokenException,
InvalidExpressionAttributeNameKey,
)
from moto.dynamodb2.parsing.tokens import ExpressionTokenizer, Token
def test_expression_tokenizer_single_set_action():
set_action = "SET attrName = :attrValue"
token_list = ExpressionTokenizer.make_list(set_action)
assert token_list == [
Token(Token.ATTRIBUTE, "SET"),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE, "attrName"),
Token(Token.WHITESPACE, " "),
Token(Token.EQUAL_SIGN, "="),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE_VALUE, ":attrValue"),
]
def test_expression_tokenizer_single_set_action_leading_space():
set_action = "Set attrName = :attrValue"
token_list = ExpressionTokenizer.make_list(set_action)
assert token_list == [
Token(Token.ATTRIBUTE, "Set"),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE, "attrName"),
Token(Token.WHITESPACE, " "),
Token(Token.EQUAL_SIGN, "="),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE_VALUE, ":attrValue"),
]
def test_expression_tokenizer_single_set_action_attribute_name_leading_space():
set_action = "SET #a = :attrValue"
token_list = ExpressionTokenizer.make_list(set_action)
assert token_list == [
Token(Token.ATTRIBUTE, "SET"),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE_NAME, "#a"),
Token(Token.WHITESPACE, " "),
Token(Token.EQUAL_SIGN, "="),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE_VALUE, ":attrValue"),
]
def test_expression_tokenizer_single_set_action_trailing_space():
set_action = "SET attrName = :attrValue "
token_list = ExpressionTokenizer.make_list(set_action)
assert token_list == [
Token(Token.ATTRIBUTE, "SET"),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE, "attrName"),
Token(Token.WHITESPACE, " "),
Token(Token.EQUAL_SIGN, "="),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE_VALUE, ":attrValue"),
Token(Token.WHITESPACE, " "),
]
def test_expression_tokenizer_single_set_action_multi_spaces():
set_action = "SET attrName = :attrValue "
token_list = ExpressionTokenizer.make_list(set_action)
assert token_list == [
Token(Token.ATTRIBUTE, "SET"),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE, "attrName"),
Token(Token.WHITESPACE, " "),
Token(Token.EQUAL_SIGN, "="),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE_VALUE, ":attrValue"),
Token(Token.WHITESPACE, " "),
]
def test_expression_tokenizer_single_set_action_with_numbers_in_identifiers():
set_action = "SET attrName3 = :attr3Value"
token_list = ExpressionTokenizer.make_list(set_action)
assert token_list == [
Token(Token.ATTRIBUTE, "SET"),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE, "attrName3"),
Token(Token.WHITESPACE, " "),
Token(Token.EQUAL_SIGN, "="),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE_VALUE, ":attr3Value"),
]
def test_expression_tokenizer_single_set_action_with_underscore_in_identifier():
set_action = "SET attr_Name = :attr_Value"
token_list = ExpressionTokenizer.make_list(set_action)
assert token_list == [
Token(Token.ATTRIBUTE, "SET"),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE, "attr_Name"),
Token(Token.WHITESPACE, " "),
Token(Token.EQUAL_SIGN, "="),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE_VALUE, ":attr_Value"),
]
def test_expression_tokenizer_leading_underscore_in_attribute_name_expression():
"""Leading underscore is not allowed for an attribute name"""
set_action = "SET attrName = _idid"
try:
ExpressionTokenizer.make_list(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "_"
assert te.near == "= _idid"
def test_expression_tokenizer_leading_underscore_in_attribute_value_expression():
"""Leading underscore is allowed in an attribute value"""
set_action = "SET attrName = :_attrValue"
token_list = ExpressionTokenizer.make_list(set_action)
assert token_list == [
Token(Token.ATTRIBUTE, "SET"),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE, "attrName"),
Token(Token.WHITESPACE, " "),
Token(Token.EQUAL_SIGN, "="),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE_VALUE, ":_attrValue"),
]
def test_expression_tokenizer_single_set_action_nested_attribute():
set_action = "SET attrName.elem = :attrValue"
token_list = ExpressionTokenizer.make_list(set_action)
assert token_list == [
Token(Token.ATTRIBUTE, "SET"),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE, "attrName"),
Token(Token.DOT, "."),
Token(Token.ATTRIBUTE, "elem"),
Token(Token.WHITESPACE, " "),
Token(Token.EQUAL_SIGN, "="),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE_VALUE, ":attrValue"),
]
def test_expression_tokenizer_list_index_with_sub_attribute():
set_action = "SET itemmap.itemlist[1].foos=:Item"
token_list = ExpressionTokenizer.make_list(set_action)
assert token_list == [
Token(Token.ATTRIBUTE, "SET"),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE, "itemmap"),
Token(Token.DOT, "."),
Token(Token.ATTRIBUTE, "itemlist"),
Token(Token.OPEN_SQUARE_BRACKET, "["),
Token(Token.NUMBER, "1"),
Token(Token.CLOSE_SQUARE_BRACKET, "]"),
Token(Token.DOT, "."),
Token(Token.ATTRIBUTE, "foos"),
Token(Token.EQUAL_SIGN, "="),
Token(Token.ATTRIBUTE_VALUE, ":Item"),
]
def test_expression_tokenizer_list_index_surrounded_with_whitespace():
set_action = "SET itemlist[ 1 ]=:Item"
token_list = ExpressionTokenizer.make_list(set_action)
assert token_list == [
Token(Token.ATTRIBUTE, "SET"),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE, "itemlist"),
Token(Token.OPEN_SQUARE_BRACKET, "["),
Token(Token.WHITESPACE, " "),
Token(Token.NUMBER, "1"),
Token(Token.WHITESPACE, " "),
Token(Token.CLOSE_SQUARE_BRACKET, "]"),
Token(Token.EQUAL_SIGN, "="),
Token(Token.ATTRIBUTE_VALUE, ":Item"),
]
def test_expression_tokenizer_single_set_action_attribute_name_invalid_key():
"""
ExpressionAttributeNames contains invalid key: Syntax error; key: "#va#l2"
"""
set_action = "SET #va#l2 = 3"
try:
ExpressionTokenizer.make_list(set_action)
assert False, "Exception not raised correctly"
except InvalidExpressionAttributeNameKey as e:
assert e.key == "#va#l2"
def test_expression_tokenizer_single_set_action_attribute_name_invalid_key_double_hash():
"""
ExpressionAttributeNames contains invalid key: Syntax error; key: "#va#l"
"""
set_action = "SET #va#l = 3"
try:
ExpressionTokenizer.make_list(set_action)
assert False, "Exception not raised correctly"
except InvalidExpressionAttributeNameKey as e:
assert e.key == "#va#l"
def test_expression_tokenizer_single_set_action_attribute_name_valid_key():
set_action = "SET attr=#val2"
token_list = ExpressionTokenizer.make_list(set_action)
assert token_list == [
Token(Token.ATTRIBUTE, "SET"),
Token(Token.WHITESPACE, " "),
Token(Token.ATTRIBUTE, "attr"),
Token(Token.EQUAL_SIGN, "="),
Token(Token.ATTRIBUTE_NAME, "#val2"),
]
def test_expression_tokenizer_just_a_pipe():
set_action = "|"
try:
ExpressionTokenizer.make_list(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "|"
assert te.near == "|"
def test_expression_tokenizer_just_a_pipe_with_leading_white_spaces():
set_action = " |"
try:
ExpressionTokenizer.make_list(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "|"
assert te.near == " |"
def test_expression_tokenizer_just_a_pipe_for_set_expression():
set_action = "SET|"
try:
ExpressionTokenizer.make_list(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "|"
assert te.near == "SET|"
def test_expression_tokenizer_just_an_attribute_and_a_pipe_for_set_expression():
set_action = "SET a|"
try:
ExpressionTokenizer.make_list(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "|"
assert te.near == "a|"

405
tests/test_dynamodb2/test_dynamodb_expressions.py Normal file
View file

@ -0,0 +1,405 @@
from moto.dynamodb2.exceptions import InvalidTokenException
from moto.dynamodb2.parsing.expressions import UpdateExpressionParser
from moto.dynamodb2.parsing.reserved_keywords import ReservedKeywords
def test_get_reserved_keywords():
reserved_keywords = ReservedKeywords.get_reserved_keywords()
assert "SET" in reserved_keywords
assert "DELETE" in reserved_keywords
assert "ADD" in reserved_keywords
# REMOVE is not part of the list of reserved keywords.
assert "REMOVE" not in reserved_keywords
def test_update_expression_numeric_literal_in_expression():
set_action = "SET attrName = 3"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "3"
assert te.near == "= 3"
def test_expression_tokenizer_multi_number_numeric_literal_in_expression():
set_action = "SET attrName = 34"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "34"
assert te.near == "= 34"
def test_expression_tokenizer_numeric_literal_unclosed_square_bracket():
set_action = "SET MyStr[ 3"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "<EOF>"
assert te.near == "3"
def test_expression_tokenizer_wrong_closing_bracket_with_space():
set_action = "SET MyStr[3 )"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == ")"
assert te.near == "3 )"
def test_expression_tokenizer_wrong_closing_bracket():
set_action = "SET MyStr[3)"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == ")"
assert te.near == "3)"
def test_expression_tokenizer_only_numeric_literal_for_set():
set_action = "SET 2"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "2"
assert te.near == "SET 2"
def test_expression_tokenizer_only_numeric_literal():
set_action = "2"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "2"
assert te.near == "2"
def test_expression_tokenizer_set_closing_round_bracket():
set_action = "SET )"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == ")"
assert te.near == "SET )"
def test_expression_tokenizer_set_closing_followed_by_numeric_literal():
set_action = "SET ) 3"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == ")"
assert te.near == "SET ) 3"
def test_expression_tokenizer_numeric_literal_unclosed_square_bracket_trailing_space():
set_action = "SET MyStr[ 3 "
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "<EOF>"
assert te.near == "3 "
def test_expression_tokenizer_unbalanced_round_brackets_only_opening():
set_action = "SET MyStr = (:_val"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "<EOF>"
assert te.near == ":_val"
def test_expression_tokenizer_unbalanced_round_brackets_only_opening_trailing_space():
set_action = "SET MyStr = (:_val "
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "<EOF>"
assert te.near == ":_val "
def test_expression_tokenizer_unbalanced_square_brackets_only_opening():
set_action = "SET MyStr = [:_val"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "["
assert te.near == "= [:_val"
def test_expression_tokenizer_unbalanced_square_brackets_only_opening_trailing_spaces():
set_action = "SET MyStr = [:_val "
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "["
assert te.near == "= [:_val"
def test_expression_tokenizer_unbalanced_round_brackets_multiple_opening():
set_action = "SET MyStr = (:_val + (:val2"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "<EOF>"
assert te.near == ":val2"
def test_expression_tokenizer_unbalanced_round_brackets_only_closing():
set_action = "SET MyStr = ):_val"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == ")"
assert te.near == "= ):_val"
def test_expression_tokenizer_unbalanced_square_brackets_only_closing():
set_action = "SET MyStr = ]:_val"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "]"
assert te.near == "= ]:_val"
def test_expression_tokenizer_unbalanced_round_brackets_only_closing_followed_by_other_parts():
set_action = "SET MyStr = ):_val + :val2"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == ")"
assert te.near == "= ):_val"
def test_update_expression_starts_with_keyword_reset_followed_by_identifier():
update_expression = "RESET NonExistent"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "RESET"
assert te.near == "RESET NonExistent"
def test_update_expression_starts_with_keyword_reset_followed_by_identifier_and_value():
update_expression = "RESET NonExistent value"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "RESET"
assert te.near == "RESET NonExistent"
def test_update_expression_starts_with_leading_spaces_and_keyword_reset_followed_by_identifier_and_value():
update_expression = " RESET NonExistent value"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "RESET"
assert te.near == " RESET NonExistent"
def test_update_expression_with_only_keyword_reset():
update_expression = "RESET"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "RESET"
assert te.near == "RESET"
def test_update_nested_expression_with_selector_just_should_fail_parsing_at_numeric_literal_value():
update_expression = "SET a[0].b = 5"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "5"
assert te.near == "= 5"
def test_update_nested_expression_with_selector_and_spaces_should_only_fail_parsing_at_numeric_literal_value():
update_expression = "SET a [ 2 ]. b = 5"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "5"
assert te.near == "= 5"
def test_update_nested_expression_with_double_selector_and_spaces_should_only_fail_parsing_at_numeric_literal_value():
update_expression = "SET a [2][ 3 ]. b = 5"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "5"
assert te.near == "= 5"
def test_update_nested_expression_should_only_fail_parsing_at_numeric_literal_value():
update_expression = "SET a . b = 5"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "5"
assert te.near == "= 5"
def test_nested_selectors_in_update_expression_should_fail_at_nesting():
update_expression = "SET a [ [2] ]. b = 5"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "["
assert te.near == "[ [2"
def test_update_expression_number_in_selector_cannot_be_splite():
update_expression = "SET a [2 1]. b = 5"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "1"
assert te.near == "2 1]"
def test_update_expression_cannot_have_successive_attributes():
update_expression = "SET #a a = 5"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "a"
assert te.near == "#a a ="
def test_update_expression_path_with_both_attribute_and_attribute_name_should_only_fail_at_numeric_value():
update_expression = "SET #a.a = 5"
try:
UpdateExpressionParser.make(update_expression)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "5"
assert te.near == "= 5"
def test_expression_tokenizer_2_same_operators_back_to_back():
set_action = "SET MyStr = NoExist + + :_val "
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "+"
assert te.near == "+ + :_val"
def test_expression_tokenizer_2_different_operators_back_to_back():
set_action = "SET MyStr = NoExist + - :_val "
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "-"
assert te.near == "+ - :_val"
def test_update_expression_remove_does_not_allow_operations():
remove_action = "REMOVE NoExist + "
try:
UpdateExpressionParser.make(remove_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "+"
assert te.near == "NoExist + "
def test_update_expression_add_does_not_allow_attribute_after_path():
"""value here is not really a value since a value starts with a colon (:)"""
add_expr = "ADD attr val foobar"
try:
UpdateExpressionParser.make(add_expr)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "val"
assert te.near == "attr val foobar"
def test_update_expression_add_does_not_allow_attribute_foobar_after_value():
add_expr = "ADD attr :val foobar"
try:
UpdateExpressionParser.make(add_expr)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "foobar"
assert te.near == ":val foobar"
def test_update_expression_delete_does_not_allow_attribute_after_path():
"""value here is not really a value since a value starts with a colon (:)"""
delete_expr = "DELETE attr val"
try:
UpdateExpressionParser.make(delete_expr)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "val"
assert te.near == "attr val"
def test_update_expression_delete_does_not_allow_attribute_foobar_after_value():
delete_expr = "DELETE attr :val foobar"
try:
UpdateExpressionParser.make(delete_expr)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "foobar"
assert te.near == ":val foobar"
def test_update_expression_parsing_is_not_keyword_aware():
"""path and VALUE are keywords. Yet a token error will be thrown for the numeric literal 1."""
delete_expr = "SET path = VALUE 1"
try:
UpdateExpressionParser.make(delete_expr)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "1"
assert te.near == "VALUE 1"
def test_expression_if_not_exists_is_not_valid_in_remove_statement():
set_action = "REMOVE if_not_exists(a,b)"
try:
UpdateExpressionParser.make(set_action)
assert False, "Exception not raised correctly"
except InvalidTokenException as te:
assert te.token == "("
assert te.near == "if_not_exists(a"

16
tests/test_dynamodb2/test_dynamodb_table_with_range_key.py
View file

@ -1254,14 +1254,22 @@ def test_update_item_with_expression():
item_key = {"forum_name": "the-key", "subject": "123"}
table.update_item(Key=item_key, UpdateExpression="SET field=2")
table.update_item(
Key=item_key,
UpdateExpression="SET field = :field_value",
ExpressionAttributeValues={":field_value": 2},
)
dict(table.get_item(Key=item_key)["Item"]).should.equal(
{"field": "2", "forum_name": "the-key", "subject": "123"}
{"field": Decimal("2"), "forum_name": "the-key", "subject": "123"}
)
table.update_item(Key=item_key, UpdateExpression="SET field = 3")
table.update_item(
Key=item_key,
UpdateExpression="SET field = :field_value",
ExpressionAttributeValues={":field_value": 3},
)
dict(table.get_item(Key=item_key)["Item"]).should.equal(
{"field": "3", "forum_name": "the-key", "subject": "123"}
{"field": Decimal("3"), "forum_name": "the-key", "subject": "123"}
)

63
tests/test_dynamodb2/test_dynamodb_table_without_range_key.py
View file

@ -443,23 +443,40 @@ def test_update_item_nested_remove():
dict(returned_item).should.equal({"username": "steve", "Meta": {}})
@mock_dynamodb2_deprecated
@mock_dynamodb2
def test_update_item_double_nested_remove():
conn = boto.dynamodb2.connect_to_region("us-east-1")
table = Table.create("messages", schema=[HashKey("username")])
conn = boto3.client("dynamodb", region_name="us-east-1")
conn.create_table(
TableName="messages",
KeySchema=[{"AttributeName": "username", "KeyType": "HASH"}],
AttributeDefinitions=[{"AttributeName": "username", "AttributeType": "S"}],
BillingMode="PAY_PER_REQUEST",
)
data = {"username": "steve", "Meta": {"Name": {"First": "Steve", "Last": "Urkel"}}}
table.put_item(data=data)
item = {
"username": {"S": "steve"},
"Meta": {
"M": {"Name": {"M": {"First": {"S": "Steve"}, "Last": {"S": "Urkel"}}}}
},
}
conn.put_item(TableName="messages", Item=item)
key_map = {"username": {"S": "steve"}}
# Then remove the Meta.FullName field
conn.update_item("messages", key_map, update_expression="REMOVE Meta.Name.First")
returned_item = table.get_item(username="steve")
dict(returned_item).should.equal(
{"username": "steve", "Meta": {"Name": {"Last": "Urkel"}}}
conn.update_item(
TableName="messages",
Key=key_map,
UpdateExpression="REMOVE Meta.#N.#F",
ExpressionAttributeNames={"#N": "Name", "#F": "First"},
)
returned_item = conn.get_item(TableName="messages", Key=key_map)
expected_item = {
"username": {"S": "steve"},
"Meta": {"M": {"Name": {"M": {"Last": {"S": "Urkel"}}}}},
}
dict(returned_item["Item"]).should.equal(expected_item)
@mock_dynamodb2_deprecated
def test_update_item_set():
@ -471,7 +488,10 @@ def test_update_item_set():
key_map = {"username": {"S": "steve"}}
conn.update_item(
"messages", key_map, update_expression="SET foo=bar, blah=baz REMOVE SentBy"
"messages",
key_map,
update_expression="SET foo=:bar, blah=:baz REMOVE SentBy",
expression_attribute_values={":bar": {"S": "bar"}, ":baz": {"S": "baz"}},
)
returned_item = table.get_item(username="steve")
@ -616,8 +636,9 @@ def test_boto3_update_item_conditions_fail():
table.put_item(Item={"username": "johndoe", "foo": "baz"})
table.update_item.when.called_with(
Key={"username": "johndoe"},
UpdateExpression="SET foo=bar",
UpdateExpression="SET foo=:bar",
Expected={"foo": {"Value": "bar"}},
ExpressionAttributeValues={":bar": "bar"},
).should.throw(botocore.client.ClientError)
@ -627,8 +648,9 @@ def test_boto3_update_item_conditions_fail_because_expect_not_exists():
table.put_item(Item={"username": "johndoe", "foo": "baz"})
table.update_item.when.called_with(
Key={"username": "johndoe"},
UpdateExpression="SET foo=bar",
UpdateExpression="SET foo=:bar",
Expected={"foo": {"Exists": False}},
ExpressionAttributeValues={":bar": "bar"},
).should.throw(botocore.client.ClientError)
@ -638,8 +660,9 @@ def test_boto3_update_item_conditions_fail_because_expect_not_exists_by_compare_
table.put_item(Item={"username": "johndoe", "foo": "baz"})
table.update_item.when.called_with(
Key={"username": "johndoe"},
UpdateExpression="SET foo=bar",
UpdateExpression="SET foo=:bar",
Expected={"foo": {"ComparisonOperator": "NULL"}},
ExpressionAttributeValues={":bar": "bar"},
).should.throw(botocore.client.ClientError)
@ -649,8 +672,9 @@ def test_boto3_update_item_conditions_pass():
table.put_item(Item={"username": "johndoe", "foo": "bar"})
table.update_item(
Key={"username": "johndoe"},
UpdateExpression="SET foo=baz",
UpdateExpression="SET foo=:baz",
Expected={"foo": {"Value": "bar"}},
ExpressionAttributeValues={":baz": "baz"},
)
returned_item = table.get_item(Key={"username": "johndoe"})
assert dict(returned_item)["Item"]["foo"].should.equal("baz")
@ -662,8 +686,9 @@ def test_boto3_update_item_conditions_pass_because_expect_not_exists():
table.put_item(Item={"username": "johndoe", "foo": "bar"})
table.update_item(
Key={"username": "johndoe"},
UpdateExpression="SET foo=baz",
UpdateExpression="SET foo=:baz",
Expected={"whatever": {"Exists": False}},
ExpressionAttributeValues={":baz": "baz"},
)
returned_item = table.get_item(Key={"username": "johndoe"})
assert dict(returned_item)["Item"]["foo"].should.equal("baz")
@ -675,8 +700,9 @@ def test_boto3_update_item_conditions_pass_because_expect_not_exists_by_compare_
table.put_item(Item={"username": "johndoe", "foo": "bar"})
table.update_item(
Key={"username": "johndoe"},
UpdateExpression="SET foo=baz",
UpdateExpression="SET foo=:baz",
Expected={"whatever": {"ComparisonOperator": "NULL"}},
ExpressionAttributeValues={":baz": "baz"},
)
returned_item = table.get_item(Key={"username": "johndoe"})
assert dict(returned_item)["Item"]["foo"].should.equal("baz")
@ -688,8 +714,9 @@ def test_boto3_update_item_conditions_pass_because_expect_exists_by_compare_to_n
table.put_item(Item={"username": "johndoe", "foo": "bar"})
table.update_item(
Key={"username": "johndoe"},
UpdateExpression="SET foo=baz",
UpdateExpression="SET foo=:baz",
Expected={"foo": {"ComparisonOperator": "NOT_NULL"}},
ExpressionAttributeValues={":baz": "baz"},
)
returned_item = table.get_item(Key={"username": "johndoe"})
assert dict(returned_item)["Item"]["foo"].should.equal("baz")

464
tests/test_dynamodb2/test_dynamodb_validation.py Normal file
View file

@ -0,0 +1,464 @@
from moto.dynamodb2.exceptions import (
AttributeIsReservedKeyword,
ExpressionAttributeValueNotDefined,
AttributeDoesNotExist,
ExpressionAttributeNameNotDefined,
IncorrectOperandType,
InvalidUpdateExpressionInvalidDocumentPath,
)
from moto.dynamodb2.models import Item, DynamoType
from moto.dynamodb2.parsing.ast_nodes import (
NodeDepthLeftTypeFetcher,
UpdateExpressionSetAction,
UpdateExpressionValue,
DDBTypedValue,
)
from moto.dynamodb2.parsing.expressions import UpdateExpressionParser
from moto.dynamodb2.parsing.validators import UpdateExpressionValidator
from parameterized import parameterized
def test_validation_of_update_expression_with_keyword():
try:
update_expression = "SET myNum = path + :val"
update_expression_values = {":val": {"N": "3"}}
update_expression_ast = UpdateExpressionParser.make(update_expression)
item = Item(
hash_key=DynamoType({"S": "id"}),
hash_key_type="TYPE",
range_key=None,
range_key_type=None,
attrs={"id": {"S": "1"}, "path": {"N": "3"}},
)
UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=None,
expression_attribute_values=update_expression_values,
item=item,
).validate()
assert False, "No exception raised"
except AttributeIsReservedKeyword as e:
assert e.keyword == "path"
@parameterized(
["SET a = #b + :val2", "SET a = :val2 + #b",]
)
def test_validation_of_a_set_statement_with_incorrect_passed_value(update_expression):
"""
By running permutations it shows that values are replaced prior to resolving attributes.
An error occurred (ValidationException) when calling the UpdateItem operation: Invalid UpdateExpression:
An expression attribute value used in expression is not defined; attribute value: :val2
"""
update_expression_ast = UpdateExpressionParser.make(update_expression)
item = Item(
hash_key=DynamoType({"S": "id"}),
hash_key_type="TYPE",
range_key=None,
range_key_type=None,
attrs={"id": {"S": "1"}, "b": {"N": "3"}},
)
try:
UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names={"#b": "ok"},
expression_attribute_values={":val": {"N": "3"}},
item=item,
).validate()
except ExpressionAttributeValueNotDefined as e:
assert e.attribute_value == ":val2"
def test_validation_of_update_expression_with_attribute_that_does_not_exist_in_item():
"""
When an update expression tries to get an attribute that does not exist it must throw the appropriate exception.
An error occurred (ValidationException) when calling the UpdateItem operation:
The provided expression refers to an attribute that does not exist in the item
"""
try:
update_expression = "SET a = nonexistent"
update_expression_ast = UpdateExpressionParser.make(update_expression)
item = Item(
hash_key=DynamoType({"S": "id"}),
hash_key_type="TYPE",
range_key=None,
range_key_type=None,
attrs={"id": {"S": "1"}, "path": {"N": "3"}},
)
UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=None,
expression_attribute_values=None,
item=item,
).validate()
assert False, "No exception raised"
except AttributeDoesNotExist:
assert True
@parameterized(
["SET a = #c", "SET a = #c + #d",]
)
def test_validation_of_update_expression_with_attribute_name_that_is_not_defined(
update_expression,
):
"""
When an update expression tries to get an attribute name that is not provided it must throw an exception.
An error occurred (ValidationException) when calling the UpdateItem operation: Invalid UpdateExpression:
An expression attribute name used in the document path is not defined; attribute name: #c
"""
try:
update_expression_ast = UpdateExpressionParser.make(update_expression)
item = Item(
hash_key=DynamoType({"S": "id"}),
hash_key_type="TYPE",
range_key=None,
range_key_type=None,
attrs={"id": {"S": "1"}, "path": {"N": "3"}},
)
UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names={"#b": "ok"},
expression_attribute_values=None,
item=item,
).validate()
assert False, "No exception raised"
except ExpressionAttributeNameNotDefined as e:
assert e.not_defined_attribute_name == "#c"
def test_validation_of_if_not_exists_not_existing_invalid_replace_value():
try:
update_expression = "SET a = if_not_exists(b, a.c)"
update_expression_ast = UpdateExpressionParser.make(update_expression)
item = Item(
hash_key=DynamoType({"S": "id"}),
hash_key_type="TYPE",
range_key=None,
range_key_type=None,
attrs={"id": {"S": "1"}, "a": {"S": "A"}},
)
UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=None,
expression_attribute_values=None,
item=item,
).validate()
assert False, "No exception raised"
except AttributeDoesNotExist:
assert True
def get_first_node_of_type(ast, node_type):
return next(NodeDepthLeftTypeFetcher(node_type, ast))
def get_set_action_value(ast):
"""
Helper that takes an AST and gets the first UpdateExpressionSetAction and retrieves the value of that action.
This should only be called on validated expressions.
Args:
ast(Node):
Returns:
DynamoType: The DynamoType object representing the Dynamo value.
"""
set_action = get_first_node_of_type(ast, UpdateExpressionSetAction)
typed_value = set_action.children[1]
assert isinstance(typed_value, DDBTypedValue)
dynamo_value = typed_value.children[0]
assert isinstance(dynamo_value, DynamoType)
return dynamo_value
def test_validation_of_if_not_exists_not_existing_value():
update_expression = "SET a = if_not_exists(b, a)"
update_expression_ast = UpdateExpressionParser.make(update_expression)
item = Item(
hash_key=DynamoType({"S": "id"}),
hash_key_type="TYPE",
range_key=None,
range_key_type=None,
attrs={"id": {"S": "1"}, "a": {"S": "A"}},
)
validated_ast = UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=None,
expression_attribute_values=None,
item=item,
).validate()
dynamo_value = get_set_action_value(validated_ast)
assert dynamo_value == DynamoType({"S": "A"})
def test_validation_of_if_not_exists_with_existing_attribute_should_return_attribute():
update_expression = "SET a = if_not_exists(b, a)"
update_expression_ast = UpdateExpressionParser.make(update_expression)
item = Item(
hash_key=DynamoType({"S": "id"}),
hash_key_type="TYPE",
range_key=None,
range_key_type=None,
attrs={"id": {"S": "1"}, "a": {"S": "A"}, "b": {"S": "B"}},
)
validated_ast = UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=None,
expression_attribute_values=None,
item=item,
).validate()
dynamo_value = get_set_action_value(validated_ast)
assert dynamo_value == DynamoType({"S": "B"})
def test_validation_of_if_not_exists_with_existing_attribute_should_return_value():
update_expression = "SET a = if_not_exists(b, :val)"
update_expression_values = {":val": {"N": "4"}}
update_expression_ast = UpdateExpressionParser.make(update_expression)
item = Item(
hash_key=DynamoType({"S": "id"}),
hash_key_type="TYPE",
range_key=None,
range_key_type=None,
attrs={"id": {"S": "1"}, "b": {"N": "3"}},
)
validated_ast = UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=None,
expression_attribute_values=update_expression_values,
item=item,
).validate()
dynamo_value = get_set_action_value(validated_ast)
assert dynamo_value == DynamoType({"N": "3"})
def test_validation_of_if_not_exists_with_non_existing_attribute_should_return_value():
update_expression = "SET a = if_not_exists(b, :val)"
update_expression_values = {":val": {"N": "4"}}
update_expression_ast = UpdateExpressionParser.make(update_expression)
item = Item(
hash_key=DynamoType({"S": "id"}),
hash_key_type="TYPE",
range_key=None,
range_key_type=None,
attrs={"id": {"S": "1"}},
)
validated_ast = UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=None,
expression_attribute_values=update_expression_values,
item=item,
).validate()
dynamo_value = get_set_action_value(validated_ast)
assert dynamo_value == DynamoType({"N": "4"})
def test_validation_of_sum_operation():
update_expression = "SET a = a + b"
update_expression_ast = UpdateExpressionParser.make(update_expression)
item = Item(
hash_key=DynamoType({"S": "id"}),
hash_key_type="TYPE",
range_key=None,
range_key_type=None,
attrs={"id": {"S": "1"}, "a": {"N": "3"}, "b": {"N": "4"}},
)
validated_ast = UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=None,
expression_attribute_values=None,
item=item,
).validate()
dynamo_value = get_set_action_value(validated_ast)
assert dynamo_value == DynamoType({"N": "7"})
def test_validation_homogeneous_list_append_function():
update_expression = "SET ri = list_append(ri, :vals)"
update_expression_ast = UpdateExpressionParser.make(update_expression)
item = Item(
hash_key=DynamoType({"S": "id"}),
hash_key_type="TYPE",
range_key=None,
range_key_type=None,
attrs={"id": {"S": "1"}, "ri": {"L": [{"S": "i1"}, {"S": "i2"}]}},
)
validated_ast = UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=None,
expression_attribute_values={":vals": {"L": [{"S": "i3"}, {"S": "i4"}]}},
item=item,
).validate()
dynamo_value = get_set_action_value(validated_ast)
assert dynamo_value == DynamoType(
{"L": [{"S": "i1"}, {"S": "i2"}, {"S": "i3"}, {"S": "i4"}]}
)
def test_validation_hetereogenous_list_append_function():
update_expression = "SET ri = list_append(ri, :vals)"
update_expression_ast = UpdateExpressionParser.make(update_expression)
item = Item(
hash_key=DynamoType({"S": "id"}),
hash_key_type="TYPE",
range_key=None,
range_key_type=None,
attrs={"id": {"S": "1"}, "ri": {"L": [{"S": "i1"}, {"S": "i2"}]}},
)
validated_ast = UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=None,
expression_attribute_values={":vals": {"L": [{"N": "3"}]}},
item=item,
).validate()
dynamo_value = get_set_action_value(validated_ast)
assert dynamo_value == DynamoType({"L": [{"S": "i1"}, {"S": "i2"}, {"N": "3"}]})
def test_validation_list_append_function_with_non_list_arg():
"""
Must error out:
Invalid UpdateExpression: Incorrect operand type for operator or function;
operator or function: list_append, operand type: S'
Returns:
"""
try:
update_expression = "SET ri = list_append(ri, :vals)"
update_expression_ast = UpdateExpressionParser.make(update_expression)
item = Item(
hash_key=DynamoType({"S": "id"}),
hash_key_type="TYPE",
range_key=None,
range_key_type=None,
attrs={"id": {"S": "1"}, "ri": {"L": [{"S": "i1"}, {"S": "i2"}]}},
)
UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=None,
expression_attribute_values={":vals": {"S": "N"}},
item=item,
).validate()
except IncorrectOperandType as e:
assert e.operand_type == "S"
assert e.operator_or_function == "list_append"
def test_sum_with_incompatible_types():
"""
Must error out:
Invalid UpdateExpression: Incorrect operand type for operator or function; operator or function: +, operand type: S'
Returns:
"""
try:
update_expression = "SET ri = :val + :val2"
update_expression_ast = UpdateExpressionParser.make(update_expression)
item = Item(
hash_key=DynamoType({"S": "id"}),
hash_key_type="TYPE",
range_key=None,
range_key_type=None,
attrs={"id": {"S": "1"}, "ri": {"L": [{"S": "i1"}, {"S": "i2"}]}},
)
UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=None,
expression_attribute_values={":val": {"S": "N"}, ":val2": {"N": "3"}},
item=item,
).validate()
except IncorrectOperandType as e:
assert e.operand_type == "S"
assert e.operator_or_function == "+"
def test_validation_of_subraction_operation():
update_expression = "SET ri = :val - :val2"
update_expression_ast = UpdateExpressionParser.make(update_expression)
item = Item(
hash_key=DynamoType({"S": "id"}),
hash_key_type="TYPE",
range_key=None,
range_key_type=None,
attrs={"id": {"S": "1"}, "a": {"N": "3"}, "b": {"N": "4"}},
)
validated_ast = UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=None,
expression_attribute_values={":val": {"N": "1"}, ":val2": {"N": "3"}},
item=item,
).validate()
dynamo_value = get_set_action_value(validated_ast)
assert dynamo_value == DynamoType({"N": "-2"})
def test_cannot_index_into_a_string():
"""
Must error out:
The document path provided in the update expression is invalid for update'
"""
try:
update_expression = "set itemstr[1]=:Item"
update_expression_ast = UpdateExpressionParser.make(update_expression)
item = Item(
hash_key=DynamoType({"S": "id"}),
hash_key_type="TYPE",
range_key=None,
range_key_type=None,
attrs={"id": {"S": "foo2"}, "itemstr": {"S": "somestring"}},
)
UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=None,
expression_attribute_values={":Item": {"S": "string_update"}},
item=item,
).validate()
assert False, "Must raise exception"
except InvalidUpdateExpressionInvalidDocumentPath:
assert True
def test_validation_set_path_does_not_need_to_be_resolvable_when_setting_a_new_attribute():
"""If this step just passes we are happy enough"""
update_expression = "set d=a"
update_expression_ast = UpdateExpressionParser.make(update_expression)
item = Item(
hash_key=DynamoType({"S": "id"}),
hash_key_type="TYPE",
range_key=None,
range_key_type=None,
attrs={"id": {"S": "foo2"}, "a": {"N": "3"}},
)
validated_ast = UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=None,
expression_attribute_values=None,
item=item,
).validate()
dynamo_value = get_set_action_value(validated_ast)
assert dynamo_value == DynamoType({"N": "3"})
def test_validation_set_path_does_not_need_to_be_resolvable_but_must_be_creatable_when_setting_a_new_attribute():
try:
update_expression = "set d.e=a"
update_expression_ast = UpdateExpressionParser.make(update_expression)
item = Item(
hash_key=DynamoType({"S": "id"}),
hash_key_type="TYPE",
range_key=None,
range_key_type=None,
attrs={"id": {"S": "foo2"}, "a": {"N": "3"}},
)
UpdateExpressionValidator(
update_expression_ast,
expression_attribute_names=None,
expression_attribute_values=None,
item=item,
).validate()
assert False, "Must raise exception"
except InvalidUpdateExpressionInvalidDocumentPath:
assert True

42
tests/test_ec2/test_instances.py
View file

@ -9,6 +9,7 @@ from nose.tools import assert_raises
import base64
import datetime
import ipaddress
import json
import six
import boto
@ -18,7 +19,7 @@ from boto.exception import EC2ResponseError, EC2ResponseError
from freezegun import freeze_time
import sure # noqa
from moto import mock_ec2_deprecated, mock_ec2
from moto import mock_ec2_deprecated, mock_ec2, mock_cloudformation
from tests.helpers import requires_boto_gte
@ -71,7 +72,7 @@ def test_instance_launch_and_terminate():
instance.id.should.equal(instance.id)
instance.state.should.equal("running")
instance.launch_time.should.equal("2014-01-01T05:00:00.000Z")
instance.vpc_id.should.equal(None)
instance.vpc_id.shouldnt.equal(None)
instance.placement.should.equal("us-east-1a")
root_device_name = instance.root_device_name
@ -1414,3 +1415,40 @@ def test_describe_instance_attribute():
invalid_instance_attribute=invalid_instance_attribute
)
ex.exception.response["Error"]["Message"].should.equal(message)
@mock_ec2
@mock_cloudformation
def test_volume_size_through_cloudformation():
ec2 = boto3.client("ec2", region_name="us-east-1")
cf = boto3.client("cloudformation", region_name="us-east-1")
volume_template = {
"AWSTemplateFormatVersion": "2010-09-09",
"Resources": {
"testInstance": {
"Type": "AWS::EC2::Instance",
"Properties": {
"ImageId": "ami-d3adb33f",
"KeyName": "dummy",
"InstanceType": "t2.micro",
"BlockDeviceMappings": [
{"DeviceName": "/dev/sda2", "Ebs": {"VolumeSize": "50"}}
],
"Tags": [
{"Key": "foo", "Value": "bar"},
{"Key": "blah", "Value": "baz"},
],
},
}
},
}
template_json = json.dumps(volume_template)
cf.create_stack(StackName="test_stack", TemplateBody=template_json)
instances = ec2.describe_instances()
volume = instances["Reservations"][0]["Instances"][0]["BlockDeviceMappings"][0][
"Ebs"
]
volumes = ec2.describe_volumes(VolumeIds=[volume["VolumeId"]])
volumes["Volumes"][0]["Size"].should.equal(50)

17
tests/test_ec2/test_subnets.py
View file

@ -599,3 +599,20 @@ def validate_subnet_details_after_creating_eni(
for eni in enis_created:
client.delete_network_interface(NetworkInterfaceId=eni["NetworkInterfaceId"])
client.delete_subnet(SubnetId=subnet["SubnetId"])
@mock_ec2
def test_run_instances_should_attach_to_default_subnet():
# https://github.com/spulec/moto/issues/2877
ec2 = boto3.resource("ec2", region_name="us-west-1")
client = boto3.client("ec2", region_name="us-west-1")
ec2.create_security_group(GroupName="sg01", Description="Test security group sg01")
# run_instances
instances = client.run_instances(MinCount=1, MaxCount=1, SecurityGroups=["sg01"],)
# Assert subnet is created appropriately
subnets = client.describe_subnets()["Subnets"]
default_subnet_id = subnets[0]["SubnetId"]
instances["Instances"][0]["NetworkInterfaces"][0]["SubnetId"].should.equal(
default_subnet_id
)
subnets[0]["AvailableIpAddressCount"].should.equal(4090)

23
tests/test_glue/test_datacatalog.py
View file

@ -52,6 +52,29 @@ def test_get_database_not_exits():
)
@mock_glue
def test_get_databases_empty():
client = boto3.client("glue", region_name="us-east-1")
response = client.get_databases()
response["DatabaseList"].should.have.length_of(0)
@mock_glue
def test_get_databases_several_items():
client = boto3.client("glue", region_name="us-east-1")
database_name_1, database_name_2 = "firstdatabase", "seconddatabase"
helpers.create_database(client, database_name_1)
helpers.create_database(client, database_name_2)
database_list = sorted(
client.get_databases()["DatabaseList"], key=lambda x: x["Name"]
)
database_list.should.have.length_of(2)
database_list[0].should.equal({"Name": database_name_1})
database_list[1].should.equal({"Name": database_name_2})
@mock_glue
def test_create_table():
client = boto3.client("glue", region_name="us-east-1")

8
tests/test_iot/test_iot.py
View file

@ -728,6 +728,14 @@ def test_principal_thing():
res = client.list_thing_principals(thingName=thing_name)
res.should.have.key("principals").which.should.have.length_of(0)
with assert_raises(ClientError) as e:
client.list_thing_principals(thingName="xxx")
e.exception.response["Error"]["Code"].should.equal("ResourceNotFoundException")
e.exception.response["Error"]["Message"].should.equal(
"Failed to list principals for thing xxx because the thing does not exist in your account"
)
@mock_iot
def test_delete_principal_thing():

31
tests/test_rds2/test_rds2.py
View file

@ -183,12 +183,12 @@ def test_start_database():
@mock_rds2
def test_fail_to_stop_multi_az():
def test_fail_to_stop_multi_az_and_sqlserver():
conn = boto3.client("rds", region_name="us-west-2")
database = conn.create_db_instance(
DBInstanceIdentifier="db-master-1",
AllocatedStorage=10,
Engine="postgres",
Engine="sqlserver-ee",
DBName="staging-postgres",
DBInstanceClass="db.m1.small",
LicenseModel="license-included",
@ -213,6 +213,33 @@ def test_fail_to_stop_multi_az():
).should.throw(ClientError)
@mock_rds2
def test_stop_multi_az_postgres():
conn = boto3.client("rds", region_name="us-west-2")
database = conn.create_db_instance(
DBInstanceIdentifier="db-master-1",
AllocatedStorage=10,
Engine="postgres",
DBName="staging-postgres",
DBInstanceClass="db.m1.small",
LicenseModel="license-included",
MasterUsername="root",
MasterUserPassword="hunter2",
Port=1234,
DBSecurityGroups=["my_sg"],
MultiAZ=True,
)
mydb = conn.describe_db_instances(
DBInstanceIdentifier=database["DBInstance"]["DBInstanceIdentifier"]
)["DBInstances"][0]
mydb["DBInstanceStatus"].should.equal("available")
response = conn.stop_db_instance(DBInstanceIdentifier=mydb["DBInstanceIdentifier"])
response["ResponseMetadata"]["HTTPStatusCode"].should.equal(200)
response["DBInstance"]["DBInstanceStatus"].should.equal("stopped")
@mock_rds2
def test_fail_to_stop_readreplica():
conn = boto3.client("rds", region_name="us-west-2")

39
tests/test_s3/test_s3.py
View file

@ -14,6 +14,7 @@ from io import BytesIO
import mimetypes
import zlib
import pickle
import uuid
import json
import boto
@ -4428,3 +4429,41 @@ def test_s3_config_dict():
assert not logging_bucket["supplementaryConfiguration"].get(
"BucketTaggingConfiguration"
)
@mock_s3
def test_creating_presigned_post():
bucket = "presigned-test"
s3 = boto3.client("s3", region_name="us-east-1")
s3.create_bucket(Bucket=bucket)
success_url = "http://localhost/completed"
fdata = b"test data\n"
file_uid = uuid.uuid4()
conditions = [
{"Content-Type": "text/plain"},
{"x-amz-server-side-encryption": "AES256"},
{"success_action_redirect": success_url},
]
conditions.append(["content-length-range", 1, 30])
data = s3.generate_presigned_post(
Bucket=bucket,
Key="{file_uid}.txt".format(file_uid=file_uid),
Fields={
"content-type": "text/plain",
"success_action_redirect": success_url,
"x-amz-server-side-encryption": "AES256",
},
Conditions=conditions,
ExpiresIn=1000,
)
resp = requests.post(
data["url"], data=data["fields"], files={"file": fdata}, allow_redirects=False
)
assert resp.headers["Location"] == success_url
assert resp.status_code == 303
assert (
s3.get_object(Bucket=bucket, Key="{file_uid}.txt".format(file_uid=file_uid))[
"Body"
].read()
== fdata
)

76
tests/test_secretsmanager/test_secretsmanager.py
View file

@ -711,3 +711,79 @@ def test_can_list_secret_version_ids():
returned_version_ids = [v["VersionId"] for v in versions_list["Versions"]]
assert [first_version_id, second_version_id].sort() == returned_version_ids.sort()
@mock_secretsmanager
def test_update_secret():
conn = boto3.client("secretsmanager", region_name="us-west-2")
created_secret = conn.create_secret(Name="test-secret", SecretString="foosecret")
assert created_secret["ARN"]
assert created_secret["Name"] == "test-secret"
assert created_secret["VersionId"] != ""
secret = conn.get_secret_value(SecretId="test-secret")
assert secret["SecretString"] == "foosecret"
updated_secret = conn.update_secret(
SecretId="test-secret", SecretString="barsecret"
)
assert updated_secret["ARN"]
assert updated_secret["Name"] == "test-secret"
assert updated_secret["VersionId"] != ""
secret = conn.get_secret_value(SecretId="test-secret")
assert secret["SecretString"] == "barsecret"
assert created_secret["VersionId"] != updated_secret["VersionId"]
@mock_secretsmanager
def test_update_secret_which_does_not_exit():
conn = boto3.client("secretsmanager", region_name="us-west-2")
with assert_raises(ClientError) as cm:
updated_secret = conn.update_secret(
SecretId="test-secret", SecretString="barsecret"
)
assert_equal(
"Secrets Manager can't find the specified secret.",
cm.exception.response["Error"]["Message"],
)
@mock_secretsmanager
def test_update_secret_marked_as_deleted():
conn = boto3.client("secretsmanager", region_name="us-west-2")
created_secret = conn.create_secret(Name="test-secret", SecretString="foosecret")
deleted_secret = conn.delete_secret(SecretId="test-secret")
with assert_raises(ClientError) as cm:
updated_secret = conn.update_secret(
SecretId="test-secret", SecretString="barsecret"
)
assert (
"because it was marked for deletion."
in cm.exception.response["Error"]["Message"]
)
@mock_secretsmanager
def test_update_secret_marked_as_deleted_after_restoring():
conn = boto3.client("secretsmanager", region_name="us-west-2")
created_secret = conn.create_secret(Name="test-secret", SecretString="foosecret")
deleted_secret = conn.delete_secret(SecretId="test-secret")
restored_secret = conn.restore_secret(SecretId="test-secret")
updated_secret = conn.update_secret(
SecretId="test-secret", SecretString="barsecret"
)
assert updated_secret["ARN"]
assert updated_secret["Name"] == "test-secret"
assert updated_secret["VersionId"] != ""