From 5a7c711a74f3f91c86ac2527af09f4c40f49b6aa Mon Sep 17 00:00:00 2001 From: Peter Gorniak Date: Fri, 25 Nov 2016 21:07:24 -0800 Subject: [PATCH 01/11] bring dynamodb2 update expression handling closer to spec --- moto/dynamodb2/models.py | 36 +++++++++++++++++------------------- 1 file changed, 17 insertions(+), 19 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index a70d6347..e9980410 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -3,6 +3,7 @@ from collections import defaultdict import datetime import decimal import json +import re from moto.compat import OrderedDict from moto.core import BaseBackend @@ -110,27 +111,24 @@ class Item(object): } def update(self, update_expression, expression_attribute_names, expression_attribute_values): - ACTION_VALUES = ['SET', 'set', 'REMOVE', 'remove'] - - action = None - for value in update_expression.split(): - if value in ACTION_VALUES: - # An action - action = value - continue - else: + parts = [p for p in re.split(r'\b(SET|REMOVE|ADD|DELETE)\b', update_expression) if p] + for action, valstr in zip(parts[:-1:1], parts[1::1]): + values = valstr.split(',') + for value in values: # A Real value value = value.lstrip(":").rstrip(",") - for k, v in expression_attribute_names.items(): - value = value.replace(k, v) - if action == "REMOVE" or action == 'remove': - self.attrs.pop(value, None) - elif action == 'SET' or action == 'set': - key, value = value.split("=") - if value in expression_attribute_values: - self.attrs[key] = DynamoType(expression_attribute_values[value]) - else: - self.attrs[key] = DynamoType({"S": value}) + for k, v in expression_attribute_names.items(): + value = value.replace(k, v) + if action == "REMOVE" or action == 'remove': + self.attrs.pop(value, None) + elif action == 'SET' or action == 'set': + key, value = value.split("=") + key = key.strip() + value = value.strip() + if value in expression_attribute_values: + self.attrs[key] = DynamoType(expression_attribute_values[value]) + else: + self.attrs[key] = DynamoType({"S": value}) def update_with_attribute_updates(self, attribute_updates): for attribute_name, update_action in attribute_updates.items(): From 2c505615631c2958e7b5f4a3c196b5386370a9e0 Mon Sep 17 00:00:00 2001 From: Peter Gorniak Date: Tue, 29 Nov 2016 14:04:23 -0800 Subject: [PATCH 02/11] fix decoding keys in query condition --- moto/dynamodb2/models.py | 7 ++++--- moto/dynamodb2/responses.py | 9 ++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index e9980410..2c6d8d60 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -119,9 +119,9 @@ class Item(object): value = value.lstrip(":").rstrip(",") for k, v in expression_attribute_names.items(): value = value.replace(k, v) - if action == "REMOVE" or action == 'remove': + if action == "REMOVE": self.attrs.pop(value, None) - elif action == 'SET' or action == 'set': + elif action == 'SET': key, value = value.split("=") key = key.strip() value = value.strip() @@ -129,6 +129,8 @@ class Item(object): self.attrs[key] = DynamoType(expression_attribute_values[value]) else: self.attrs[key] = DynamoType({"S": value}) + else: + raise NotImplementedError('{} update action not yet supported'.format(action)) def update_with_attribute_updates(self, attribute_updates): for attribute_name, update_action in attribute_updates.items(): @@ -323,7 +325,6 @@ class Table(object): def query(self, hash_key, range_comparison, range_objs, limit, exclusive_start_key, scan_index_forward, index_name=None, **filter_kwargs): results = [] - if index_name: all_indexes = (self.global_indexes or []) + (self.indexes or []) indexes_by_name = dict((i['IndexName'], i) for i in all_indexes) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 081afc2c..eea2bace 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -279,19 +279,22 @@ class DynamoHandler(BaseResponse): else: index = table.schema - key_map = [column for _, column in sorted((k, v) for k, v in self.body['ExpressionAttributeNames'].items())] + reverse_attribute_lookup = {v: k for k, v in self.body['ExpressionAttributeNames'].iteritems()} if " AND " in key_condition_expression: expressions = key_condition_expression.split(" AND ", 1) index_hash_key = [key for key in index if key['KeyType'] == 'HASH'][0] - hash_key_index_in_key_map = key_map.index(index_hash_key['AttributeName']) + hash_key_var = reverse_attribute_lookup.get(index_hash_key['AttributeName'], index_hash_key['AttributeName']) + i, hash_key_expression = ((i, e) for i, e in enumerate(expressions) if re.search(r'[\s(]#n1\b'.format(hash_key_var), e)).next() + hash_key_expression = hash_key_expression.strip('()') + expressions.pop(i) - hash_key_expression = expressions.pop(hash_key_index_in_key_map).strip('()') # TODO implement more than one range expression and OR operators range_key_expression = expressions[0].strip('()') range_key_expression_components = range_key_expression.split() range_comparison = range_key_expression_components[1] + if 'AND' in range_key_expression: range_comparison = 'BETWEEN' range_values = [ From 98a39cf4b53ffeff99cfc31c990b6fafefe54089 Mon Sep 17 00:00:00 2001 From: Peter Gorniak Date: Tue, 6 Dec 2016 12:14:57 -0800 Subject: [PATCH 03/11] account for keys potentially being substrings of other keys (e.g. #c1 and #c10) --- moto/dynamodb2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 2c6d8d60..5e788884 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -118,7 +118,7 @@ class Item(object): # A Real value value = value.lstrip(":").rstrip(",") for k, v in expression_attribute_names.items(): - value = value.replace(k, v) + value = re.sub(r'{}\b'.format(k), v, value) if action == "REMOVE": self.attrs.pop(value, None) elif action == 'SET': From 390bef77521a683c3599198fc57a3375d6d79137 Mon Sep 17 00:00:00 2001 From: Peter Gorniak Date: Tue, 6 Dec 2016 16:57:36 -0800 Subject: [PATCH 04/11] fake change to force push because github was broken --- moto/dynamodb2/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 5e788884..0e88d594 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -119,6 +119,7 @@ class Item(object): value = value.lstrip(":").rstrip(",") for k, v in expression_attribute_names.items(): value = re.sub(r'{}\b'.format(k), v, value) + if action == "REMOVE": self.attrs.pop(value, None) elif action == 'SET': From 3c128fdb51d7cdb81f8128fe73a5adca6daf4c6a Mon Sep 17 00:00:00 2001 From: Peter Gorniak Date: Wed, 7 Dec 2016 11:47:48 -0800 Subject: [PATCH 05/11] correct looping through update actions, value stripping, hash key regex --- moto/dynamodb2/models.py | 4 ++-- moto/dynamodb2/responses.py | 3 ++- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 0e88d594..15a3e3ba 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -112,11 +112,11 @@ class Item(object): def update(self, update_expression, expression_attribute_names, expression_attribute_values): parts = [p for p in re.split(r'\b(SET|REMOVE|ADD|DELETE)\b', update_expression) if p] - for action, valstr in zip(parts[:-1:1], parts[1::1]): + for action, valstr in zip(parts[:-1:2], parts[1::2]): values = valstr.split(',') for value in values: # A Real value - value = value.lstrip(":").rstrip(",") + value = value.lstrip(":").rstrip(",").strip() for k, v in expression_attribute_names.items(): value = re.sub(r'{}\b'.format(k), v, value) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index eea2bace..39cdaae4 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -286,7 +286,8 @@ class DynamoHandler(BaseResponse): index_hash_key = [key for key in index if key['KeyType'] == 'HASH'][0] hash_key_var = reverse_attribute_lookup.get(index_hash_key['AttributeName'], index_hash_key['AttributeName']) - i, hash_key_expression = ((i, e) for i, e in enumerate(expressions) if re.search(r'[\s(]#n1\b'.format(hash_key_var), e)).next() + hash_key_regex = r'(^|[\s(]){}\b'.format(hash_key_var) + i, hash_key_expression = ((i, e) for i, e in enumerate(expressions) if re.search(hash_key_regex, e)).next() hash_key_expression = hash_key_expression.strip('()') expressions.pop(i) From 0c875fd268d06d6b7fe8084537ae2bd3d653f8e6 Mon Sep 17 00:00:00 2001 From: Peter Gorniak Date: Wed, 7 Dec 2016 13:31:15 -0800 Subject: [PATCH 06/11] fixes for python 2.6 and 3 --- moto/dynamodb2/responses.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 39cdaae4..815bd9f5 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -279,7 +279,7 @@ class DynamoHandler(BaseResponse): else: index = table.schema - reverse_attribute_lookup = {v: k for k, v in self.body['ExpressionAttributeNames'].iteritems()} + reverse_attribute_lookup = dict((v, k) for k, v in six.iteritems(self.body['ExpressionAttributeNames'])) if " AND " in key_condition_expression: expressions = key_condition_expression.split(" AND ", 1) From 114de9ba0b8e745a0054408713dafdfb535d0ec3 Mon Sep 17 00:00:00 2001 From: Peter Gorniak Date: Wed, 7 Dec 2016 13:55:26 -0800 Subject: [PATCH 07/11] more fixes for 2.6 and 3 --- moto/dynamodb2/responses.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 815bd9f5..636a0f9d 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -286,8 +286,8 @@ class DynamoHandler(BaseResponse): index_hash_key = [key for key in index if key['KeyType'] == 'HASH'][0] hash_key_var = reverse_attribute_lookup.get(index_hash_key['AttributeName'], index_hash_key['AttributeName']) - hash_key_regex = r'(^|[\s(]){}\b'.format(hash_key_var) - i, hash_key_expression = ((i, e) for i, e in enumerate(expressions) if re.search(hash_key_regex, e)).next() + hash_key_regex = r'(^|[\s(]){0}\b'.format(hash_key_var) + i, hash_key_expression = next((i, e) for i, e in enumerate(expressions) if re.search(hash_key_regex, e)) hash_key_expression = hash_key_expression.strip('()') expressions.pop(i) From d4a31e5e50c1eca8fb6f885c60b56c15a50dce09 Mon Sep 17 00:00:00 2001 From: Peter Gorniak Date: Thu, 8 Dec 2016 14:34:21 -0800 Subject: [PATCH 08/11] unit tests did not catch this, but this will not work under python 2.6 --- moto/dynamodb2/models.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 15a3e3ba..4bca8358 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -118,7 +118,7 @@ class Item(object): # A Real value value = value.lstrip(":").rstrip(",").strip() for k, v in expression_attribute_names.items(): - value = re.sub(r'{}\b'.format(k), v, value) + value = re.sub(r'{0}\b'.format(k), v, value) if action == "REMOVE": self.attrs.pop(value, None) From 5eb866146a71455ac94bfe7750a1d88503620790 Mon Sep 17 00:00:00 2001 From: Peter Gorniak Date: Fri, 2 Jun 2017 13:19:45 -0700 Subject: [PATCH 09/11] add assert to catch odd numbers in operator/value parsing --- moto/dynamodb2/models.py | 1 + 1 file changed, 1 insertion(+) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 4bca8358..1a609beb 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -112,6 +112,7 @@ class Item(object): def update(self, update_expression, expression_attribute_names, expression_attribute_values): parts = [p for p in re.split(r'\b(SET|REMOVE|ADD|DELETE)\b', update_expression) if p] + assert len(parts) % 2 == 0, "Mismatched operators and values in update expression: '{}'".format(update_expression) for action, valstr in zip(parts[:-1:2], parts[1::2]): values = valstr.split(',') for value in values: From b713eef491b84f440a85de25b10d6304874f817f Mon Sep 17 00:00:00 2001 From: Peter Gorniak Date: Fri, 2 Jun 2017 13:41:33 -0700 Subject: [PATCH 10/11] cleanup after merge --- moto/dynamodb2/responses.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/moto/dynamodb2/responses.py b/moto/dynamodb2/responses.py index 39b67240..d3fa68b7 100644 --- a/moto/dynamodb2/responses.py +++ b/moto/dynamodb2/responses.py @@ -316,16 +316,18 @@ class DynamoHandler(BaseResponse): else: index = table.schema - reverse_attribute_lookup = dict((v, k) for k, v in - six.iteritems(self.body['ExpressionAttributeNames'])) + reverse_attribute_lookup = dict((v, k) for k, v in + six.iteritems(self.body['ExpressionAttributeNames'])) if " AND " in key_condition_expression: expressions = key_condition_expression.split(" AND ", 1) index_hash_key = [key for key in index if key['KeyType'] == 'HASH'][0] - hash_key_var = reverse_attribute_lookup.get(index_hash_key['AttributeName'], index_hash_key['AttributeName']) + hash_key_var = reverse_attribute_lookup.get(index_hash_key['AttributeName'], + index_hash_key['AttributeName']) hash_key_regex = r'(^|[\s(]){0}\b'.format(hash_key_var) - i, hash_key_expression = next((i, e) for i, e in enumerate(expressions) if re.search(hash_key_regex, e)) + i, hash_key_expression = next((i, e) for i, e in enumerate(expressions) + if re.search(hash_key_regex, e)) hash_key_expression = hash_key_expression.strip('()') expressions.pop(i) From a0471b04072d9538aa885c97964f22617e9ac879 Mon Sep 17 00:00:00 2001 From: Peter Gorniak Date: Thu, 15 Jun 2017 15:34:58 -0700 Subject: [PATCH 11/11] add comment about splitting update expression by operator keywords --- moto/dynamodb2/models.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/moto/dynamodb2/models.py b/moto/dynamodb2/models.py index 32dbfadb..e6f05078 100644 --- a/moto/dynamodb2/models.py +++ b/moto/dynamodb2/models.py @@ -116,7 +116,10 @@ class Item(BaseModel): } def update(self, update_expression, expression_attribute_names, expression_attribute_values): + # Update subexpressions are identifiable by the operator keyword, so split on that and + # get rid of the empty leading string. parts = [p for p in re.split(r'\b(SET|REMOVE|ADD|DELETE)\b', update_expression) if p] + # make sure that we correctly found only operator/value pairs assert len(parts) % 2 == 0, "Mismatched operators and values in update expression: '{}'".format(update_expression) for action, valstr in zip(parts[:-1:2], parts[1::2]): values = valstr.split(',')