DynamoDB - Add 1MB item size check

This commit is contained in:
Bert Blommers 2020-03-21 12:20:09 +00:00
commit e82e1e3f39
2 changed files with 49 additions and 0 deletions

View file

@ -285,6 +285,9 @@ class Item(BaseModel):
def __repr__(self):
return "Item: {0}".format(self.to_json())
def size(self):
return sum([bytesize(key) + value.size() for key, value in self.attrs.items()])
def to_json(self):
attributes = {}
for attribute_key, attribute in self.attrs.items():
@ -1123,6 +1126,14 @@ class Table(BaseModel):
break
last_evaluated_key = None
size_limit = 1000000 # DynamoDB has a 1MB size limit
item_size = sum([res.size() for res in results])
if item_size > size_limit:
item_size = idx = 0
while item_size + results[idx].size() < size_limit:
item_size += results[idx].size()
idx += 1
limit = min(limit, idx) if limit else idx
if limit and len(results) > limit:
results = results[:limit]
last_evaluated_key = {self.hash_key_attr: results[-1].hash_key}