DynamoDB - Add 1MB item size check
This commit is contained in:
parent
66b26cd7b0
commit
e82e1e3f39
2 changed files with 49 additions and 0 deletions
|
|
@ -285,6 +285,9 @@ class Item(BaseModel):
|
|||
def __repr__(self):
|
||||
return "Item: {0}".format(self.to_json())
|
||||
|
||||
def size(self):
|
||||
return sum([bytesize(key) + value.size() for key, value in self.attrs.items()])
|
||||
|
||||
def to_json(self):
|
||||
attributes = {}
|
||||
for attribute_key, attribute in self.attrs.items():
|
||||
|
|
@ -1123,6 +1126,14 @@ class Table(BaseModel):
|
|||
break
|
||||
|
||||
last_evaluated_key = None
|
||||
size_limit = 1000000 # DynamoDB has a 1MB size limit
|
||||
item_size = sum([res.size() for res in results])
|
||||
if item_size > size_limit:
|
||||
item_size = idx = 0
|
||||
while item_size + results[idx].size() < size_limit:
|
||||
item_size += results[idx].size()
|
||||
idx += 1
|
||||
limit = min(limit, idx) if limit else idx
|
||||
if limit and len(results) > limit:
|
||||
results = results[:limit]
|
||||
last_evaluated_key = {self.hash_key_attr: results[-1].hash_key}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue