Move S3 storage to SpooledTemporaryFile

moto/s3/models.py

@@ -8,6 +8,7 @@ import itertools
 import codecs
 import random
 import string
+import tempfile
 
 import six
 
@@ -21,6 +22,7 @@ from .utils import clean_key_name, _VersionedKeyStore
 UPLOAD_ID_BYTES = 43
 UPLOAD_PART_MIN_SIZE = 5242880
 STORAGE_CLASS = ["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA"]
+DEFAULT_KEY_BUFFER_SIZE = 2 ** 24
 
 
 class FakeDeleteMarker(BaseModel):
@@ -42,9 +44,9 @@ class FakeDeleteMarker(BaseModel):
 
 class FakeKey(BaseModel):
 
-    def __init__(self, name, value, storage="STANDARD", etag=None, is_versioned=False, version_id=0):
+    def __init__(self, name, value, storage="STANDARD", etag=None, is_versioned=False, version_id=0,
+                 max_buffer_size=DEFAULT_KEY_BUFFER_SIZE):
         self.name = name
-        self.value = value
         self.last_modified = datetime.datetime.utcnow()
         self.acl = get_canned_acl('private')
         self.website_redirect_location = None
@@ -56,10 +58,24 @@ class FakeKey(BaseModel):
         self._is_versioned = is_versioned
         self._tagging = FakeTagging()
 
+        self.value_buffer = tempfile.SpooledTemporaryFile(max_size=max_buffer_size)
+        self.value = value
+
     @property
     def version_id(self):
         return self._version_id
 
+    @property
+    def value(self):
+        self.value_buffer.seek(0)
+        return self.value_buffer.read()
+
+    @value.setter
+    def value(self, new_value):
+        self.value_buffer.seek(0)
+        self.value_buffer.truncate()
+        self.value_buffer.write(new_value)
+
     def copy(self, new_name=None):
         r = copy.deepcopy(self)
         if new_name is not None:
@@ -83,7 +99,9 @@ class FakeKey(BaseModel):
         self.acl = acl
 
     def append_to_value(self, value):
-        self.value += value
+        self.value_buffer.seek(0, os.SEEK_END)
+        self.value_buffer.write(value)
+
         self.last_modified = datetime.datetime.utcnow()
         self._etag = None  # must recalculate etag
         if self._is_versioned:
@@ -101,11 +119,14 @@ class FakeKey(BaseModel):
     def etag(self):
         if self._etag is None:
             value_md5 = hashlib.md5()
-            if isinstance(self.value, six.text_type):
-                value = self.value.encode("utf-8")
-            else:
-                value = self.value
-            value_md5.update(value)
+
+            self.value_buffer.seek(0)
+            while True:
+                block = self.value_buffer.read(DEFAULT_KEY_BUFFER_SIZE)
+                if not block:
+                    break
+                value_md5.update(block)
+
             self._etag = value_md5.hexdigest()
         return '"{0}"'.format(self._etag)
 
@@ -132,7 +153,7 @@ class FakeKey(BaseModel):
         res = {
             'ETag': self.etag,
             'last-modified': self.last_modified_RFC1123,
-            'content-length': str(len(self.value)),
+            'content-length': str(self.size),
         }
         if self._storage_class != 'STANDARD':
             res['x-amz-storage-class'] = self._storage_class
@@ -150,7 +171,8 @@ class FakeKey(BaseModel):
 
     @property
     def size(self):
-        return len(self.value)
+        self.value_buffer.seek(0, os.SEEK_END)
+        return self.value_buffer.tell()
 
     @property
     def storage_class(self):