catalin/moto
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Env variable for default key buffer size (#4001)

Browse source 
* - introduce environment variable for DEFAULT_KEY_BUFFER_SIZE

* - prefix env variable with MOTO_S3 to avoid env variable conflicts

* - black formatting

* - fix formatting
This commit is contained in:
benediktbrandt 2021-06-10 05:48:28 -04:00 • committed by GitHub
commit 479ce861a7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 36 additions and 6 deletions
Download patch file Download diff file Expand all files Collapse all files

12
moto/s3/models.py
View file

@ -45,6 +45,7 @@ from .exceptions import (
)
from .cloud_formation import cfn_to_api_encryption, is_replacement_update
from .utils import clean_key_name, _VersionedKeyStore
from ..settings import get_s3_default_key_buffer_size
MAX_BUCKET_NAME_LENGTH = 63
MIN_BUCKET_NAME_LENGTH = 3
@ -59,7 +60,6 @@ STORAGE_CLASS = [
"GLACIER",
"DEEP_ARCHIVE",
]
DEFAULT_KEY_BUFFER_SIZE = 16 * 1024 * 1024
DEFAULT_TEXT_ENCODING = sys.getdefaultencoding()
OWNER = "75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a"
@ -96,7 +96,7 @@ class FakeKey(BaseModel):
etag=None,
is_versioned=False,
version_id=0,
max_buffer_size=DEFAULT_KEY_BUFFER_SIZE,
max_buffer_size=None,
multipart=None,
bucket_name=None,
encryption=None,
@ -116,8 +116,10 @@ class FakeKey(BaseModel):
self.multipart = multipart
self.bucket_name = bucket_name
self._value_buffer = tempfile.SpooledTemporaryFile(max_size=max_buffer_size)
self._max_buffer_size = max_buffer_size
self._max_buffer_size = (
max_buffer_size if max_buffer_size else get_s3_default_key_buffer_size()
)
self._value_buffer = tempfile.SpooledTemporaryFile(self._max_buffer_size)
self.value = value
self.lock = threading.Lock()
@ -207,7 +209,7 @@ class FakeKey(BaseModel):
value_md5 = hashlib.md5()
self._value_buffer.seek(0)
while True:
block = self._value_buffer.read(DEFAULT_KEY_BUFFER_SIZE)
block = self._value_buffer.read(16 * 1024 * 1024) # read in 16MB chunks
if not block:
break
value_md5.update(block)

4
moto/settings.py
View file

@ -21,3 +21,7 @@ def get_sf_execution_history_type():
returned. Default value is SUCCESS, currently supports (SUCCESS || FAILURE)
"""
return os.environ.get("SF_EXECUTION_HISTORY_TYPE", "SUCCESS")
def get_s3_default_key_buffer_size():
return int(os.environ.get("MOTO_S3_DEFAULT_KEY_BUFFER_SIZE", 16 * 1024 * 1024))