catalin/moto
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Reduce default value for DEFAULT_KEY_BUFFER_SIZE (#4003)

Browse source 
* - introduce environment variable for DEFAULT_KEY_BUFFER_SIZE

* - prefix env variable with MOTO_S3 to avoid env variable conflicts

* - reduce the DEFAULT_KEY_BUFFER_SIZE to be less than the S3_UPLOAD_PART_MIN_SIZE to prevent in memory caching of multi part uploads

* - black formatting

* - fix formatting

* - fix missing import
This commit is contained in:
benediktbrandt 2021-07-07 11:38:50 -04:00 • committed by GitHub
commit eb6515cf50
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 20 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

5
moto/s3/models.py
View file

@ -45,12 +45,11 @@ from .exceptions import (
)
from .cloud_formation import cfn_to_api_encryption, is_replacement_update
from .utils import clean_key_name, _VersionedKeyStore
from ..settings import get_s3_default_key_buffer_size
from ..settings import get_s3_default_key_buffer_size, S3_UPLOAD_PART_MIN_SIZE
MAX_BUCKET_NAME_LENGTH = 63
MIN_BUCKET_NAME_LENGTH = 3
UPLOAD_ID_BYTES = 43
UPLOAD_PART_MIN_SIZE = 5242880
STORAGE_CLASS = [
"STANDARD",
"REDUCED_REDUNDANCY",
@ -320,7 +319,7 @@ class FakeMultipart(BaseModel):
etag = etag.replace('"', "")
if part is None or part_etag != etag:
raise InvalidPart()
if last is not None and last.contentsize < UPLOAD_PART_MIN_SIZE:
if last is not None and last.contentsize < S3_UPLOAD_PART_MIN_SIZE:
raise EntityTooSmall()
md5s.extend(decode_hex(part_etag)[0])
total.extend(part.value)

9
moto/settings.py
View file

@ -23,5 +23,12 @@ def get_sf_execution_history_type():
return os.environ.get("SF_EXECUTION_HISTORY_TYPE", "SUCCESS")
S3_UPLOAD_PART_MIN_SIZE = 5242880
def get_s3_default_key_buffer_size():
return int(os.environ.get("MOTO_S3_DEFAULT_KEY_BUFFER_SIZE", 16 * 1024 * 1024))
return int(
os.environ.get(
"MOTO_S3_DEFAULT_KEY_BUFFER_SIZE", S3_UPLOAD_PART_MIN_SIZE - 1024
)
)