Run black on moto & test directories.
This commit is contained in:
Asher Foa
parent
c820395dbf
commit
96e5b1993d
507 changed files with 52541 additions and 47814 deletions
|
|
@ -6,8 +6,15 @@ from moto.s3 import s3_backends
|
|||
|
||||
|
||||
class S3ConfigQuery(ConfigQueryModel):
|
||||
|
||||
def list_config_service_resources(self, resource_ids, resource_name, limit, next_token, backend_region=None, resource_region=None):
|
||||
def list_config_service_resources(
|
||||
self,
|
||||
resource_ids,
|
||||
resource_name,
|
||||
limit,
|
||||
next_token,
|
||||
backend_region=None,
|
||||
resource_region=None,
|
||||
):
|
||||
# The resource_region only matters for aggregated queries as you can filter on bucket regions for them.
|
||||
# For other resource types, you would need to iterate appropriately for the backend_region.
|
||||
|
||||
|
|
@ -20,14 +27,14 @@ class S3ConfigQuery(ConfigQueryModel):
|
|||
|
||||
# If no filter was passed in for resource names/ids then return them all:
|
||||
if not resource_ids and not resource_name:
|
||||
bucket_list = list(self.backends['global'].buckets.keys())
|
||||
bucket_list = list(self.backends["global"].buckets.keys())
|
||||
|
||||
else:
|
||||
# Match the resource name / ID:
|
||||
bucket_list = []
|
||||
filter_buckets = [resource_name] if resource_name else resource_ids
|
||||
|
||||
for bucket in self.backends['global'].buckets.keys():
|
||||
for bucket in self.backends["global"].buckets.keys():
|
||||
if bucket in filter_buckets:
|
||||
bucket_list.append(bucket)
|
||||
|
||||
|
|
@ -37,7 +44,7 @@ class S3ConfigQuery(ConfigQueryModel):
|
|||
region_buckets = []
|
||||
|
||||
for bucket in bucket_list:
|
||||
if self.backends['global'].buckets[bucket].region_name == region_filter:
|
||||
if self.backends["global"].buckets[bucket].region_name == region_filter:
|
||||
region_buckets.append(bucket)
|
||||
|
||||
bucket_list = region_buckets
|
||||
|
|
@ -61,17 +68,29 @@ class S3ConfigQuery(ConfigQueryModel):
|
|||
start = sorted_buckets.index(next_token)
|
||||
|
||||
# Get the list of items to collect:
|
||||
bucket_list = sorted_buckets[start:(start + limit)]
|
||||
bucket_list = sorted_buckets[start : (start + limit)]
|
||||
|
||||
if len(sorted_buckets) > (start + limit):
|
||||
new_token = sorted_buckets[start + limit]
|
||||
|
||||
return [{'type': 'AWS::S3::Bucket', 'id': bucket, 'name': bucket, 'region': self.backends['global'].buckets[bucket].region_name}
|
||||
for bucket in bucket_list], new_token
|
||||
return (
|
||||
[
|
||||
{
|
||||
"type": "AWS::S3::Bucket",
|
||||
"id": bucket,
|
||||
"name": bucket,
|
||||
"region": self.backends["global"].buckets[bucket].region_name,
|
||||
}
|
||||
for bucket in bucket_list
|
||||
],
|
||||
new_token,
|
||||
)
|
||||
|
||||
def get_config_resource(self, resource_id, resource_name=None, backend_region=None, resource_region=None):
|
||||
def get_config_resource(
|
||||
self, resource_id, resource_name=None, backend_region=None, resource_region=None
|
||||
):
|
||||
# Get the bucket:
|
||||
bucket = self.backends['global'].buckets.get(resource_id, {})
|
||||
bucket = self.backends["global"].buckets.get(resource_id, {})
|
||||
|
||||
if not bucket:
|
||||
return
|
||||
|
|
@ -89,12 +108,12 @@ class S3ConfigQuery(ConfigQueryModel):
|
|||
config_data = bucket.to_config_dict()
|
||||
|
||||
# The 'configuration' field is also a JSON string:
|
||||
config_data['configuration'] = json.dumps(config_data['configuration'])
|
||||
config_data["configuration"] = json.dumps(config_data["configuration"])
|
||||
|
||||
# Supplementary config need all values converted to JSON strings if they are not strings already:
|
||||
for field, value in config_data['supplementaryConfiguration'].items():
|
||||
for field, value in config_data["supplementaryConfiguration"].items():
|
||||
if not isinstance(value, str):
|
||||
config_data['supplementaryConfiguration'][field] = json.dumps(value)
|
||||
config_data["supplementaryConfiguration"][field] = json.dumps(value)
|
||||
|
||||
return config_data
|
||||
|
||||
|
|
|
|||
|
|
@ -12,18 +12,16 @@ ERROR_WITH_KEY_NAME = """{% extends 'single_error' %}
|
|||
|
||||
|
||||
class S3ClientError(RESTError):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
kwargs.setdefault('template', 'single_error')
|
||||
self.templates['bucket_error'] = ERROR_WITH_BUCKET_NAME
|
||||
kwargs.setdefault("template", "single_error")
|
||||
self.templates["bucket_error"] = ERROR_WITH_BUCKET_NAME
|
||||
super(S3ClientError, self).__init__(*args, **kwargs)
|
||||
|
||||
|
||||
class BucketError(S3ClientError):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
kwargs.setdefault('template', 'bucket_error')
|
||||
self.templates['bucket_error'] = ERROR_WITH_BUCKET_NAME
|
||||
kwargs.setdefault("template", "bucket_error")
|
||||
self.templates["bucket_error"] = ERROR_WITH_BUCKET_NAME
|
||||
super(BucketError, self).__init__(*args, **kwargs)
|
||||
|
||||
|
||||
|
|
@ -33,10 +31,14 @@ class BucketAlreadyExists(BucketError):
|
|||
def __init__(self, *args, **kwargs):
|
||||
super(BucketAlreadyExists, self).__init__(
|
||||
"BucketAlreadyExists",
|
||||
("The requested bucket name is not available. The bucket "
|
||||
"namespace is shared by all users of the system. Please "
|
||||
"select a different name and try again"),
|
||||
*args, **kwargs)
|
||||
(
|
||||
"The requested bucket name is not available. The bucket "
|
||||
"namespace is shared by all users of the system. Please "
|
||||
"select a different name and try again"
|
||||
),
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class MissingBucket(BucketError):
|
||||
|
|
@ -44,9 +46,8 @@ class MissingBucket(BucketError):
|
|||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(MissingBucket, self).__init__(
|
||||
"NoSuchBucket",
|
||||
"The specified bucket does not exist",
|
||||
*args, **kwargs)
|
||||
"NoSuchBucket", "The specified bucket does not exist", *args, **kwargs
|
||||
)
|
||||
|
||||
|
||||
class MissingKey(S3ClientError):
|
||||
|
|
@ -54,9 +55,7 @@ class MissingKey(S3ClientError):
|
|||
|
||||
def __init__(self, key_name):
|
||||
super(MissingKey, self).__init__(
|
||||
"NoSuchKey",
|
||||
"The specified key does not exist.",
|
||||
Key=key_name,
|
||||
"NoSuchKey", "The specified key does not exist.", Key=key_name
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -77,9 +76,13 @@ class InvalidPartOrder(S3ClientError):
|
|||
def __init__(self, *args, **kwargs):
|
||||
super(InvalidPartOrder, self).__init__(
|
||||
"InvalidPartOrder",
|
||||
("The list of parts was not in ascending order. The parts "
|
||||
"list must be specified in order by part number."),
|
||||
*args, **kwargs)
|
||||
(
|
||||
"The list of parts was not in ascending order. The parts "
|
||||
"list must be specified in order by part number."
|
||||
),
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class InvalidPart(S3ClientError):
|
||||
|
|
@ -88,10 +91,14 @@ class InvalidPart(S3ClientError):
|
|||
def __init__(self, *args, **kwargs):
|
||||
super(InvalidPart, self).__init__(
|
||||
"InvalidPart",
|
||||
("One or more of the specified parts could not be found. "
|
||||
"The part might not have been uploaded, or the specified "
|
||||
"entity tag might not have matched the part's entity tag."),
|
||||
*args, **kwargs)
|
||||
(
|
||||
"One or more of the specified parts could not be found. "
|
||||
"The part might not have been uploaded, or the specified "
|
||||
"entity tag might not have matched the part's entity tag."
|
||||
),
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class EntityTooSmall(S3ClientError):
|
||||
|
|
@ -101,7 +108,9 @@ class EntityTooSmall(S3ClientError):
|
|||
super(EntityTooSmall, self).__init__(
|
||||
"EntityTooSmall",
|
||||
"Your proposed upload is smaller than the minimum allowed object size.",
|
||||
*args, **kwargs)
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class InvalidRequest(S3ClientError):
|
||||
|
|
@ -110,8 +119,12 @@ class InvalidRequest(S3ClientError):
|
|||
def __init__(self, method, *args, **kwargs):
|
||||
super(InvalidRequest, self).__init__(
|
||||
"InvalidRequest",
|
||||
"Found unsupported HTTP method in CORS config. Unsupported method is {}".format(method),
|
||||
*args, **kwargs)
|
||||
"Found unsupported HTTP method in CORS config. Unsupported method is {}".format(
|
||||
method
|
||||
),
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class MalformedXML(S3ClientError):
|
||||
|
|
@ -121,7 +134,9 @@ class MalformedXML(S3ClientError):
|
|||
super(MalformedXML, self).__init__(
|
||||
"MalformedXML",
|
||||
"The XML you provided was not well-formed or did not validate against our published schema",
|
||||
*args, **kwargs)
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class MalformedACLError(S3ClientError):
|
||||
|
|
@ -131,14 +146,18 @@ class MalformedACLError(S3ClientError):
|
|||
super(MalformedACLError, self).__init__(
|
||||
"MalformedACLError",
|
||||
"The XML you provided was not well-formed or did not validate against our published schema",
|
||||
*args, **kwargs)
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class InvalidTargetBucketForLogging(S3ClientError):
|
||||
code = 400
|
||||
|
||||
def __init__(self, msg):
|
||||
super(InvalidTargetBucketForLogging, self).__init__("InvalidTargetBucketForLogging", msg)
|
||||
super(InvalidTargetBucketForLogging, self).__init__(
|
||||
"InvalidTargetBucketForLogging", msg
|
||||
)
|
||||
|
||||
|
||||
class CrossLocationLoggingProhibitted(S3ClientError):
|
||||
|
|
@ -146,8 +165,7 @@ class CrossLocationLoggingProhibitted(S3ClientError):
|
|||
|
||||
def __init__(self):
|
||||
super(CrossLocationLoggingProhibitted, self).__init__(
|
||||
"CrossLocationLoggingProhibitted",
|
||||
"Cross S3 location logging not allowed."
|
||||
"CrossLocationLoggingProhibitted", "Cross S3 location logging not allowed."
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -156,9 +174,8 @@ class InvalidNotificationARN(S3ClientError):
|
|||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(InvalidNotificationARN, self).__init__(
|
||||
"InvalidArgument",
|
||||
"The ARN is not well formed",
|
||||
*args, **kwargs)
|
||||
"InvalidArgument", "The ARN is not well formed", *args, **kwargs
|
||||
)
|
||||
|
||||
|
||||
class InvalidNotificationDestination(S3ClientError):
|
||||
|
|
@ -168,7 +185,9 @@ class InvalidNotificationDestination(S3ClientError):
|
|||
super(InvalidNotificationDestination, self).__init__(
|
||||
"InvalidArgument",
|
||||
"The notification destination service region is not valid for the bucket location constraint",
|
||||
*args, **kwargs)
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class InvalidNotificationEvent(S3ClientError):
|
||||
|
|
@ -178,7 +197,9 @@ class InvalidNotificationEvent(S3ClientError):
|
|||
super(InvalidNotificationEvent, self).__init__(
|
||||
"InvalidArgument",
|
||||
"The event is not supported for notifications",
|
||||
*args, **kwargs)
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class InvalidStorageClass(S3ClientError):
|
||||
|
|
@ -188,7 +209,9 @@ class InvalidStorageClass(S3ClientError):
|
|||
super(InvalidStorageClass, self).__init__(
|
||||
"InvalidStorageClass",
|
||||
"The storage class you specified is not valid",
|
||||
*args, **kwargs)
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class InvalidBucketName(S3ClientError):
|
||||
|
|
@ -196,9 +219,7 @@ class InvalidBucketName(S3ClientError):
|
|||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(InvalidBucketName, self).__init__(
|
||||
"InvalidBucketName",
|
||||
"The specified bucket is not valid.",
|
||||
*args, **kwargs
|
||||
"InvalidBucketName", "The specified bucket is not valid.", *args, **kwargs
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -209,35 +230,51 @@ class DuplicateTagKeys(S3ClientError):
|
|||
super(DuplicateTagKeys, self).__init__(
|
||||
"InvalidTag",
|
||||
"Cannot provide multiple Tags with the same key",
|
||||
*args, **kwargs)
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class S3AccessDeniedError(S3ClientError):
|
||||
code = 403
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(S3AccessDeniedError, self).__init__('AccessDenied', 'Access Denied', *args, **kwargs)
|
||||
super(S3AccessDeniedError, self).__init__(
|
||||
"AccessDenied", "Access Denied", *args, **kwargs
|
||||
)
|
||||
|
||||
|
||||
class BucketAccessDeniedError(BucketError):
|
||||
code = 403
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(BucketAccessDeniedError, self).__init__('AccessDenied', 'Access Denied', *args, **kwargs)
|
||||
super(BucketAccessDeniedError, self).__init__(
|
||||
"AccessDenied", "Access Denied", *args, **kwargs
|
||||
)
|
||||
|
||||
|
||||
class S3InvalidTokenError(S3ClientError):
|
||||
code = 400
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(S3InvalidTokenError, self).__init__('InvalidToken', 'The provided token is malformed or otherwise invalid.', *args, **kwargs)
|
||||
super(S3InvalidTokenError, self).__init__(
|
||||
"InvalidToken",
|
||||
"The provided token is malformed or otherwise invalid.",
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class BucketInvalidTokenError(BucketError):
|
||||
code = 400
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(BucketInvalidTokenError, self).__init__('InvalidToken', 'The provided token is malformed or otherwise invalid.', *args, **kwargs)
|
||||
super(BucketInvalidTokenError, self).__init__(
|
||||
"InvalidToken",
|
||||
"The provided token is malformed or otherwise invalid.",
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class S3InvalidAccessKeyIdError(S3ClientError):
|
||||
|
|
@ -245,8 +282,11 @@ class S3InvalidAccessKeyIdError(S3ClientError):
|
|||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(S3InvalidAccessKeyIdError, self).__init__(
|
||||
'InvalidAccessKeyId',
|
||||
"The AWS Access Key Id you provided does not exist in our records.", *args, **kwargs)
|
||||
"InvalidAccessKeyId",
|
||||
"The AWS Access Key Id you provided does not exist in our records.",
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class BucketInvalidAccessKeyIdError(S3ClientError):
|
||||
|
|
@ -254,8 +294,11 @@ class BucketInvalidAccessKeyIdError(S3ClientError):
|
|||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(BucketInvalidAccessKeyIdError, self).__init__(
|
||||
'InvalidAccessKeyId',
|
||||
"The AWS Access Key Id you provided does not exist in our records.", *args, **kwargs)
|
||||
"InvalidAccessKeyId",
|
||||
"The AWS Access Key Id you provided does not exist in our records.",
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class S3SignatureDoesNotMatchError(S3ClientError):
|
||||
|
|
@ -263,8 +306,11 @@ class S3SignatureDoesNotMatchError(S3ClientError):
|
|||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(S3SignatureDoesNotMatchError, self).__init__(
|
||||
'SignatureDoesNotMatch',
|
||||
"The request signature we calculated does not match the signature you provided. Check your key and signing method.", *args, **kwargs)
|
||||
"SignatureDoesNotMatch",
|
||||
"The request signature we calculated does not match the signature you provided. Check your key and signing method.",
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
||||
|
||||
class BucketSignatureDoesNotMatchError(S3ClientError):
|
||||
|
|
@ -272,5 +318,8 @@ class BucketSignatureDoesNotMatchError(S3ClientError):
|
|||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super(BucketSignatureDoesNotMatchError, self).__init__(
|
||||
'SignatureDoesNotMatch',
|
||||
"The request signature we calculated does not match the signature you provided. Check your key and signing method.", *args, **kwargs)
|
||||
"SignatureDoesNotMatch",
|
||||
"The request signature we calculated does not match the signature you provided. Check your key and signing method.",
|
||||
*args,
|
||||
**kwargs
|
||||
)
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
File diff suppressed because it is too large
Load diff
|
|
@ -4,17 +4,16 @@ from .responses import S3ResponseInstance
|
|||
|
||||
url_bases = [
|
||||
"https?://s3(.*).amazonaws.com",
|
||||
r"https?://(?P<bucket_name>[a-zA-Z0-9\-_.]*)\.?s3(.*).amazonaws.com"
|
||||
r"https?://(?P<bucket_name>[a-zA-Z0-9\-_.]*)\.?s3(.*).amazonaws.com",
|
||||
]
|
||||
|
||||
url_paths = {
|
||||
# subdomain bucket
|
||||
'{0}/$': S3ResponseInstance.bucket_response,
|
||||
|
||||
"{0}/$": S3ResponseInstance.bucket_response,
|
||||
# subdomain key of path-based bucket
|
||||
'{0}/(?P<key_or_bucket_name>[^/]+)/?$': S3ResponseInstance.ambiguous_response,
|
||||
"{0}/(?P<key_or_bucket_name>[^/]+)/?$": S3ResponseInstance.ambiguous_response,
|
||||
# path-based bucket + key
|
||||
'{0}/(?P<bucket_name_path>[^/]+)/(?P<key_name>.+)': S3ResponseInstance.key_response,
|
||||
"{0}/(?P<bucket_name_path>[^/]+)/(?P<key_name>.+)": S3ResponseInstance.key_response,
|
||||
# subdomain bucket + key with empty first part of path
|
||||
'{0}//(?P<key_name>.*)$': S3ResponseInstance.key_response,
|
||||
"{0}//(?P<key_name>.*)$": S3ResponseInstance.key_response,
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,19 +16,19 @@ bucket_name_regex = re.compile("(.+).s3(.*).amazonaws.com")
|
|||
|
||||
|
||||
def bucket_name_from_url(url):
|
||||
if os.environ.get('S3_IGNORE_SUBDOMAIN_BUCKETNAME', '') in ['1', 'true']:
|
||||
if os.environ.get("S3_IGNORE_SUBDOMAIN_BUCKETNAME", "") in ["1", "true"]:
|
||||
return None
|
||||
domain = urlparse(url).netloc
|
||||
|
||||
if domain.startswith('www.'):
|
||||
if domain.startswith("www."):
|
||||
domain = domain[4:]
|
||||
|
||||
if 'amazonaws.com' in domain:
|
||||
if "amazonaws.com" in domain:
|
||||
bucket_result = bucket_name_regex.search(domain)
|
||||
if bucket_result:
|
||||
return bucket_result.groups()[0]
|
||||
else:
|
||||
if '.' in domain:
|
||||
if "." in domain:
|
||||
return domain.split(".")[0]
|
||||
else:
|
||||
# No subdomain found.
|
||||
|
|
@ -36,23 +36,23 @@ def bucket_name_from_url(url):
|
|||
|
||||
|
||||
REGION_URL_REGEX = re.compile(
|
||||
r'^https?://(s3[-\.](?P<region1>.+)\.amazonaws\.com/(.+)|'
|
||||
r'(.+)\.s3-(?P<region2>.+)\.amazonaws\.com)/?')
|
||||
r"^https?://(s3[-\.](?P<region1>.+)\.amazonaws\.com/(.+)|"
|
||||
r"(.+)\.s3-(?P<region2>.+)\.amazonaws\.com)/?"
|
||||
)
|
||||
|
||||
|
||||
def parse_region_from_url(url):
|
||||
match = REGION_URL_REGEX.search(url)
|
||||
if match:
|
||||
region = match.group('region1') or match.group('region2')
|
||||
region = match.group("region1") or match.group("region2")
|
||||
else:
|
||||
region = 'us-east-1'
|
||||
region = "us-east-1"
|
||||
return region
|
||||
|
||||
|
||||
def metadata_from_headers(headers):
|
||||
metadata = {}
|
||||
meta_regex = re.compile(
|
||||
'^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE)
|
||||
meta_regex = re.compile("^x-amz-meta-([a-zA-Z0-9\-_]+)$", flags=re.IGNORECASE)
|
||||
for header, value in headers.items():
|
||||
if isinstance(header, six.string_types):
|
||||
result = meta_regex.match(header)
|
||||
|
|
@ -70,13 +70,13 @@ def metadata_from_headers(headers):
|
|||
|
||||
def clean_key_name(key_name):
|
||||
if six.PY2:
|
||||
return unquote(key_name.encode('utf-8')).decode('utf-8')
|
||||
return unquote(key_name.encode("utf-8")).decode("utf-8")
|
||||
return unquote(key_name)
|
||||
|
||||
|
||||
def undo_clean_key_name(key_name):
|
||||
if six.PY2:
|
||||
return quote(key_name.encode('utf-8')).decode('utf-8')
|
||||
return quote(key_name.encode("utf-8")).decode("utf-8")
|
||||
return quote(key_name)
|
||||
|
||||
|
||||
|
|
@ -140,6 +140,7 @@ class _VersionedKeyStore(dict):
|
|||
values = itervalues = _itervalues
|
||||
|
||||
if sys.version_info[0] < 3:
|
||||
|
||||
def items(self):
|
||||
return list(self.iteritems())
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue