Lints.
This commit is contained in:
Steve Pulec
parent
1433f28846
commit
f37bad0e00
260 changed files with 6363 additions and 3766 deletions
|
|
@ -3,4 +3,4 @@ from .models import s3_backend
|
|||
|
||||
s3_backends = {"global": s3_backend}
|
||||
mock_s3 = s3_backend.decorator
|
||||
mock_s3_deprecated = s3_backend.deprecated_decorator
|
||||
mock_s3_deprecated = s3_backend.deprecated_decorator
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@ ERROR_WITH_KEY_NAME = """{% extends 'single_error' %}
|
|||
|
||||
|
||||
class S3ClientError(RESTError):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
kwargs.setdefault('template', 'single_error')
|
||||
self.templates['bucket_error'] = ERROR_WITH_BUCKET_NAME
|
||||
|
|
@ -19,6 +20,7 @@ class S3ClientError(RESTError):
|
|||
|
||||
|
||||
class BucketError(S3ClientError):
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
kwargs.setdefault('template', 'bucket_error')
|
||||
self.templates['bucket_error'] = ERROR_WITH_BUCKET_NAME
|
||||
|
|
|
|||
|
|
@ -120,6 +120,7 @@ class FakeKey(object):
|
|||
|
||||
|
||||
class FakeMultipart(object):
|
||||
|
||||
def __init__(self, key_name, metadata):
|
||||
self.key_name = key_name
|
||||
self.metadata = metadata
|
||||
|
|
@ -167,6 +168,7 @@ class FakeMultipart(object):
|
|||
|
||||
|
||||
class FakeGrantee(object):
|
||||
|
||||
def __init__(self, id='', uri='', display_name=''):
|
||||
self.id = id
|
||||
self.uri = uri
|
||||
|
|
@ -177,9 +179,12 @@ class FakeGrantee(object):
|
|||
return 'Group' if self.uri else 'CanonicalUser'
|
||||
|
||||
|
||||
ALL_USERS_GRANTEE = FakeGrantee(uri='http://acs.amazonaws.com/groups/global/AllUsers')
|
||||
AUTHENTICATED_USERS_GRANTEE = FakeGrantee(uri='http://acs.amazonaws.com/groups/global/AuthenticatedUsers')
|
||||
LOG_DELIVERY_GRANTEE = FakeGrantee(uri='http://acs.amazonaws.com/groups/s3/LogDelivery')
|
||||
ALL_USERS_GRANTEE = FakeGrantee(
|
||||
uri='http://acs.amazonaws.com/groups/global/AllUsers')
|
||||
AUTHENTICATED_USERS_GRANTEE = FakeGrantee(
|
||||
uri='http://acs.amazonaws.com/groups/global/AuthenticatedUsers')
|
||||
LOG_DELIVERY_GRANTEE = FakeGrantee(
|
||||
uri='http://acs.amazonaws.com/groups/s3/LogDelivery')
|
||||
|
||||
PERMISSION_FULL_CONTROL = 'FULL_CONTROL'
|
||||
PERMISSION_WRITE = 'WRITE'
|
||||
|
|
@ -189,27 +194,32 @@ PERMISSION_READ_ACP = 'READ_ACP'
|
|||
|
||||
|
||||
class FakeGrant(object):
|
||||
|
||||
def __init__(self, grantees, permissions):
|
||||
self.grantees = grantees
|
||||
self.permissions = permissions
|
||||
|
||||
|
||||
class FakeAcl(object):
|
||||
|
||||
def __init__(self, grants=[]):
|
||||
self.grants = grants
|
||||
|
||||
|
||||
def get_canned_acl(acl):
|
||||
owner_grantee = FakeGrantee(id='75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a')
|
||||
owner_grantee = FakeGrantee(
|
||||
id='75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a')
|
||||
grants = [FakeGrant([owner_grantee], [PERMISSION_FULL_CONTROL])]
|
||||
if acl == 'private':
|
||||
pass # no other permissions
|
||||
elif acl == 'public-read':
|
||||
grants.append(FakeGrant([ALL_USERS_GRANTEE], [PERMISSION_READ]))
|
||||
elif acl == 'public-read-write':
|
||||
grants.append(FakeGrant([ALL_USERS_GRANTEE], [PERMISSION_READ, PERMISSION_WRITE]))
|
||||
grants.append(FakeGrant([ALL_USERS_GRANTEE], [
|
||||
PERMISSION_READ, PERMISSION_WRITE]))
|
||||
elif acl == 'authenticated-read':
|
||||
grants.append(FakeGrant([AUTHENTICATED_USERS_GRANTEE], [PERMISSION_READ]))
|
||||
grants.append(
|
||||
FakeGrant([AUTHENTICATED_USERS_GRANTEE], [PERMISSION_READ]))
|
||||
elif acl == 'bucket-owner-read':
|
||||
pass # TODO: bucket owner ACL
|
||||
elif acl == 'bucket-owner-full-control':
|
||||
|
|
@ -217,13 +227,15 @@ def get_canned_acl(acl):
|
|||
elif acl == 'aws-exec-read':
|
||||
pass # TODO: bucket owner, EC2 Read
|
||||
elif acl == 'log-delivery-write':
|
||||
grants.append(FakeGrant([LOG_DELIVERY_GRANTEE], [PERMISSION_READ_ACP, PERMISSION_WRITE]))
|
||||
grants.append(FakeGrant([LOG_DELIVERY_GRANTEE], [
|
||||
PERMISSION_READ_ACP, PERMISSION_WRITE]))
|
||||
else:
|
||||
assert False, 'Unknown canned acl: %s' % (acl,)
|
||||
return FakeAcl(grants=grants)
|
||||
|
||||
|
||||
class LifecycleRule(object):
|
||||
|
||||
def __init__(self, id=None, prefix=None, status=None, expiration_days=None,
|
||||
expiration_date=None, transition_days=None,
|
||||
transition_date=None, storage_class=None):
|
||||
|
|
@ -271,7 +283,8 @@ class FakeBucket(object):
|
|||
expiration_date=expiration.get('Date') if expiration else None,
|
||||
transition_days=transition.get('Days') if transition else None,
|
||||
transition_date=transition.get('Date') if transition else None,
|
||||
storage_class=transition['StorageClass'] if transition else None,
|
||||
storage_class=transition[
|
||||
'StorageClass'] if transition else None,
|
||||
))
|
||||
|
||||
def delete_lifecycle(self):
|
||||
|
|
@ -283,9 +296,11 @@ class FakeBucket(object):
|
|||
def get_cfn_attribute(self, attribute_name):
|
||||
from moto.cloudformation.exceptions import UnformattedGetAttTemplateException
|
||||
if attribute_name == 'DomainName':
|
||||
raise NotImplementedError('"Fn::GetAtt" : [ "{0}" , "DomainName" ]"')
|
||||
raise NotImplementedError(
|
||||
'"Fn::GetAtt" : [ "{0}" , "DomainName" ]"')
|
||||
elif attribute_name == 'WebsiteURL':
|
||||
raise NotImplementedError('"Fn::GetAtt" : [ "{0}" , "WebsiteURL" ]"')
|
||||
raise NotImplementedError(
|
||||
'"Fn::GetAtt" : [ "{0}" , "WebsiteURL" ]"')
|
||||
raise UnformattedGetAttTemplateException()
|
||||
|
||||
def set_acl(self, acl):
|
||||
|
|
@ -470,20 +485,24 @@ class S3Backend(BaseBackend):
|
|||
key_without_prefix = key_name.replace(prefix, "", 1)
|
||||
if delimiter and delimiter in key_without_prefix:
|
||||
# If delimiter, we need to split out folder_results
|
||||
key_without_delimiter = key_without_prefix.split(delimiter)[0]
|
||||
folder_results.add("{0}{1}{2}".format(prefix, key_without_delimiter, delimiter))
|
||||
key_without_delimiter = key_without_prefix.split(delimiter)[
|
||||
0]
|
||||
folder_results.add("{0}{1}{2}".format(
|
||||
prefix, key_without_delimiter, delimiter))
|
||||
else:
|
||||
key_results.add(key)
|
||||
else:
|
||||
for key_name, key in bucket.keys.items():
|
||||
if delimiter and delimiter in key_name:
|
||||
# If delimiter, we need to split out folder_results
|
||||
folder_results.add(key_name.split(delimiter)[0] + delimiter)
|
||||
folder_results.add(key_name.split(
|
||||
delimiter)[0] + delimiter)
|
||||
else:
|
||||
key_results.add(key)
|
||||
|
||||
key_results = sorted(key_results, key=lambda key: key.name)
|
||||
folder_results = [folder_name for folder_name in sorted(folder_results, key=lambda key: key)]
|
||||
folder_results = [folder_name for folder_name in sorted(
|
||||
folder_results, key=lambda key: key)]
|
||||
|
||||
return key_results, folder_results
|
||||
|
||||
|
|
@ -502,7 +521,8 @@ class S3Backend(BaseBackend):
|
|||
src_key_name = clean_key_name(src_key_name)
|
||||
dest_key_name = clean_key_name(dest_key_name)
|
||||
dest_bucket = self.get_bucket(dest_bucket_name)
|
||||
key = self.get_key(src_bucket_name, src_key_name, version_id=src_version_id)
|
||||
key = self.get_key(src_bucket_name, src_key_name,
|
||||
version_id=src_version_id)
|
||||
if dest_key_name != src_key_name:
|
||||
key = key.copy(dest_key_name)
|
||||
dest_bucket.keys[dest_key_name] = key
|
||||
|
|
|
|||
|
|
@ -33,6 +33,7 @@ def is_delete_keys(request, path, bucket_name):
|
|||
|
||||
|
||||
class ResponseObject(_TemplateEnvironmentMixin):
|
||||
|
||||
def __init__(self, backend):
|
||||
super(ResponseObject, self).__init__()
|
||||
self.backend = backend
|
||||
|
|
@ -70,7 +71,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
|||
if match:
|
||||
return False
|
||||
|
||||
path_based = (host == 's3.amazonaws.com' or re.match(r"s3[\.\-]([^.]*)\.amazonaws\.com", host))
|
||||
path_based = (host == 's3.amazonaws.com' or re.match(
|
||||
r"s3[\.\-]([^.]*)\.amazonaws\.com", host))
|
||||
return not path_based
|
||||
|
||||
def is_delete_keys(self, request, path, bucket_name):
|
||||
|
|
@ -148,7 +150,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
|||
elif method == 'POST':
|
||||
return self._bucket_response_post(request, body, bucket_name, headers)
|
||||
else:
|
||||
raise NotImplementedError("Method {0} has not been impelemented in the S3 backend yet".format(method))
|
||||
raise NotImplementedError(
|
||||
"Method {0} has not been impelemented in the S3 backend yet".format(method))
|
||||
|
||||
def _bucket_response_head(self, bucket_name, headers):
|
||||
self.backend.get_bucket(bucket_name)
|
||||
|
|
@ -158,11 +161,14 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
|||
if 'uploads' in querystring:
|
||||
for unsup in ('delimiter', 'max-uploads'):
|
||||
if unsup in querystring:
|
||||
raise NotImplementedError("Listing multipart uploads with {} has not been implemented yet.".format(unsup))
|
||||
multiparts = list(self.backend.get_all_multiparts(bucket_name).values())
|
||||
raise NotImplementedError(
|
||||
"Listing multipart uploads with {} has not been implemented yet.".format(unsup))
|
||||
multiparts = list(
|
||||
self.backend.get_all_multiparts(bucket_name).values())
|
||||
if 'prefix' in querystring:
|
||||
prefix = querystring.get('prefix', [None])[0]
|
||||
multiparts = [upload for upload in multiparts if upload.key_name.startswith(prefix)]
|
||||
multiparts = [
|
||||
upload for upload in multiparts if upload.key_name.startswith(prefix)]
|
||||
template = self.response_template(S3_ALL_MULTIPARTS)
|
||||
return template.render(
|
||||
bucket_name=bucket_name,
|
||||
|
|
@ -175,7 +181,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
|||
bucket = self.backend.get_bucket(bucket_name)
|
||||
if not bucket.rules:
|
||||
return 404, {}, "NoSuchLifecycleConfiguration"
|
||||
template = self.response_template(S3_BUCKET_LIFECYCLE_CONFIGURATION)
|
||||
template = self.response_template(
|
||||
S3_BUCKET_LIFECYCLE_CONFIGURATION)
|
||||
return template.render(rules=bucket.rules)
|
||||
elif 'versioning' in querystring:
|
||||
versioning = self.backend.get_bucket_versioning(bucket_name)
|
||||
|
|
@ -188,7 +195,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
|||
return 404, {}, template.render(bucket_name=bucket_name)
|
||||
return 200, {}, policy
|
||||
elif 'website' in querystring:
|
||||
website_configuration = self.backend.get_bucket_website_configuration(bucket_name)
|
||||
website_configuration = self.backend.get_bucket_website_configuration(
|
||||
bucket_name)
|
||||
return website_configuration
|
||||
elif 'acl' in querystring:
|
||||
bucket = self.backend.get_bucket(bucket_name)
|
||||
|
|
@ -226,7 +234,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
|||
bucket = self.backend.get_bucket(bucket_name)
|
||||
prefix = querystring.get('prefix', [None])[0]
|
||||
delimiter = querystring.get('delimiter', [None])[0]
|
||||
result_keys, result_folders = self.backend.prefix_query(bucket, prefix, delimiter)
|
||||
result_keys, result_folders = self.backend.prefix_query(
|
||||
bucket, prefix, delimiter)
|
||||
template = self.response_template(S3_BUCKET_GET_RESPONSE)
|
||||
return 200, {}, template.render(
|
||||
bucket=bucket,
|
||||
|
|
@ -242,7 +251,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
|||
|
||||
prefix = querystring.get('prefix', [None])[0]
|
||||
delimiter = querystring.get('delimiter', [None])[0]
|
||||
result_keys, result_folders = self.backend.prefix_query(bucket, prefix, delimiter)
|
||||
result_keys, result_folders = self.backend.prefix_query(
|
||||
bucket, prefix, delimiter)
|
||||
|
||||
fetch_owner = querystring.get('fetch-owner', [False])[0]
|
||||
max_keys = int(querystring.get('max-keys', [1000])[0])
|
||||
|
|
@ -308,7 +318,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
|||
return ""
|
||||
else:
|
||||
try:
|
||||
new_bucket = self.backend.create_bucket(bucket_name, region_name)
|
||||
new_bucket = self.backend.create_bucket(
|
||||
bucket_name, region_name)
|
||||
except BucketAlreadyExists:
|
||||
if region_name == DEFAULT_REGION_NAME:
|
||||
# us-east-1 has different behavior
|
||||
|
|
@ -335,7 +346,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
|||
return 204, {}, template.render(bucket=removed_bucket)
|
||||
else:
|
||||
# Tried to delete a bucket that still has keys
|
||||
template = self.response_template(S3_DELETE_BUCKET_WITH_ITEMS_ERROR)
|
||||
template = self.response_template(
|
||||
S3_DELETE_BUCKET_WITH_ITEMS_ERROR)
|
||||
return 409, {}, template.render(bucket=removed_bucket)
|
||||
|
||||
def _bucket_response_post(self, request, body, bucket_name, headers):
|
||||
|
|
@ -393,7 +405,9 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
|||
if ',' in rspec:
|
||||
raise NotImplementedError(
|
||||
"Multiple range specifiers not supported")
|
||||
toint = lambda i: int(i) if i else None
|
||||
|
||||
def toint(i):
|
||||
return int(i) if i else None
|
||||
begin, end = map(toint, rspec.split('-'))
|
||||
if begin is not None: # byte range
|
||||
end = last if end is None else min(end, last)
|
||||
|
|
@ -455,7 +469,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
|||
elif method == 'POST':
|
||||
return self._key_response_post(request, body, bucket_name, query, key_name, headers)
|
||||
else:
|
||||
raise NotImplementedError("Method {0} has not been impelemented in the S3 backend yet".format(method))
|
||||
raise NotImplementedError(
|
||||
"Method {0} has not been impelemented in the S3 backend yet".format(method))
|
||||
|
||||
def _key_response_get(self, bucket_name, query, key_name, headers):
|
||||
response_headers = {}
|
||||
|
|
@ -489,7 +504,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
|||
if 'x-amz-copy-source' in request.headers:
|
||||
src = request.headers.get("x-amz-copy-source")
|
||||
src_bucket, src_key = src.split("/", 1)
|
||||
src_range = request.headers.get('x-amz-copy-source-range', '').split("bytes=")[-1]
|
||||
src_range = request.headers.get(
|
||||
'x-amz-copy-source-range', '').split("bytes=")[-1]
|
||||
|
||||
try:
|
||||
start_byte, end_byte = src_range.split("-")
|
||||
|
|
@ -522,7 +538,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
|||
# Copy key
|
||||
src_key_parsed = urlparse(request.headers.get("x-amz-copy-source"))
|
||||
src_bucket, src_key = src_key_parsed.path.split("/", 1)
|
||||
src_version_id = parse_qs(src_key_parsed.query).get('versionId', [None])[0]
|
||||
src_version_id = parse_qs(src_key_parsed.query).get(
|
||||
'versionId', [None])[0]
|
||||
self.backend.copy_key(src_bucket, src_key, bucket_name, key_name,
|
||||
storage=storage_class, acl=acl, src_version_id=src_version_id)
|
||||
new_key = self.backend.get_key(bucket_name, key_name)
|
||||
|
|
@ -557,7 +574,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
|||
def _key_response_head(self, bucket_name, query, key_name, headers):
|
||||
response_headers = {}
|
||||
version_id = query.get('versionId', [None])[0]
|
||||
key = self.backend.get_key(bucket_name, key_name, version_id=version_id)
|
||||
key = self.backend.get_key(
|
||||
bucket_name, key_name, version_id=version_id)
|
||||
if key:
|
||||
response_headers.update(key.metadata)
|
||||
response_headers.update(key.response_dict)
|
||||
|
|
@ -585,7 +603,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
|||
|
||||
grantees = []
|
||||
for key_and_value in value.split(","):
|
||||
key, value = re.match('([^=]+)="([^"]+)"', key_and_value.strip()).groups()
|
||||
key, value = re.match(
|
||||
'([^=]+)="([^"]+)"', key_and_value.strip()).groups()
|
||||
if key.lower() == 'id':
|
||||
grantees.append(FakeGrantee(id=value))
|
||||
else:
|
||||
|
|
@ -610,7 +629,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
|||
ps = minidom.parseString(body).getElementsByTagName('Part')
|
||||
prev = 0
|
||||
for p in ps:
|
||||
pn = int(p.getElementsByTagName('PartNumber')[0].firstChild.wholeText)
|
||||
pn = int(p.getElementsByTagName(
|
||||
'PartNumber')[0].firstChild.wholeText)
|
||||
if pn <= prev:
|
||||
raise InvalidPartOrder()
|
||||
yield (pn, p.getElementsByTagName('ETag')[0].firstChild.wholeText)
|
||||
|
|
@ -618,7 +638,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
|||
def _key_response_post(self, request, body, bucket_name, query, key_name, headers):
|
||||
if body == b'' and 'uploads' in query:
|
||||
metadata = metadata_from_headers(request.headers)
|
||||
multipart = self.backend.initiate_multipart(bucket_name, key_name, metadata)
|
||||
multipart = self.backend.initiate_multipart(
|
||||
bucket_name, key_name, metadata)
|
||||
|
||||
template = self.response_template(S3_MULTIPART_INITIATE_RESPONSE)
|
||||
response = template.render(
|
||||
|
|
@ -648,7 +669,9 @@ class ResponseObject(_TemplateEnvironmentMixin):
|
|||
key.restore(int(days))
|
||||
return r, {}, ""
|
||||
else:
|
||||
raise NotImplementedError("Method POST had only been implemented for multipart uploads and restore operations, so far")
|
||||
raise NotImplementedError(
|
||||
"Method POST had only been implemented for multipart uploads and restore operations, so far")
|
||||
|
||||
|
||||
S3ResponseInstance = ResponseObject(s3_backend)
|
||||
|
||||
|
|
|
|||
|
|
@ -29,7 +29,8 @@ def bucket_name_from_url(url):
|
|||
|
||||
def metadata_from_headers(headers):
|
||||
metadata = {}
|
||||
meta_regex = re.compile('^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE)
|
||||
meta_regex = re.compile(
|
||||
'^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE)
|
||||
for header, value in headers.items():
|
||||
if isinstance(header, six.string_types):
|
||||
result = meta_regex.match(header)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue