Merge remote-tracking branch 'spulec/master'
Conflicts: moto/s3/responses.py
This commit is contained in:
Konstantinos Koukopoulos
commit
3628e40f3c
42 changed files with 1173 additions and 308 deletions
|
|
@ -186,7 +186,7 @@ class S3Backend(BaseBackend):
|
|||
if delimiter and delimiter in key_without_prefix:
|
||||
# If delimiter, we need to split out folder_results
|
||||
key_without_delimiter = key_without_prefix.split(delimiter)[0]
|
||||
folder_results.add("{}{}{}".format(prefix, key_without_delimiter, delimiter))
|
||||
folder_results.add("{0}{1}{2}".format(prefix, key_without_delimiter, delimiter))
|
||||
else:
|
||||
key_results.add(key)
|
||||
else:
|
||||
|
|
|
|||
|
|
@ -7,223 +7,233 @@ from .models import s3_backend
|
|||
from .utils import bucket_name_from_url
|
||||
|
||||
|
||||
def all_buckets():
|
||||
# No bucket specified. Listing all buckets
|
||||
all_buckets = s3_backend.get_all_buckets()
|
||||
template = Template(S3_ALL_BUCKETS)
|
||||
return template.render(buckets=all_buckets)
|
||||
def parse_key_name(pth):
|
||||
return pth.lstrip("/")
|
||||
|
||||
|
||||
def bucket_response(request, full_url, headers):
|
||||
response = _bucket_response(request, full_url, headers)
|
||||
if isinstance(response, basestring):
|
||||
return 200, headers, response
|
||||
class ResponseObject(object):
|
||||
def __init__(self, backend, bucket_name_from_url, parse_key_name):
|
||||
self.backend = backend
|
||||
self.bucket_name_from_url = bucket_name_from_url
|
||||
self.parse_key_name = parse_key_name
|
||||
|
||||
else:
|
||||
status_code, headers, response_content = response
|
||||
return status_code, headers, response_content
|
||||
def all_buckets(self):
|
||||
# No bucket specified. Listing all buckets
|
||||
all_buckets = self.backend.get_all_buckets()
|
||||
template = Template(S3_ALL_BUCKETS)
|
||||
return template.render(buckets=all_buckets)
|
||||
|
||||
|
||||
def _bucket_response(request, full_url, headers):
|
||||
parsed_url = urlparse(full_url)
|
||||
querystring = parse_qs(parsed_url.query)
|
||||
method = request.method
|
||||
|
||||
bucket_name = bucket_name_from_url(full_url)
|
||||
if not bucket_name:
|
||||
# If no bucket specified, list all buckets
|
||||
return all_buckets()
|
||||
|
||||
if method == 'GET':
|
||||
bucket = s3_backend.get_bucket(bucket_name)
|
||||
if bucket:
|
||||
prefix = querystring.get('prefix', [None])[0]
|
||||
delimiter = querystring.get('delimiter', [None])[0]
|
||||
result_keys, result_folders = s3_backend.prefix_query(bucket, prefix, delimiter)
|
||||
template = Template(S3_BUCKET_GET_RESPONSE)
|
||||
return template.render(
|
||||
bucket=bucket,
|
||||
prefix=prefix,
|
||||
delimiter=delimiter,
|
||||
result_keys=result_keys,
|
||||
result_folders=result_folders
|
||||
)
|
||||
else:
|
||||
return 404, headers, ""
|
||||
elif method == 'PUT':
|
||||
new_bucket = s3_backend.create_bucket(bucket_name)
|
||||
template = Template(S3_BUCKET_CREATE_RESPONSE)
|
||||
return template.render(bucket=new_bucket)
|
||||
elif method == 'DELETE':
|
||||
removed_bucket = s3_backend.delete_bucket(bucket_name)
|
||||
if removed_bucket is None:
|
||||
# Non-existant bucket
|
||||
template = Template(S3_DELETE_NON_EXISTING_BUCKET)
|
||||
return 404, headers, template.render(bucket_name=bucket_name)
|
||||
elif removed_bucket:
|
||||
# Bucket exists
|
||||
template = Template(S3_DELETE_BUCKET_SUCCESS)
|
||||
return 204, headers, template.render(bucket=removed_bucket)
|
||||
else:
|
||||
# Tried to delete a bucket that still has keys
|
||||
template = Template(S3_DELETE_BUCKET_WITH_ITEMS_ERROR)
|
||||
return 409, headers, template.render(bucket=removed_bucket)
|
||||
elif method == 'POST':
|
||||
#POST to bucket-url should create file from form
|
||||
if hasattr(request, 'form'):
|
||||
#Not HTTPretty
|
||||
form = request.form
|
||||
else:
|
||||
#HTTPretty, build new form object
|
||||
form = {}
|
||||
for kv in request.body.split('&'):
|
||||
k, v = kv.split('=')
|
||||
form[k] = v
|
||||
|
||||
key = form['key']
|
||||
f = form['file']
|
||||
|
||||
new_key = s3_backend.set_key(bucket_name, key, f)
|
||||
|
||||
#Metadata
|
||||
meta_regex = re.compile('^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE)
|
||||
for form_id in form:
|
||||
result = meta_regex.match(form_id)
|
||||
if result:
|
||||
meta_key = result.group(0).lower()
|
||||
metadata = form[form_id]
|
||||
new_key.set_metadata(meta_key, metadata)
|
||||
return 200, headers, ""
|
||||
else:
|
||||
raise NotImplementedError("Method {} has not been impelemented in the S3 backend yet".format(method))
|
||||
|
||||
|
||||
def key_response(request, full_url, headers):
|
||||
response = _key_response(request, full_url, headers)
|
||||
if isinstance(response, basestring):
|
||||
return 200, headers, response
|
||||
else:
|
||||
status_code, headers, response_content = response
|
||||
return status_code, headers, response_content
|
||||
|
||||
|
||||
def _key_response(request, full_url, headers):
|
||||
parsed_url = urlparse(full_url)
|
||||
method = request.method
|
||||
|
||||
key_name = parsed_url.path.lstrip('/')
|
||||
query = parse_qs(parsed_url.query)
|
||||
bucket_name = bucket_name_from_url(full_url)
|
||||
if hasattr(request, 'body'):
|
||||
# Boto
|
||||
body = request.body
|
||||
else:
|
||||
# Flask server
|
||||
body = request.data
|
||||
|
||||
if method == 'GET':
|
||||
if 'uploadId' in query:
|
||||
upload_id = query['uploadId'][0]
|
||||
parts = s3_backend.list_multipart(bucket_name, upload_id)
|
||||
template = Template(S3_MULTIPART_LIST_RESPONSE)
|
||||
return 200, headers, template.render(
|
||||
bucket_name=bucket_name,
|
||||
key_name=key_name,
|
||||
upload_id=upload_id,
|
||||
count=len(parts),
|
||||
parts=parts
|
||||
)
|
||||
key = s3_backend.get_key(bucket_name, key_name)
|
||||
if key:
|
||||
headers.update(key.metadata)
|
||||
return 200, headers, key.value
|
||||
else:
|
||||
return 404, headers, ""
|
||||
if method == 'PUT':
|
||||
if 'uploadId' in query and 'partNumber' in query and body:
|
||||
upload_id = query['uploadId'][0]
|
||||
part_number = int(query['partNumber'][0])
|
||||
key = s3_backend.set_part(bucket_name, upload_id, part_number, body)
|
||||
template = Template(S3_MULTIPART_UPLOAD_RESPONSE)
|
||||
headers.update(key.response_dict)
|
||||
return 200, headers, template.render(part=key)
|
||||
|
||||
if 'x-amz-copy-source' in request.headers:
|
||||
# Copy key
|
||||
src_bucket, src_key = request.headers.get("x-amz-copy-source").split("/")
|
||||
s3_backend.copy_key(src_bucket, src_key, bucket_name, key_name)
|
||||
template = Template(S3_OBJECT_COPY_RESPONSE)
|
||||
return template.render(key=src_key)
|
||||
streaming_request = hasattr(request, 'streaming') and request.streaming
|
||||
closing_connection = headers.get('connection') == 'close'
|
||||
if closing_connection and streaming_request:
|
||||
# Closing the connection of a streaming request. No more data
|
||||
new_key = s3_backend.get_key(bucket_name, key_name)
|
||||
elif streaming_request:
|
||||
# Streaming request, more data
|
||||
new_key = s3_backend.append_to_key(bucket_name, key_name, body)
|
||||
else:
|
||||
# Initial data
|
||||
new_key = s3_backend.set_key(bucket_name, key_name, body)
|
||||
request.streaming = True
|
||||
|
||||
#Metadata
|
||||
meta_regex = re.compile('^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE)
|
||||
for header in request.headers:
|
||||
if isinstance(header, basestring):
|
||||
result = meta_regex.match(header)
|
||||
if result:
|
||||
meta_key = result.group(0).lower()
|
||||
metadata = request.headers[header]
|
||||
new_key.set_metadata(meta_key, metadata)
|
||||
template = Template(S3_OBJECT_RESPONSE)
|
||||
headers.update(new_key.response_dict)
|
||||
return 200, headers, template.render(key=new_key)
|
||||
elif method == 'HEAD':
|
||||
key = s3_backend.get_key(bucket_name, key_name)
|
||||
if key:
|
||||
headers.update(key.metadata)
|
||||
headers.update(key.response_dict)
|
||||
return 200, headers, ""
|
||||
else:
|
||||
return 404, headers, ""
|
||||
elif method == 'DELETE':
|
||||
if 'uploadId' in query:
|
||||
upload_id = query['uploadId'][0]
|
||||
s3_backend.cancel_multipart(bucket_name, upload_id)
|
||||
return 204, headers, ""
|
||||
removed_key = s3_backend.delete_key(bucket_name, key_name)
|
||||
template = Template(S3_DELETE_OBJECT_SUCCESS)
|
||||
return 204, headers, template.render(bucket=removed_key)
|
||||
elif method == 'POST':
|
||||
if body == '' and parsed_url.query == 'uploads':
|
||||
multipart = s3_backend.initiate_multipart(bucket_name, key_name)
|
||||
template = Template(S3_MULTIPART_INITIATE_RESPONSE)
|
||||
response = template.render(
|
||||
bucket_name=bucket_name,
|
||||
key_name=key_name,
|
||||
upload_id=multipart.id,
|
||||
)
|
||||
def bucket_response(self, request, full_url, headers):
|
||||
response = self._bucket_response(request, full_url, headers)
|
||||
if isinstance(response, basestring):
|
||||
return 200, headers, response
|
||||
|
||||
if 'uploadId' in query:
|
||||
upload_id = query['uploadId'][0]
|
||||
key = s3_backend.complete_multipart(bucket_name, upload_id)
|
||||
|
||||
if key is not None:
|
||||
template = Template(S3_MULTIPART_COMPLETE_RESPONSE)
|
||||
return template.render(
|
||||
bucket_name=bucket_name,
|
||||
key_name=key.name,
|
||||
etag=key.etag,
|
||||
)
|
||||
template = Template(S3_MULTIPART_COMPLETE_TOO_SMALL_ERROR)
|
||||
return 400, headers, template.render()
|
||||
else:
|
||||
raise NotImplementedError("Method POST had only been implemented for multipart uploads so far")
|
||||
else:
|
||||
raise NotImplementedError("Method {} has not been impelemented in the S3 backend yet".format(method))
|
||||
status_code, headers, response_content = response
|
||||
return status_code, headers, response_content
|
||||
|
||||
def _bucket_response(self, request, full_url, headers):
|
||||
parsed_url = urlparse(full_url)
|
||||
querystring = parse_qs(parsed_url.query)
|
||||
method = request.method
|
||||
|
||||
bucket_name = self.bucket_name_from_url(full_url)
|
||||
if not bucket_name:
|
||||
# If no bucket specified, list all buckets
|
||||
return self.all_buckets()
|
||||
|
||||
if method == 'GET':
|
||||
bucket = self.backend.get_bucket(bucket_name)
|
||||
if bucket:
|
||||
prefix = querystring.get('prefix', [None])[0]
|
||||
delimiter = querystring.get('delimiter', [None])[0]
|
||||
result_keys, result_folders = self.backend.prefix_query(bucket, prefix, delimiter)
|
||||
template = Template(S3_BUCKET_GET_RESPONSE)
|
||||
return template.render(
|
||||
bucket=bucket,
|
||||
prefix=prefix,
|
||||
delimiter=delimiter,
|
||||
result_keys=result_keys,
|
||||
result_folders=result_folders
|
||||
)
|
||||
else:
|
||||
return 404, headers, ""
|
||||
elif method == 'PUT':
|
||||
new_bucket = self.backend.create_bucket(bucket_name)
|
||||
template = Template(S3_BUCKET_CREATE_RESPONSE)
|
||||
return template.render(bucket=new_bucket)
|
||||
elif method == 'DELETE':
|
||||
removed_bucket = self.backend.delete_bucket(bucket_name)
|
||||
if removed_bucket is None:
|
||||
# Non-existant bucket
|
||||
template = Template(S3_DELETE_NON_EXISTING_BUCKET)
|
||||
return 404, headers, template.render(bucket_name=bucket_name)
|
||||
elif removed_bucket:
|
||||
# Bucket exists
|
||||
template = Template(S3_DELETE_BUCKET_SUCCESS)
|
||||
return 204, headers, template.render(bucket=removed_bucket)
|
||||
else:
|
||||
# Tried to delete a bucket that still has keys
|
||||
template = Template(S3_DELETE_BUCKET_WITH_ITEMS_ERROR)
|
||||
return 409, headers, template.render(bucket=removed_bucket)
|
||||
elif method == 'POST':
|
||||
#POST to bucket-url should create file from form
|
||||
if hasattr(request, 'form'):
|
||||
#Not HTTPretty
|
||||
form = request.form
|
||||
else:
|
||||
#HTTPretty, build new form object
|
||||
form = {}
|
||||
for kv in request.body.split('&'):
|
||||
k, v = kv.split('=')
|
||||
form[k] = v
|
||||
|
||||
key = form['key']
|
||||
f = form['file']
|
||||
|
||||
new_key = self.backend.set_key(bucket_name, key, f)
|
||||
|
||||
#Metadata
|
||||
meta_regex = re.compile('^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE)
|
||||
|
||||
for form_id in form:
|
||||
result = meta_regex.match(form_id)
|
||||
if result:
|
||||
meta_key = result.group(0).lower()
|
||||
metadata = form[form_id]
|
||||
new_key.set_metadata(meta_key, metadata)
|
||||
return 200, headers, ""
|
||||
else:
|
||||
raise NotImplementedError("Method {0} has not been impelemented in the S3 backend yet".format(method))
|
||||
|
||||
def key_response(self, request, full_url, headers):
|
||||
response = self._key_response(request, full_url, headers)
|
||||
if isinstance(response, basestring):
|
||||
return 200, headers, response
|
||||
else:
|
||||
status_code, headers, response_content = response
|
||||
return status_code, headers, response_content
|
||||
|
||||
def _key_response(self, request, full_url, headers):
|
||||
parsed_url = urlparse(full_url)
|
||||
query = parse_qs(parsed_url.query)
|
||||
method = request.method
|
||||
|
||||
key_name = self.parse_key_name(parsed_url.path)
|
||||
|
||||
bucket_name = self.bucket_name_from_url(full_url)
|
||||
|
||||
if hasattr(request, 'body'):
|
||||
# Boto
|
||||
body = request.body
|
||||
else:
|
||||
# Flask server
|
||||
body = request.data
|
||||
|
||||
if method == 'GET':
|
||||
if 'uploadId' in query:
|
||||
upload_id = query['uploadId'][0]
|
||||
parts = self.backend.list_multipart(bucket_name, upload_id)
|
||||
template = Template(S3_MULTIPART_LIST_RESPONSE)
|
||||
return 200, headers, template.render(
|
||||
bucket_name=bucket_name,
|
||||
key_name=key_name,
|
||||
upload_id=upload_id,
|
||||
count=len(parts),
|
||||
parts=parts
|
||||
)
|
||||
key = self.backend.get_key(bucket_name, key_name)
|
||||
if key:
|
||||
headers.update(key.metadata)
|
||||
return 200, headers, key.value
|
||||
else:
|
||||
return 404, headers, ""
|
||||
if method == 'PUT':
|
||||
if 'uploadId' in query and 'partNumber' in query and body:
|
||||
upload_id = query['uploadId'][0]
|
||||
part_number = int(query['partNumber'][0])
|
||||
key = self.backend.set_part(bucket_name, upload_id, part_number, body)
|
||||
template = Template(S3_MULTIPART_UPLOAD_RESPONSE)
|
||||
headers.update(key.response_dict)
|
||||
return 200, headers, template.render(part=key)
|
||||
|
||||
if 'x-amz-copy-source' in request.headers:
|
||||
# Copy key
|
||||
src_bucket, src_key = request.headers.get("x-amz-copy-source").split("/", 1)
|
||||
self.backend.copy_key(src_bucket, src_key, bucket_name, key_name)
|
||||
template = Template(S3_OBJECT_COPY_RESPONSE)
|
||||
return template.render(key=src_key)
|
||||
streaming_request = hasattr(request, 'streaming') and request.streaming
|
||||
closing_connection = headers.get('connection') == 'close'
|
||||
if closing_connection and streaming_request:
|
||||
# Closing the connection of a streaming request. No more data
|
||||
new_key = self.backend.get_key(bucket_name, key_name)
|
||||
elif streaming_request:
|
||||
# Streaming request, more data
|
||||
new_key = self.backend.append_to_key(bucket_name, key_name, body)
|
||||
else:
|
||||
# Initial data
|
||||
new_key = self.backend.set_key(bucket_name, key_name, body)
|
||||
request.streaming = True
|
||||
|
||||
#Metadata
|
||||
meta_regex = re.compile('^x-amz-meta-([a-zA-Z0-9\-_]+)$', flags=re.IGNORECASE)
|
||||
for header in request.headers:
|
||||
if isinstance(header, basestring):
|
||||
result = meta_regex.match(header)
|
||||
if result:
|
||||
meta_key = result.group(0).lower()
|
||||
metadata = request.headers[header]
|
||||
new_key.set_metadata(meta_key, metadata)
|
||||
template = Template(S3_OBJECT_RESPONSE)
|
||||
headers.update(new_key.response_dict)
|
||||
return 200, headers, template.render(key=new_key)
|
||||
elif method == 'HEAD':
|
||||
key = self.backend.get_key(bucket_name, key_name)
|
||||
if key:
|
||||
headers.update(key.metadata)
|
||||
headers.update(key.response_dict)
|
||||
return 200, headers, ""
|
||||
else:
|
||||
return 404, headers, ""
|
||||
elif method == 'DELETE':
|
||||
if 'uploadId' in query:
|
||||
upload_id = query['uploadId'][0]
|
||||
self.backend.cancel_multipart(bucket_name, upload_id)
|
||||
return 204, headers, ""
|
||||
removed_key = self.backend.delete_key(bucket_name, key_name)
|
||||
template = Template(S3_DELETE_OBJECT_SUCCESS)
|
||||
return 204, headers, template.render(bucket=removed_key)
|
||||
elif method == 'POST':
|
||||
if body == '' and parsed_url.query == 'uploads':
|
||||
multipart = self.backend.initiate_multipart(bucket_name, key_name)
|
||||
template = Template(S3_MULTIPART_INITIATE_RESPONSE)
|
||||
response = template.render(
|
||||
bucket_name=bucket_name,
|
||||
key_name=key_name,
|
||||
upload_id=multipart.id,
|
||||
)
|
||||
return 200, headers, response
|
||||
|
||||
if 'uploadId' in query:
|
||||
upload_id = query['uploadId'][0]
|
||||
key = self.backend.complete_multipart(bucket_name, upload_id)
|
||||
|
||||
if key is not None:
|
||||
template = Template(S3_MULTIPART_COMPLETE_RESPONSE)
|
||||
return template.render(
|
||||
bucket_name=bucket_name,
|
||||
key_name=key.name,
|
||||
etag=key.etag,
|
||||
)
|
||||
template = Template(S3_MULTIPART_COMPLETE_TOO_SMALL_ERROR)
|
||||
return 400, headers, template.render()
|
||||
else:
|
||||
raise NotImplementedError("Method POST had only been implemented for multipart uploads so far")
|
||||
else:
|
||||
raise NotImplementedError("Method {0} has not been impelemented in the S3 backend yet".format(method))
|
||||
|
||||
S3ResponseInstance = ResponseObject(s3_backend, bucket_name_from_url, parse_key_name)
|
||||
|
||||
S3_ALL_BUCKETS = """<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01">
|
||||
<Owner>
|
||||
|
|
|
|||
|
|
@ -1,10 +1,10 @@
|
|||
from .responses import bucket_response, key_response
|
||||
from .responses import S3ResponseInstance
|
||||
|
||||
url_bases = [
|
||||
"https?://(?P<bucket_name>[a-zA-Z0-9\-_.]*)\.?s3.amazonaws.com"
|
||||
]
|
||||
|
||||
url_paths = {
|
||||
'{0}/$': bucket_response,
|
||||
'{0}/(?P<key_name>[a-zA-Z0-9\-_.]+)': key_response,
|
||||
'{0}/$': S3ResponseInstance.bucket_response,
|
||||
'{0}/(?P<key_name>[a-zA-Z0-9\-_.]+)': S3ResponseInstance.key_response,
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue