diff --git a/AUTHORS.md b/AUTHORS.md index 90ed0b35..c1bb7ef7 100644 --- a/AUTHORS.md +++ b/AUTHORS.md @@ -20,3 +20,4 @@ Moto is written by Steve Pulec with contributions from: * [Chris St. Pierre](https://github.com/stpierre) * [Frank Mata](https://github.com/matafc) * [Clint Ecker](https://github.com/clintecker) +* [Richard Eames](https://github.com/Naddiseo) diff --git a/moto/s3/models.py b/moto/s3/models.py index 07aaee51..67b40d5e 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -3,18 +3,20 @@ import base64 import datetime import hashlib import copy +import itertools from moto.core import BaseBackend from moto.core.utils import iso_8601_datetime, rfc_1123_datetime from .exceptions import BucketAlreadyExists -from .utils import clean_key_name +from .utils import clean_key_name, _VersionedKeyStore UPLOAD_ID_BYTES = 43 UPLOAD_PART_MIN_SIZE = 5242880 class FakeKey(object): - def __init__(self, name, value, storage="STANDARD", etag=None): + + def __init__(self, name, value, storage="STANDARD", etag=None, is_versioned=False, version_id=0): self.name = name self.value = value self.last_modified = datetime.datetime.now() @@ -22,6 +24,8 @@ class FakeKey(object): self._metadata = {} self._expiry = None self._etag = etag + self._version_id = version_id + self._is_versioned = is_versioned def copy(self, new_name=None): r = copy.deepcopy(self) @@ -42,6 +46,10 @@ class FakeKey(object): self.value += value self.last_modified = datetime.datetime.now() self._etag = None # must recalculate etag + if self._is_versioned: + self._version_id += 1 + else: + self._is_versioned = 0 def restore(self, days): self._expiry = datetime.datetime.now() + datetime.timedelta(days) @@ -79,6 +87,10 @@ class FakeKey(object): if self._expiry is not None: rhdr = 'ongoing-request="false", expiry-date="{0}"' r['x-amz-restore'] = rhdr.format(self.expiry_date) + + if self._is_versioned: + r['x-amz-version-id'] = self._version_id + return r @property @@ -137,10 +149,16 @@ class FakeMultipart(object): class FakeBucket(object): + def __init__(self, name): self.name = name - self.keys = {} + self.keys = _VersionedKeyStore() self.multiparts = {} + self.versioning_status = None + + @property + def is_versioned(self): + return self.versioning_status == 'Enabled' class S3Backend(BaseBackend): @@ -171,12 +189,42 @@ class S3Backend(BaseBackend): return self.buckets.pop(bucket_name) return None + def set_bucket_versioning(self, bucket_name, status): + self.buckets[bucket_name].versioning_status = status + + def get_bucket_versioning(self, bucket_name): + return self.buckets[bucket_name].versioning_status + + def get_bucket_versions(self, bucket_name, delimiter=None, + encoding_type=None, + key_marker=None, + max_keys=None, + version_id_marker=None): + bucket = self.buckets[bucket_name] + + if any((delimiter, encoding_type, key_marker, version_id_marker)): + raise NotImplementedError( + "Called get_bucket_versions with some of delimiter, encoding_type, key_marker, version_id_marker") + + return itertools.chain(*(l for _, l in bucket.keys.iterlists())) def set_key(self, bucket_name, key_name, value, storage=None, etag=None): key_name = clean_key_name(key_name) bucket = self.buckets[bucket_name] - new_key = FakeKey(name=key_name, value=value, - storage=storage, etag=etag) + + old_key = bucket.keys.get(key_name, None) + if old_key is not None and bucket.is_versioned: + new_version_id = old_key._version_id + 1 + else: + new_version_id = 0 + + new_key = FakeKey( + name=key_name, + value=value, + storage=storage, + etag=etag, + is_versioned=bucket.is_versioned, + version_id=new_version_id) bucket.keys[key_name] = new_key return new_key @@ -188,11 +236,16 @@ class S3Backend(BaseBackend): key.append_to_value(value) return key - def get_key(self, bucket_name, key_name): + def get_key(self, bucket_name, key_name, version_id=None): key_name = clean_key_name(key_name) bucket = self.get_bucket(bucket_name) if bucket: - return bucket.keys.get(key_name) + if version_id is None: + return bucket.keys.get(key_name) + else: + for key in bucket.keys.getlist(key_name): + if str(key._version_id) == str(version_id): + return key def initiate_multipart(self, bucket_name, key_name): bucket = self.buckets[bucket_name] diff --git a/moto/s3/responses.py b/moto/s3/responses.py index f180a97a..9e657430 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -48,7 +48,7 @@ class ResponseObject(object): elif method == 'GET': return self._bucket_response_get(bucket_name, querystring, headers) elif method == 'PUT': - return self._bucket_response_put(bucket_name, headers) + return self._bucket_response_put(request, bucket_name, querystring, headers) elif method == 'DELETE': return self._bucket_response_delete(bucket_name, headers) elif method == 'POST': @@ -73,6 +73,36 @@ class ResponseObject(object): return 200, headers, template.render( bucket_name=bucket_name, uploads=multiparts) + elif 'versioning' in querystring: + versioning = self.backend.get_bucket_versioning(bucket_name) + template = Template(S3_BUCKET_GET_VERSIONING) + return 200, headers, template.render(status=versioning) + elif 'versions' in querystring: + delimiter = querystring.get('delimiter', [None])[0] + encoding_type = querystring.get('encoding-type', [None])[0] + key_marker = querystring.get('key-marker', [None])[0] + max_keys = querystring.get('max-keys', [None])[0] + prefix = querystring.get('prefix', [None])[0] + version_id_marker = querystring.get('version-id-marker', [None])[0] + + bucket = self.backend.get_bucket(bucket_name) + versions = self.backend.get_bucket_versions( + bucket_name, + delimiter=delimiter, + encoding_type=encoding_type, + key_marker=key_marker, + max_keys=max_keys, + version_id_marker=version_id_marker + ) + template = Template(S3_BUCKET_GET_VERSIONS) + return 200, headers, template.render( + key_list=versions, + bucket=bucket, + prefix='', + max_keys='', + delimiter='', + is_truncated='false', + ) bucket = self.backend.get_bucket(bucket_name) if bucket: @@ -80,7 +110,7 @@ class ResponseObject(object): delimiter = querystring.get('delimiter', [None])[0] result_keys, result_folders = self.backend.prefix_query(bucket, prefix, delimiter) template = Template(S3_BUCKET_GET_RESPONSE) - return template.render( + return 200, headers, template.render( bucket=bucket, prefix=prefix, delimiter=delimiter, @@ -90,13 +120,22 @@ class ResponseObject(object): else: return 404, headers, "" - def _bucket_response_put(self, bucket_name, headers): - try: - new_bucket = self.backend.create_bucket(bucket_name) - except BucketAlreadyExists: - return 409, headers, "" - template = Template(S3_BUCKET_CREATE_RESPONSE) - return template.render(bucket=new_bucket) + def _bucket_response_put(self, request, bucket_name, querystring, headers): + if 'versioning' in querystring: + ver = re.search('([A-Za-z]+)', request.body) + if ver: + self.backend.set_bucket_versioning(bucket_name, ver.group(1)) + template = Template(S3_BUCKET_VERSIONING) + return template.render(bucket_versioning_status=ver.group(1)) + else: + return 404, headers, "" + else: + try: + new_bucket = self.backend.create_bucket(bucket_name) + except BucketAlreadyExists: + return 409, headers, "" + template = Template(S3_BUCKET_CREATE_RESPONSE) + return 200, headers, template.render(bucket=new_bucket) def _bucket_response_delete(self, bucket_name, headers): removed_bucket = self.backend.delete_bucket(bucket_name) @@ -224,7 +263,9 @@ class ResponseObject(object): count=len(parts), parts=parts ) - key = self.backend.get_key(bucket_name, key_name) + version_id = query.get('versionId', [None])[0] + key = self.backend.get_key( + bucket_name, key_name, version_id=version_id) if key: headers.update(key.metadata) return 200, headers, key.value @@ -411,6 +452,49 @@ S3_DELETE_BUCKET_WITH_ITEMS_ERROR = """ sdfgdsfgdsfgdfsdsfgdfs """ +S3_BUCKET_VERSIONING = """ + + + {{ bucket_versioning_status }} + +""" + +S3_BUCKET_GET_VERSIONING = """ + +{% if status is none %} + +{% else %} + + {{ status }} + +{% endif %} +""" + +S3_BUCKET_GET_VERSIONS = """ + + {{ bucket.name }} + {{ prefix }} + {{ key_marker }} + {{ max_keys }} + {{ is_truncated }} + {% for key in key_list %} + + {{ key.name }} + {{ key._version_id }} + false + {{ key.last_modified_ISO8601 }} + {{ key.etag }} + {{ key.size }} + {{ key.storage_class }} + + 75aa57f09aa0c8caeab4f8c24e99d10f8e7faeebf76c078efc7c6caea54ba06a + webfile + + + {% endfor %} + +""" + S3_DELETE_KEYS_RESPONSE = """ {% for k in deleted %} diff --git a/moto/s3/utils.py b/moto/s3/utils.py index 19b0cfdf..0fecfb2c 100644 --- a/moto/s3/utils.py +++ b/moto/s3/utils.py @@ -1,4 +1,5 @@ import re +import sys import urllib2 import urlparse @@ -25,3 +26,73 @@ def bucket_name_from_url(url): def clean_key_name(key_name): return urllib2.unquote(key_name) + + +class _VersionedKeyStore(dict): + + """ A simplified/modified version of Django's `MultiValueDict` taken from: + https://github.com/django/django/blob/70576740b0bb5289873f5a9a9a4e1a26b2c330e5/django/utils/datastructures.py#L282 + """ + + def __sgetitem__(self, key): + return super(_VersionedKeyStore, self).__getitem__(key) + + def __getitem__(self, key): + return self.__sgetitem__(key)[-1] + + def __setitem__(self, key, value): + try: + current = self.__sgetitem__(key) + current.append(value) + except (KeyError, IndexError): + current = [value] + + super(_VersionedKeyStore, self).__setitem__(key, current) + + def get(self, key, default=None): + try: + return self[key] + except (KeyError, IndexError): + pass + return default + + def getlist(self, key, default=None): + try: + return self.__sgetitem__(key) + except (KeyError, IndexError): + pass + return default + + def setlist(self, key, list_): + if isinstance(list_, tuple): + list_ = list(list_) + elif not isinstance(list_, list): + list_ = [list_] + + super(_VersionedKeyStore, self).__setitem__(key, list_) + + def _iteritems(self): + for key in self: + yield key, self[key] + + def _itervalues(self): + for key in self: + yield self[key] + + def _iterlists(self): + for key in self: + yield key, self.getlist(key) + + items = iteritems = _iteritems + lists = iterlists = _iterlists + values = itervalues = _itervalues + + if sys.version_info[0] < 3: + def items(self): + return list(self.iteritems()) + + def values(self): + return list(self.itervalues()) + + def lists(self): + return list(self.iterlists()) diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index 72066bcf..8102f33c 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -508,3 +508,64 @@ def test_restore_key_headers(): key.ongoing_restore.should_not.be.none key.ongoing_restore.should.be.false key.expiry_date.should.equal("Mon, 02 Jan 2012 12:00:00 GMT") + + +@mock_s3 +def test_get_versioning_status(): + conn = boto.connect_s3('the_key', 'the_secret') + bucket = conn.create_bucket('foobar') + d = bucket.get_versioning_status() + d.should.be.empty + + bucket.configure_versioning(versioning=True) + d = bucket.get_versioning_status() + d.shouldnt.be.empty + d.should.have.key('Versioning').being.equal('Enabled') + + bucket.configure_versioning(versioning=False) + d = bucket.get_versioning_status() + d.should.have.key('Versioning').being.equal('Suspended') + + +@mock_s3 +def test_key_version(): + conn = boto.connect_s3('the_key', 'the_secret') + bucket = conn.create_bucket('foobar') + bucket.configure_versioning(versioning=True) + + key = Key(bucket) + key.key = 'the-key' + key.version_id.should.be.none + key.set_contents_from_string('some string') + key.version_id.should.equal('0') + key.set_contents_from_string('some string') + key.version_id.should.equal('1') + + key = bucket.get_key('the-key') + key.version_id.should.equal('1') + + +@mock_s3 +def test_list_versions(): + conn = boto.connect_s3('the_key', 'the_secret') + bucket = conn.create_bucket('foobar') + bucket.configure_versioning(versioning=True) + + key = Key(bucket, 'the-key') + key.version_id.should.be.none + key.set_contents_from_string("Version 1") + key.version_id.should.equal('0') + key.set_contents_from_string("Version 2") + key.version_id.should.equal('1') + + versions = list(bucket.list_versions()) + + versions.should.have.length_of(2) + + versions[0].name.should.equal('the-key') + versions[0].version_id.should.equal('0') + versions[0].get_contents_as_string().should.equal("Version 1") + + versions[1].name.should.equal('the-key') + versions[1].version_id.should.equal('1') + versions[1].get_contents_as_string().should.equal("Version 2") diff --git a/tests/test_s3/test_s3_utils.py b/tests/test_s3/test_s3_utils.py index cb8bd8b8..d2eee840 100644 --- a/tests/test_s3/test_s3_utils.py +++ b/tests/test_s3/test_s3_utils.py @@ -1,5 +1,5 @@ from sure import expect -from moto.s3.utils import bucket_name_from_url +from moto.s3.utils import bucket_name_from_url, _VersionedKeyStore def test_base_url(): @@ -12,3 +12,40 @@ def test_localhost_bucket(): def test_localhost_without_bucket(): expect(bucket_name_from_url('https://www.localhost:5000/def')).should.equal(None) + +def test_versioned_key_store(): + d = _VersionedKeyStore() + + d.should.have.length_of(0) + + d['key'] = [1] + + d.should.have.length_of(1) + + d['key'] = 2 + d.should.have.length_of(1) + + d.should.have.key('key').being.equal(2) + + d.get.when.called_with('key').should.return_value(2) + d.get.when.called_with('badkey').should.return_value(None) + d.get.when.called_with('badkey', 'HELLO').should.return_value('HELLO') + + # Tests key[ + d.shouldnt.have.key('badkey') + d.__getitem__.when.called_with('badkey').should.throw(KeyError) + + d.getlist('key').should.have.length_of(2) + d.getlist('key').should.be.equal([[1], 2]) + d.getlist('badkey').should.be.none + + d.setlist('key', 1) + d.getlist('key').should.be.equal([1]) + + d.setlist('key', (1, 2)) + d.getlist('key').shouldnt.be.equal((1, 2)) + d.getlist('key').should.be.equal([1, 2]) + + d.setlist('key', [[1], [2]]) + d['key'].should.have.length_of(1) + d.getlist('key').should.be.equal([[1], [2]])