Merge branch 'master' into master

moto/__init__.py

@@ -16,6 +16,7 @@ from .cognitoidp import mock_cognitoidp, mock_cognitoidp_deprecated  # flake8: n
 from .datapipeline import mock_datapipeline, mock_datapipeline_deprecated  # flake8: noqa
 from .dynamodb import mock_dynamodb, mock_dynamodb_deprecated  # flake8: noqa
 from .dynamodb2 import mock_dynamodb2, mock_dynamodb2_deprecated  # flake8: noqa
+from .dynamodbstreams import mock_dynamodbstreams # flake8: noqa
 from .ec2 import mock_ec2, mock_ec2_deprecated  # flake8: noqa
 from .ecr import mock_ecr, mock_ecr_deprecated  # flake8: noqa
 from .ecs import mock_ecs, mock_ecs_deprecated  # flake8: noqa

moto/autoscaling/responses.py

@@ -508,6 +508,15 @@ DESCRIBE_AUTOSCALING_GROUPS_TEMPLATE = """<DescribeAutoScalingGroupsResponse xml
         {% else %}
           <LoadBalancerNames/>
         {% endif %}
+        {% if group.target_group_arns %}
+          <TargetGroupARNs>
+          {% for target_group_arn in group.target_group_arns %}
+            <member>{{ target_group_arn }}</member>
+          {% endfor %}
+          </TargetGroupARNs>
+        {% else %}
+          <TargetGroupARNs/>
+        {% endif %}
         <MinSize>{{ group.min_size }}</MinSize>
         {% if group.vpc_zone_identifier %}
           <VPCZoneIdentifier>{{ group.vpc_zone_identifier }}</VPCZoneIdentifier>

moto/backends.py

@@ -12,6 +12,7 @@ from moto.core import moto_api_backends
 from moto.datapipeline import datapipeline_backends
 from moto.dynamodb import dynamodb_backends
 from moto.dynamodb2 import dynamodb_backends2
+from moto.dynamodbstreams import dynamodbstreams_backends
 from moto.ec2 import ec2_backends
 from moto.ecr import ecr_backends
 from moto.ecs import ecs_backends
@@ -59,6 +60,7 @@ BACKENDS = {
     'datapipeline': datapipeline_backends,
     'dynamodb': dynamodb_backends,
     'dynamodb2': dynamodb_backends2,
+    'dynamodbstreams': dynamodbstreams_backends,
     'ec2': ec2_backends,
     'ecr': ecr_backends,
     'ecs': ecs_backends,

moto/batch/responses.py

@@ -27,7 +27,7 @@ class BatchResponse(BaseResponse):
         elif not hasattr(self, '_json'):
             try:
                 self._json = json.loads(self.body)
-            except json.JSONDecodeError:
+            except ValueError:
                 print()
         return self._json
 

moto/cloudformation/models.py

@@ -13,6 +13,7 @@ from .utils import (
     generate_changeset_id,
     generate_stack_id,
     yaml_tag_constructor,
+    validate_template_cfn_lint,
 )
 from .exceptions import ValidationError
 
@@ -270,6 +271,9 @@ class CloudFormationBackend(BaseBackend):
             next_token = str(token + 100) if len(all_exports) > token + 100 else None
         return exports, next_token
 
+    def validate_template(self, template):
+        return validate_template_cfn_lint(template)
+
     def _validate_export_uniqueness(self, stack):
         new_stack_export_names = [x.name for x in stack.exports]
         export_names = self.exports.keys()

moto/cloudformation/responses.py

@@ -1,6 +1,7 @@
 from __future__ import unicode_literals
 
 import json
+import yaml
 from six.moves.urllib.parse import urlparse
 
 from moto.core.responses import BaseResponse
@@ -87,7 +88,8 @@ class CloudFormationResponse(BaseResponse):
         role_arn = self._get_param('RoleARN')
         update_or_create = self._get_param('ChangeSetType', 'CREATE')
         parameters_list = self._get_list_prefix("Parameters.member")
-        tags = {tag[0]: tag[1] for tag in self._get_list_prefix("Tags.member")}
+        tags = dict((item['key'], item['value'])
+                    for item in self._get_list_prefix("Tags.member"))
         parameters = {param['parameter_key']: param['parameter_value']
                       for param in parameters_list}
         if template_url:
@@ -294,6 +296,32 @@ class CloudFormationResponse(BaseResponse):
         template = self.response_template(LIST_EXPORTS_RESPONSE)
         return template.render(exports=exports, next_token=next_token)
 
+    def validate_template(self):
+        cfn_lint = self.cloudformation_backend.validate_template(self._get_param('TemplateBody'))
+        if cfn_lint:
+            raise ValidationError(cfn_lint[0].message)
+        description = ""
+        try:
+            description = json.loads(self._get_param('TemplateBody'))['Description']
+        except (ValueError, KeyError):
+            pass
+        try:
+            description = yaml.load(self._get_param('TemplateBody'))['Description']
+        except (yaml.ParserError, KeyError):
+            pass
+        template = self.response_template(VALIDATE_STACK_RESPONSE_TEMPLATE)
+        return template.render(description=description)
+
+
+VALIDATE_STACK_RESPONSE_TEMPLATE = """<ValidateTemplateResponse>
+        <ValidateTemplateResult>
+        <Capabilities></Capabilities>
+<CapabilitiesReason></CapabilitiesReason>
+<DeclaredTransforms></DeclaredTransforms>
+<Description>{{ description }}</Description>
+<Parameters></Parameters>
+</ValidateTemplateResult>
+</ValidateTemplateResponse>"""
 
 CREATE_STACK_RESPONSE_TEMPLATE = """<CreateStackResponse>
   <CreateStackResult>

moto/cloudformation/utils.py

@@ -3,6 +3,9 @@ import uuid
 import six
 import random
 import yaml
+import os
+
+from cfnlint import decode, core
 
 
 def generate_stack_id(stack_name):
@@ -38,3 +41,33 @@ def yaml_tag_constructor(loader, tag, node):
         key = 'Fn::{}'.format(tag[1:])
 
     return {key: _f(loader, tag, node)}
+
+
+def validate_template_cfn_lint(template):
+
+    # Save the template to a temporary file -- cfn-lint requires a file
+    filename = "file.tmp"
+    with open(filename, "w") as file:
+        file.write(template)
+    abs_filename = os.path.abspath(filename)
+
+    # decode handles both yaml and json
+    template, matches = decode.decode(abs_filename, False)
+
+    # Set cfn-lint to info
+    core.configure_logging(None)
+
+    # Initialize the ruleset to be applied (no overrules, no excludes)
+    rules = core.get_rules([], [], [])
+
+    # Use us-east-1 region (spec file) for validation
+    regions = ['us-east-1']
+
+    # Process all the rules and gather the errors
+    matches = core.run_checks(
+        abs_filename,
+        template,
+        rules,
+        regions)
+
+    return matches

moto/cognitoidp/exceptions.py

@@ -24,6 +24,16 @@ class UserNotFoundError(BadRequest):
         })
 
 
+class GroupExistsException(BadRequest):
+
+    def __init__(self, message):
+        super(GroupExistsException, self).__init__()
+        self.description = json.dumps({
+            "message": message,
+            '__type': 'GroupExistsException',
+        })
+
+
 class NotAuthorizedError(BadRequest):
 
     def __init__(self, message):

moto/cognitoidp/models.py

@@ -1,6 +1,8 @@
 from __future__ import unicode_literals
 
 import datetime
+import functools
+import itertools
 import json
 import os
 import time
@@ -11,8 +13,7 @@ from jose import jws
 
 from moto.compat import OrderedDict
 from moto.core import BaseBackend, BaseModel
-from .exceptions import NotAuthorizedError, ResourceNotFoundError, UserNotFoundError
-
+from .exceptions import GroupExistsException, NotAuthorizedError, ResourceNotFoundError, UserNotFoundError
 
 UserStatus = {
     "FORCE_CHANGE_PASSWORD": "FORCE_CHANGE_PASSWORD",
@@ -20,6 +21,39 @@ UserStatus = {
 }
 
 
+def paginate(limit, start_arg="next_token", limit_arg="max_results"):
+    """Returns a limited result list, and an offset into list of remaining items
+
+    Takes the next_token, and max_results kwargs given to a function and handles
+    the slicing of the results. The kwarg `next_token` is the offset into the
+    list to begin slicing from. `max_results` is the size of the result required
+
+    If the max_results is not supplied then the `limit` parameter is used as a
+    default
+
+    :param limit_arg: the name of argument in the decorated function that
+    controls amount of items returned
+    :param start_arg: the name of the argument in the decorated that provides
+    the starting offset
+    :param limit: A default maximum items to return
+    :return: a tuple containing a list of items, and the offset into the list
+    """
+    default_start = 0
+
+    def outer_wrapper(func):
+        @functools.wraps(func)
+        def wrapper(*args, **kwargs):
+            start = int(default_start if kwargs.get(start_arg) is None else kwargs[start_arg])
+            lim = int(limit if kwargs.get(limit_arg) is None else kwargs[limit_arg])
+            stop = start + lim
+            result = func(*args, **kwargs)
+            limited_results = list(itertools.islice(result, start, stop))
+            next_token = stop if stop < len(result) else None
+            return limited_results, next_token
+        return wrapper
+    return outer_wrapper
+
+
 class CognitoIdpUserPool(BaseModel):
 
     def __init__(self, region, name, extended_config):
@@ -33,6 +67,7 @@ class CognitoIdpUserPool(BaseModel):
 
         self.clients = OrderedDict()
         self.identity_providers = OrderedDict()
+        self.groups = OrderedDict()
         self.users = OrderedDict()
         self.refresh_tokens = {}
         self.access_tokens = {}
@@ -185,6 +220,33 @@ class CognitoIdpIdentityProvider(BaseModel):
         return identity_provider_json
 
 
+class CognitoIdpGroup(BaseModel):
+
+    def __init__(self, user_pool_id, group_name, description, role_arn, precedence):
+        self.user_pool_id = user_pool_id
+        self.group_name = group_name
+        self.description = description or ""
+        self.role_arn = role_arn
+        self.precedence = precedence
+        self.last_modified_date = datetime.datetime.now()
+        self.creation_date = self.last_modified_date
+
+        # Users who are members of this group.
+        # Note that these links are bidirectional.
+        self.users = set()
+
+    def to_json(self):
+        return {
+            "GroupName": self.group_name,
+            "UserPoolId": self.user_pool_id,
+            "Description": self.description,
+            "RoleArn": self.role_arn,
+            "Precedence": self.precedence,
+            "LastModifiedDate": time.mktime(self.last_modified_date.timetuple()),
+            "CreationDate": time.mktime(self.creation_date.timetuple()),
+        }
+
+
 class CognitoIdpUser(BaseModel):
 
     def __init__(self, user_pool_id, username, password, status, attributes):
@@ -198,6 +260,10 @@ class CognitoIdpUser(BaseModel):
         self.create_date = datetime.datetime.utcnow()
         self.last_modified_date = datetime.datetime.utcnow()
 
+        # Groups this user is a member of.
+        # Note that these links are bidirectional.
+        self.groups = set()
+
     def _base_json(self):
         return {
             "UserPoolId": self.user_pool_id,
@@ -242,7 +308,8 @@ class CognitoIdpBackend(BaseBackend):
         self.user_pools[user_pool.id] = user_pool
         return user_pool
 
-    def list_user_pools(self):
+    @paginate(60)
+    def list_user_pools(self, max_results=None, next_token=None):
         return self.user_pools.values()
 
     def describe_user_pool(self, user_pool_id):
@@ -289,7 +356,8 @@ class CognitoIdpBackend(BaseBackend):
         user_pool.clients[user_pool_client.id] = user_pool_client
         return user_pool_client
 
-    def list_user_pool_clients(self, user_pool_id):
+    @paginate(60)
+    def list_user_pool_clients(self, user_pool_id, max_results=None, next_token=None):
         user_pool = self.user_pools.get(user_pool_id)
         if not user_pool:
             raise ResourceNotFoundError(user_pool_id)
@@ -339,7 +407,8 @@ class CognitoIdpBackend(BaseBackend):
         user_pool.identity_providers[name] = identity_provider
         return identity_provider
 
-    def list_identity_providers(self, user_pool_id):
+    @paginate(60)
+    def list_identity_providers(self, user_pool_id, max_results=None, next_token=None):
         user_pool = self.user_pools.get(user_pool_id)
         if not user_pool:
             raise ResourceNotFoundError(user_pool_id)
@@ -367,6 +436,72 @@ class CognitoIdpBackend(BaseBackend):
 
         del user_pool.identity_providers[name]
 
+    # Group
+    def create_group(self, user_pool_id, group_name, description, role_arn, precedence):
+        user_pool = self.user_pools.get(user_pool_id)
+        if not user_pool:
+            raise ResourceNotFoundError(user_pool_id)
+
+        group = CognitoIdpGroup(user_pool_id, group_name, description, role_arn, precedence)
+        if group.group_name in user_pool.groups:
+            raise GroupExistsException("A group with the name already exists")
+        user_pool.groups[group.group_name] = group
+
+        return group
+
+    def get_group(self, user_pool_id, group_name):
+        user_pool = self.user_pools.get(user_pool_id)
+        if not user_pool:
+            raise ResourceNotFoundError(user_pool_id)
+
+        if group_name not in user_pool.groups:
+            raise ResourceNotFoundError(group_name)
+
+        return user_pool.groups[group_name]
+
+    def list_groups(self, user_pool_id):
+        user_pool = self.user_pools.get(user_pool_id)
+        if not user_pool:
+            raise ResourceNotFoundError(user_pool_id)
+
+        return user_pool.groups.values()
+
+    def delete_group(self, user_pool_id, group_name):
+        user_pool = self.user_pools.get(user_pool_id)
+        if not user_pool:
+            raise ResourceNotFoundError(user_pool_id)
+
+        if group_name not in user_pool.groups:
+            raise ResourceNotFoundError(group_name)
+
+        group = user_pool.groups[group_name]
+        for user in group.users:
+            user.groups.remove(group)
+
+        del user_pool.groups[group_name]
+
+    def admin_add_user_to_group(self, user_pool_id, group_name, username):
+        group = self.get_group(user_pool_id, group_name)
+        user = self.admin_get_user(user_pool_id, username)
+
+        group.users.add(user)
+        user.groups.add(group)
+
+    def list_users_in_group(self, user_pool_id, group_name):
+        group = self.get_group(user_pool_id, group_name)
+        return list(group.users)
+
+    def admin_list_groups_for_user(self, user_pool_id, username):
+        user = self.admin_get_user(user_pool_id, username)
+        return list(user.groups)
+
+    def admin_remove_user_from_group(self, user_pool_id, group_name, username):
+        group = self.get_group(user_pool_id, group_name)
+        user = self.admin_get_user(user_pool_id, username)
+
+        group.users.discard(user)
+        user.groups.discard(group)
+
     # User
     def admin_create_user(self, user_pool_id, username, temporary_password, attributes):
         user_pool = self.user_pools.get(user_pool_id)
@@ -387,7 +522,8 @@ class CognitoIdpBackend(BaseBackend):
 
         return user_pool.users[username]
 
-    def list_users(self, user_pool_id):
+    @paginate(60, "pagination_token", "limit")
+    def list_users(self, user_pool_id, pagination_token=None, limit=None):
         user_pool = self.user_pools.get(user_pool_id)
         if not user_pool:
             raise ResourceNotFoundError(user_pool_id)
@@ -410,6 +546,10 @@ class CognitoIdpBackend(BaseBackend):
         if username not in user_pool.users:
             raise UserNotFoundError(username)
 
+        user = user_pool.users[username]
+        for group in user.groups:
+            group.users.remove(user)
+
         del user_pool.users[username]
 
     def _log_user_in(self, user_pool, client, username):

moto/cognitoidp/responses.py

@@ -22,10 +22,17 @@ class CognitoIdpResponse(BaseResponse):
         })
 
     def list_user_pools(self):
-        user_pools = cognitoidp_backends[self.region].list_user_pools()
-        return json.dumps({
-            "UserPools": [user_pool.to_json() for user_pool in user_pools]
-        })
+        max_results = self._get_param("MaxResults")
+        next_token = self._get_param("NextToken", "0")
+        user_pools, next_token = cognitoidp_backends[self.region].list_user_pools(
+            max_results=max_results, next_token=next_token
+        )
+        response = {
+            "UserPools": [user_pool.to_json() for user_pool in user_pools],
+        }
+        if next_token:
+            response["NextToken"] = str(next_token)
+        return json.dumps(response)
 
     def describe_user_pool(self):
         user_pool_id = self._get_param("UserPoolId")
@@ -72,10 +79,16 @@ class CognitoIdpResponse(BaseResponse):
 
     def list_user_pool_clients(self):
         user_pool_id = self._get_param("UserPoolId")
-        user_pool_clients = cognitoidp_backends[self.region].list_user_pool_clients(user_pool_id)
-        return json.dumps({
+        max_results = self._get_param("MaxResults")
+        next_token = self._get_param("NextToken", "0")
+        user_pool_clients, next_token = cognitoidp_backends[self.region].list_user_pool_clients(user_pool_id,
+            max_results=max_results, next_token=next_token)
+        response = {
             "UserPoolClients": [user_pool_client.to_json() for user_pool_client in user_pool_clients]
-        })
+        }
+        if next_token:
+            response["NextToken"] = str(next_token)
+        return json.dumps(response)
 
     def describe_user_pool_client(self):
         user_pool_id = self._get_param("UserPoolId")
@@ -110,10 +123,17 @@ class CognitoIdpResponse(BaseResponse):
 
     def list_identity_providers(self):
         user_pool_id = self._get_param("UserPoolId")
-        identity_providers = cognitoidp_backends[self.region].list_identity_providers(user_pool_id)
-        return json.dumps({
+        max_results = self._get_param("MaxResults")
+        next_token = self._get_param("NextToken", "0")
+        identity_providers, next_token = cognitoidp_backends[self.region].list_identity_providers(
+            user_pool_id, max_results=max_results, next_token=next_token
+        )
+        response = {
             "Providers": [identity_provider.to_json() for identity_provider in identity_providers]
-        })
+        }
+        if next_token:
+            response["NextToken"] = str(next_token)
+        return json.dumps(response)
 
     def describe_identity_provider(self):
         user_pool_id = self._get_param("UserPoolId")
@@ -129,6 +149,89 @@ class CognitoIdpResponse(BaseResponse):
         cognitoidp_backends[self.region].delete_identity_provider(user_pool_id, name)
         return ""
 
+    # Group
+    def create_group(self):
+        group_name = self._get_param("GroupName")
+        user_pool_id = self._get_param("UserPoolId")
+        description = self._get_param("Description")
+        role_arn = self._get_param("RoleArn")
+        precedence = self._get_param("Precedence")
+
+        group = cognitoidp_backends[self.region].create_group(
+            user_pool_id,
+            group_name,
+            description,
+            role_arn,
+            precedence,
+        )
+
+        return json.dumps({
+            "Group": group.to_json(),
+        })
+
+    def get_group(self):
+        group_name = self._get_param("GroupName")
+        user_pool_id = self._get_param("UserPoolId")
+        group = cognitoidp_backends[self.region].get_group(user_pool_id, group_name)
+        return json.dumps({
+            "Group": group.to_json(),
+        })
+
+    def list_groups(self):
+        user_pool_id = self._get_param("UserPoolId")
+        groups = cognitoidp_backends[self.region].list_groups(user_pool_id)
+        return json.dumps({
+            "Groups": [group.to_json() for group in groups],
+        })
+
+    def delete_group(self):
+        group_name = self._get_param("GroupName")
+        user_pool_id = self._get_param("UserPoolId")
+        cognitoidp_backends[self.region].delete_group(user_pool_id, group_name)
+        return ""
+
+    def admin_add_user_to_group(self):
+        user_pool_id = self._get_param("UserPoolId")
+        username = self._get_param("Username")
+        group_name = self._get_param("GroupName")
+
+        cognitoidp_backends[self.region].admin_add_user_to_group(
+            user_pool_id,
+            group_name,
+            username,
+        )
+
+        return ""
+
+    def list_users_in_group(self):
+        user_pool_id = self._get_param("UserPoolId")
+        group_name = self._get_param("GroupName")
+        users = cognitoidp_backends[self.region].list_users_in_group(user_pool_id, group_name)
+        return json.dumps({
+            "Users": [user.to_json(extended=True) for user in users],
+        })
+
+    def admin_list_groups_for_user(self):
+        username = self._get_param("Username")
+        user_pool_id = self._get_param("UserPoolId")
+        groups = cognitoidp_backends[self.region].admin_list_groups_for_user(user_pool_id, username)
+        return json.dumps({
+            "Groups": [group.to_json() for group in groups],
+        })
+
+    def admin_remove_user_from_group(self):
+        user_pool_id = self._get_param("UserPoolId")
+        username = self._get_param("Username")
+        group_name = self._get_param("GroupName")
+
+        cognitoidp_backends[self.region].admin_remove_user_from_group(
+            user_pool_id,
+            group_name,
+            username,
+        )
+
+        return ""
+
     # User
     def admin_create_user(self):
         user_pool_id = self._get_param("UserPoolId")
@@ -155,10 +258,15 @@ class CognitoIdpResponse(BaseResponse):
 
     def list_users(self):
         user_pool_id = self._get_param("UserPoolId")
-        users = cognitoidp_backends[self.region].list_users(user_pool_id)
-        return json.dumps({
-            "Users": [user.to_json(extended=True) for user in users]
-        })
+        limit = self._get_param("Limit")
+        token = self._get_param("PaginationToken")
+        users, token = cognitoidp_backends[self.region].list_users(user_pool_id,
+                                                                   limit=limit,
+                                                                   pagination_token=token)
+        response = {"Users": [user.to_json(extended=True) for user in users]}
+        if token:
+            response["PaginationToken"] = str(token)
+        return json.dumps(response)
 
     def admin_disable_user(self):
         user_pool_id = self._get_param("UserPoolId")

moto/core/models.py

@@ -4,6 +4,7 @@ from __future__ import absolute_import
 
 import functools
 import inspect
+import os
 import re
 import six
 from io import BytesIO
@@ -21,6 +22,11 @@ from .utils import (
 )
 
 
+# "Mock" the AWS credentials as they can't be mocked in Botocore currently
+os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key")
+os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret")
+
+
 class BaseMockAWS(object):
     nested_count = 0
 

moto/dynamodb2/models.py

@@ -5,6 +5,7 @@ import datetime
 import decimal
 import json
 import re
+import uuid
 
 import boto3
 from moto.compat import OrderedDict
@@ -292,9 +293,82 @@ class Item(BaseModel):
                         'ADD not supported for %s' % ', '.join(update_action['Value'].keys()))
 
 
+class StreamRecord(BaseModel):
+    def __init__(self, table, stream_type, event_name, old, new, seq):
+        old_a = old.to_json()['Attributes'] if old is not None else {}
+        new_a = new.to_json()['Attributes'] if new is not None else {}
+
+        rec = old if old is not None else new
+        keys = {table.hash_key_attr: rec.hash_key.to_json()}
+        if table.range_key_attr is not None:
+            keys[table.range_key_attr] = rec.range_key.to_json()
+
+        self.record = {
+            'eventID': uuid.uuid4().hex,
+            'eventName': event_name,
+            'eventSource': 'aws:dynamodb',
+            'eventVersion': '1.0',
+            'awsRegion': 'us-east-1',
+            'dynamodb': {
+                'StreamViewType': stream_type,
+                'ApproximateCreationDateTime': datetime.datetime.utcnow().isoformat(),
+                'SequenceNumber': seq,
+                'SizeBytes': 1,
+                'Keys': keys
+            }
+        }
+
+        if stream_type in ('NEW_IMAGE', 'NEW_AND_OLD_IMAGES'):
+            self.record['dynamodb']['NewImage'] = new_a
+        if stream_type in ('OLD_IMAGE', 'NEW_AND_OLD_IMAGES'):
+            self.record['dynamodb']['OldImage'] = old_a
+
+        # This is a substantial overestimate but it's the easiest to do now
+        self.record['dynamodb']['SizeBytes'] = len(
+            json.dumps(self.record['dynamodb']))
+
+    def to_json(self):
+        return self.record
+
+
+class StreamShard(BaseModel):
+    def __init__(self, table):
+        self.table = table
+        self.id = 'shardId-00000001541626099285-f35f62ef'
+        self.starting_sequence_number = 1100000000017454423009
+        self.items = []
+        self.created_on = datetime.datetime.utcnow()
+
+    def to_json(self):
+        return {
+            'ShardId': self.id,
+            'SequenceNumberRange': {
+                'StartingSequenceNumber': str(self.starting_sequence_number)
+            }
+        }
+
+    def add(self, old, new):
+        t = self.table.stream_specification['StreamViewType']
+        if old is None:
+            event_name = 'INSERT'
+        elif new is None:
+            event_name = 'DELETE'
+        else:
+            event_name = 'MODIFY'
+        seq = len(self.items) + self.starting_sequence_number
+        self.items.append(
+            StreamRecord(self.table, t, event_name, old, new, seq))
+
+    def get(self, start, quantity):
+        start -= self.starting_sequence_number
+        assert start >= 0
+        end = start + quantity
+        return [i.to_json() for i in self.items[start:end]]
+
+
 class Table(BaseModel):
 
-    def __init__(self, table_name, schema=None, attr=None, throughput=None, indexes=None, global_indexes=None):
+    def __init__(self, table_name, schema=None, attr=None, throughput=None, indexes=None, global_indexes=None, streams=None):
         self.name = table_name
         self.attr = attr
         self.schema = schema
@@ -325,10 +399,22 @@ class Table(BaseModel):
             'TimeToLiveStatus': 'DISABLED'  # One of 'ENABLING'|'DISABLING'|'ENABLED'|'DISABLED',
             # 'AttributeName': 'string'  # Can contain this
         }
+        self.set_stream_specification(streams)
 
     def _generate_arn(self, name):
         return 'arn:aws:dynamodb:us-east-1:123456789011:table/' + name
 
+    def set_stream_specification(self, streams):
+        self.stream_specification = streams
+        if streams and (streams.get('StreamEnabled') or streams.get('StreamViewType')):
+            self.stream_specification['StreamEnabled'] = True
+            self.latest_stream_label = datetime.datetime.utcnow().isoformat()
+            self.stream_shard = StreamShard(self)
+        else:
+            self.stream_specification = {'StreamEnabled': False}
+            self.latest_stream_label = None
+            self.stream_shard = None
+
     def describe(self, base_key='TableDescription'):
         results = {
             base_key: {
@@ -345,6 +431,11 @@ class Table(BaseModel):
                 'LocalSecondaryIndexes': [index for index in self.indexes],
             }
         }
+        if self.stream_specification and self.stream_specification['StreamEnabled']:
+            results[base_key]['StreamSpecification'] = self.stream_specification
+            if self.latest_stream_label:
+                results[base_key]['LatestStreamLabel'] = self.latest_stream_label
+                results[base_key]['LatestStreamArn'] = self.table_arn + '/stream/' + self.latest_stream_label
         return results
 
     def __len__(self):
@@ -385,23 +476,22 @@ class Table(BaseModel):
         else:
             range_value = None
 
+        if expected is None:
+            expected = {}
+            lookup_range_value = range_value
+        else:
+            expected_range_value = expected.get(
+                self.range_key_attr, {}).get("Value")
+            if(expected_range_value is None):
+                lookup_range_value = range_value
+            else:
+                lookup_range_value = DynamoType(expected_range_value)
+        current = self.get_item(hash_value, lookup_range_value)
+
         item = Item(hash_value, self.hash_key_type, range_value,
                     self.range_key_type, item_attrs)
 
         if not overwrite:
-            if expected is None:
-                expected = {}
-                lookup_range_value = range_value
-            else:
-                expected_range_value = expected.get(
-                    self.range_key_attr, {}).get("Value")
-                if(expected_range_value is None):
-                    lookup_range_value = range_value
-                else:
-                    lookup_range_value = DynamoType(expected_range_value)
-
-            current = self.get_item(hash_value, lookup_range_value)
-
             if current is None:
                 current_attr = {}
             elif hasattr(current, 'attrs'):
@@ -432,6 +522,10 @@ class Table(BaseModel):
             self.items[hash_value][range_value] = item
         else:
             self.items[hash_value] = item
+
+        if self.stream_shard is not None:
+            self.stream_shard.add(current, item)
+
         return item
 
     def __nonzero__(self):
@@ -462,9 +556,14 @@ class Table(BaseModel):
     def delete_item(self, hash_key, range_key):
         try:
             if range_key:
-                return self.items[hash_key].pop(range_key)
+                item = self.items[hash_key].pop(range_key)
             else:
-                return self.items.pop(hash_key)
+                item = self.items.pop(hash_key)
+
+            if self.stream_shard is not None:
+                self.stream_shard.add(item, None)
+
+            return item
         except KeyError:
             return None
 
@@ -680,6 +779,13 @@ class DynamoDBBackend(BaseBackend):
         table.throughput = throughput
         return table
 
+    def update_table_streams(self, name, stream_specification):
+        table = self.tables[name]
+        if (stream_specification.get('StreamEnabled') or stream_specification.get('StreamViewType')) and table.latest_stream_label:
+            raise ValueError('Table already has stream enabled')
+        table.set_stream_specification(stream_specification)
+        return table
+
     def update_table_global_indexes(self, name, global_index_updates):
         table = self.tables[name]
         gsis_by_name = dict((i['IndexName'], i) for i in table.global_indexes)

moto/dynamodb2/responses.py

@@ -104,13 +104,16 @@ class DynamoHandler(BaseResponse):
         # getting the indexes
         global_indexes = body.get("GlobalSecondaryIndexes", [])
         local_secondary_indexes = body.get("LocalSecondaryIndexes", [])
+        # get the stream specification
+        streams = body.get("StreamSpecification")
 
         table = self.dynamodb_backend.create_table(table_name,
                                                    schema=key_schema,
                                                    throughput=throughput,
                                                    attr=attr,
                                                    global_indexes=global_indexes,
-                                                   indexes=local_secondary_indexes)
+                                                   indexes=local_secondary_indexes,
+                                                   streams=streams)
         if table is not None:
             return dynamo_json_dump(table.describe())
         else:
@@ -163,12 +166,20 @@ class DynamoHandler(BaseResponse):
 
     def update_table(self):
         name = self.body['TableName']
+        table = self.dynamodb_backend.get_table(name)
         if 'GlobalSecondaryIndexUpdates' in self.body:
             table = self.dynamodb_backend.update_table_global_indexes(
                 name, self.body['GlobalSecondaryIndexUpdates'])
         if 'ProvisionedThroughput' in self.body:
             throughput = self.body["ProvisionedThroughput"]
             table = self.dynamodb_backend.update_table_throughput(name, throughput)
+        if 'StreamSpecification' in self.body:
+            try:
+                table = self.dynamodb_backend.update_table_streams(name, self.body['StreamSpecification'])
+            except ValueError:
+                er = 'com.amazonaws.dynamodb.v20111205#ResourceInUseException'
+                return self.error(er, 'Cannot enable stream')
+
         return dynamo_json_dump(table.describe())
 
     def describe_table(self):
@@ -183,6 +194,11 @@ class DynamoHandler(BaseResponse):
     def put_item(self):
         name = self.body['TableName']
         item = self.body['Item']
+        return_values = self.body.get('ReturnValues', 'NONE')
+
+        if return_values not in ('ALL_OLD', 'NONE'):
+            er = 'com.amazonaws.dynamodb.v20111205#ValidationException'
+            return self.error(er, 'Return values set to invalid value')
 
         if has_empty_keys_or_values(item):
             return get_empty_str_error()
@@ -193,6 +209,13 @@ class DynamoHandler(BaseResponse):
         else:
             expected = None
 
+        if return_values == 'ALL_OLD':
+            existing_item = self.dynamodb_backend.get_item(name, item)
+            if existing_item:
+                existing_attributes = existing_item.to_json()['Attributes']
+            else:
+                existing_attributes = {}
+
         # Attempt to parse simple ConditionExpressions into an Expected
         # expression
         if not expected:
@@ -228,6 +251,10 @@ class DynamoHandler(BaseResponse):
                 'TableName': name,
                 'CapacityUnits': 1
             }
+            if return_values == 'ALL_OLD':
+                item_dict['Attributes'] = existing_attributes
+            else:
+                item_dict.pop('Attributes', None)
             return dynamo_json_dump(item_dict)
         else:
             er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException'
@@ -512,7 +539,11 @@ class DynamoHandler(BaseResponse):
     def delete_item(self):
         name = self.body['TableName']
         keys = self.body['Key']
-        return_values = self.body.get('ReturnValues', '')
+        return_values = self.body.get('ReturnValues', 'NONE')
+        if return_values not in ('ALL_OLD', 'NONE'):
+            er = 'com.amazonaws.dynamodb.v20111205#ValidationException'
+            return self.error(er, 'Return values set to invalid value')
+
         table = self.dynamodb_backend.get_table(name)
         if not table:
             er = 'com.amazonaws.dynamodb.v20120810#ConditionalCheckFailedException'
@@ -527,9 +558,9 @@ class DynamoHandler(BaseResponse):
         return dynamo_json_dump(item_dict)
 
     def update_item(self):
-
         name = self.body['TableName']
         key = self.body['Key']
+        return_values = self.body.get('ReturnValues', 'NONE')
         update_expression = self.body.get('UpdateExpression')
         attribute_updates = self.body.get('AttributeUpdates')
         expression_attribute_names = self.body.get(
@@ -537,6 +568,15 @@ class DynamoHandler(BaseResponse):
         expression_attribute_values = self.body.get(
             'ExpressionAttributeValues', {})
         existing_item = self.dynamodb_backend.get_item(name, key)
+        if existing_item:
+            existing_attributes = existing_item.to_json()['Attributes']
+        else:
+            existing_attributes = {}
+
+        if return_values not in ('NONE', 'ALL_OLD', 'ALL_NEW', 'UPDATED_OLD',
+                                 'UPDATED_NEW'):
+            er = 'com.amazonaws.dynamodb.v20111205#ValidationException'
+            return self.error(er, 'Return values set to invalid value')
 
         if has_empty_keys_or_values(expression_attribute_values):
             return get_empty_str_error()
@@ -591,8 +631,26 @@ class DynamoHandler(BaseResponse):
             'TableName': name,
             'CapacityUnits': 0.5
         }
-        if not existing_item:
+        unchanged_attributes = {
+            k for k in existing_attributes.keys()
+            if existing_attributes[k] == item_dict['Attributes'].get(k)
+        }
+        changed_attributes = set(existing_attributes.keys()).union(item_dict['Attributes'].keys()).difference(unchanged_attributes)
+
+        if return_values == 'NONE':
             item_dict['Attributes'] = {}
+        elif return_values == 'ALL_OLD':
+            item_dict['Attributes'] = existing_attributes
+        elif return_values == 'UPDATED_OLD':
+            item_dict['Attributes'] = {
+                k: v for k, v in existing_attributes.items()
+                if k in changed_attributes
+            }
+        elif return_values == 'UPDATED_NEW':
+            item_dict['Attributes'] = {
+                k: v for k, v in item_dict['Attributes'].items()
+                if k in changed_attributes
+            }
 
         return dynamo_json_dump(item_dict)
 

moto/dynamodbstreams/__init__.py

@@ -0,0 +1,6 @@
+from __future__ import unicode_literals
+from .models import dynamodbstreams_backends
+from ..core.models import base_decorator
+
+dynamodbstreams_backend = dynamodbstreams_backends['us-east-1']
+mock_dynamodbstreams = base_decorator(dynamodbstreams_backends)

moto/dynamodbstreams/models.py

@@ -0,0 +1,129 @@
+from __future__ import unicode_literals
+
+import os
+import json
+import boto3
+import base64
+
+from moto.core import BaseBackend, BaseModel
+from moto.dynamodb2.models import dynamodb_backends
+
+
+class ShardIterator(BaseModel):
+    def __init__(self, streams_backend, stream_shard, shard_iterator_type, sequence_number=None):
+        self.id = base64.b64encode(os.urandom(472)).decode('utf-8')
+        self.streams_backend = streams_backend
+        self.stream_shard = stream_shard
+        self.shard_iterator_type = shard_iterator_type
+        if shard_iterator_type == 'TRIM_HORIZON':
+            self.sequence_number = stream_shard.starting_sequence_number
+        elif shard_iterator_type == 'LATEST':
+            self.sequence_number = stream_shard.starting_sequence_number + len(stream_shard.items)
+        elif shard_iterator_type == 'AT_SEQUENCE_NUMBER':
+            self.sequence_number = sequence_number
+        elif shard_iterator_type == 'AFTER_SEQUENCE_NUMBER':
+            self.sequence_number = sequence_number + 1
+
+    @property
+    def arn(self):
+        return '{}/stream/{}|1|{}'.format(
+            self.stream_shard.table.table_arn,
+            self.stream_shard.table.latest_stream_label,
+            self.id)
+
+    def to_json(self):
+        return {
+            'ShardIterator': self.arn
+        }
+
+    def get(self, limit=1000):
+        items = self.stream_shard.get(self.sequence_number, limit)
+        try:
+            last_sequence_number = max(i['dynamodb']['SequenceNumber'] for i in items)
+            new_shard_iterator = ShardIterator(self.streams_backend,
+                                               self.stream_shard,
+                                               'AFTER_SEQUENCE_NUMBER',
+                                               last_sequence_number)
+        except ValueError:
+            new_shard_iterator = ShardIterator(self.streams_backend,
+                                               self.stream_shard,
+                                               'AT_SEQUENCE_NUMBER',
+                                               self.sequence_number)
+
+        self.streams_backend.shard_iterators[new_shard_iterator.arn] = new_shard_iterator
+        return {
+            'NextShardIterator': new_shard_iterator.arn,
+            'Records': items
+        }
+
+
+class DynamoDBStreamsBackend(BaseBackend):
+    def __init__(self, region):
+        self.region = region
+        self.shard_iterators = {}
+
+    def reset(self):
+        region = self.region
+        self.__dict__ = {}
+        self.__init__(region)
+
+    @property
+    def dynamodb(self):
+        return dynamodb_backends[self.region]
+
+    def _get_table_from_arn(self, arn):
+        table_name = arn.split(':', 6)[5].split('/')[1]
+        return self.dynamodb.get_table(table_name)
+
+    def describe_stream(self, arn):
+        table = self._get_table_from_arn(arn)
+        resp = {'StreamDescription': {
+            'StreamArn': arn,
+            'StreamLabel': table.latest_stream_label,
+            'StreamStatus': ('ENABLED' if table.latest_stream_label
+                             else 'DISABLED'),
+            'StreamViewType': table.stream_specification['StreamViewType'],
+            'CreationRequestDateTime': table.stream_shard.created_on.isoformat(),
+            'TableName': table.name,
+            'KeySchema': table.schema,
+            'Shards': ([table.stream_shard.to_json()] if table.stream_shard
+                       else [])
+        }}
+
+        return json.dumps(resp)
+
+    def list_streams(self, table_name=None):
+        streams = []
+        for table in self.dynamodb.tables.values():
+            if table_name is not None and table.name != table_name:
+                continue
+            if table.latest_stream_label:
+                d = table.describe(base_key='Table')
+                streams.append({
+                    'StreamArn': d['Table']['LatestStreamArn'],
+                    'TableName': d['Table']['TableName'],
+                    'StreamLabel': d['Table']['LatestStreamLabel']
+                })
+
+        return json.dumps({'Streams': streams})
+
+    def get_shard_iterator(self, arn, shard_id, shard_iterator_type, sequence_number=None):
+        table = self._get_table_from_arn(arn)
+        assert table.stream_shard.id == shard_id
+
+        shard_iterator = ShardIterator(self, table.stream_shard,
+                                       shard_iterator_type,
+                                       sequence_number)
+        self.shard_iterators[shard_iterator.arn] = shard_iterator
+
+        return json.dumps(shard_iterator.to_json())
+
+    def get_records(self, iterator_arn, limit):
+        shard_iterator = self.shard_iterators[iterator_arn]
+        return json.dumps(shard_iterator.get(limit))
+
+
+available_regions = boto3.session.Session().get_available_regions(
+    'dynamodbstreams')
+dynamodbstreams_backends = {region: DynamoDBStreamsBackend(region=region)
+                            for region in available_regions}

moto/dynamodbstreams/responses.py

@@ -0,0 +1,34 @@
+from __future__ import unicode_literals
+
+from moto.core.responses import BaseResponse
+
+from .models import dynamodbstreams_backends
+
+
+class DynamoDBStreamsHandler(BaseResponse):
+
+    @property
+    def backend(self):
+        return dynamodbstreams_backends[self.region]
+
+    def describe_stream(self):
+        arn = self._get_param('StreamArn')
+        return self.backend.describe_stream(arn)
+
+    def list_streams(self):
+        table_name = self._get_param('TableName')
+        return self.backend.list_streams(table_name)
+
+    def get_shard_iterator(self):
+        arn = self._get_param('StreamArn')
+        shard_id = self._get_param('ShardId')
+        shard_iterator_type = self._get_param('ShardIteratorType')
+        return self.backend.get_shard_iterator(arn, shard_id,
+                                               shard_iterator_type)
+
+    def get_records(self):
+        arn = self._get_param('ShardIterator')
+        limit = self._get_param('Limit')
+        if limit is None:
+            limit = 1000
+        return self.backend.get_records(arn, limit)

moto/dynamodbstreams/urls.py

@@ -0,0 +1,10 @@
+from __future__ import unicode_literals
+from .responses import DynamoDBStreamsHandler
+
+url_bases = [
+    "https?://streams.dynamodb.(.+).amazonaws.com"
+]
+
+url_paths = {
+    "{0}/$": DynamoDBStreamsHandler.dispatch,
+}

moto/ec2/models.py

@@ -2230,6 +2230,10 @@ class VPCPeeringConnectionStatus(object):
         self.code = code
         self.message = message
 
+    def deleted(self):
+        self.code = 'deleted'
+        self.message = 'Deleted by {deleter ID}'
+
     def initiating(self):
         self.code = 'initiating-request'
         self.message = 'Initiating Request to {accepter ID}'
@@ -2292,9 +2296,8 @@ class VPCPeeringConnectionBackend(object):
         return self.vpc_pcxs.get(vpc_pcx_id)
 
     def delete_vpc_peering_connection(self, vpc_pcx_id):
-        deleted = self.vpc_pcxs.pop(vpc_pcx_id, None)
-        if not deleted:
-            raise InvalidVPCPeeringConnectionIdError(vpc_pcx_id)
+        deleted = self.get_vpc_peering_connection(vpc_pcx_id)
+        deleted._status.deleted()
         return deleted
 
     def accept_vpc_peering_connection(self, vpc_pcx_id):

moto/ecs/models.py

@@ -769,6 +769,8 @@ class EC2ContainerServiceBackend(BaseBackend):
                             Container instances status should be one of [ACTIVE,DRAINING]")
         failures = []
         container_instance_objects = []
+        list_container_instance_ids = [x.split('/')[-1]
+                            for x in list_container_instance_ids]
         for container_instance_id in list_container_instance_ids:
             container_instance = self.container_instances[cluster_name].get(container_instance_id, None)
             if container_instance is not None:

moto/emr/responses.py

@@ -613,13 +613,11 @@ DESCRIBE_STEP_TEMPLATE = """<DescribeStepResponse xmlns="http://elasticmapreduce
       <Id>{{ step.id }}</Id>
       <Name>{{ step.name | escape }}</Name>
       <Status>
-<!-- does not exist for botocore 1.4.28
         <FailureDetails>
           <Reason/>
           <Message/>
           <LogFile/>
         </FailureDetails>
--->
         <State>{{ step.state }}</State>
         <StateChangeReason>{{ step.state_change_reason }}</StateChangeReason>
         <Timeline>

moto/iam/exceptions.py

@@ -24,3 +24,11 @@ class IAMReportNotPresentException(RESTError):
     def __init__(self, message):
         super(IAMReportNotPresentException, self).__init__(
             "ReportNotPresent", message)
+
+
+class MalformedCertificate(RESTError):
+    code = 400
+
+    def __init__(self, cert):
+        super(MalformedCertificate, self).__init__(
+            'MalformedCertificate', 'Certificate {cert} is malformed'.format(cert=cert))

moto/iam/models.py

@@ -1,14 +1,18 @@
 from __future__ import unicode_literals
 import base64
+import sys
 from datetime import datetime
 import json
 
+from cryptography import x509
+from cryptography.hazmat.backends import default_backend
+
 import pytz
 from moto.core import BaseBackend, BaseModel
 from moto.core.utils import iso_8601_datetime_without_milliseconds
 
 from .aws_managed_policies import aws_managed_policies_data
-from .exceptions import IAMNotFoundException, IAMConflictException, IAMReportNotPresentException
+from .exceptions import IAMNotFoundException, IAMConflictException, IAMReportNotPresentException, MalformedCertificate
 from .utils import random_access_key, random_alphanumeric, random_resource_id, random_policy_id
 
 ACCOUNT_ID = 123456789012
@@ -114,9 +118,10 @@ class Role(BaseModel):
         self.id = role_id
         self.name = name
         self.assume_role_policy_document = assume_role_policy_document
-        self.path = path
+        self.path = path or '/'
         self.policies = {}
         self.managed_policies = {}
+        self.create_date = datetime.now(pytz.utc)
 
     @classmethod
     def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name):
@@ -166,8 +171,9 @@ class InstanceProfile(BaseModel):
     def __init__(self, instance_profile_id, name, path, roles):
         self.id = instance_profile_id
         self.name = name
-        self.path = path
+        self.path = path or '/'
         self.roles = roles if roles else []
+        self.create_date = datetime.now(pytz.utc)
 
     @classmethod
     def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name):
@@ -213,6 +219,16 @@ class Certificate(BaseModel):
         return "arn:aws:iam::{0}:server-certificate{1}{2}".format(ACCOUNT_ID, self.path, self.cert_name)
 
 
+class SigningCertificate(BaseModel):
+
+    def __init__(self, id, user_name, body):
+        self.id = id
+        self.user_name = user_name
+        self.body = body
+        self.upload_date = datetime.strftime(datetime.utcnow(), "%Y-%m-%d-%H-%M-%S")
+        self.status = 'Active'
+
+
 class AccessKey(BaseModel):
 
     def __init__(self, user_name):
@@ -297,6 +313,7 @@ class User(BaseModel):
         self.access_keys = []
         self.password = None
         self.password_reset_required = False
+        self.signing_certificates = {}
 
     @property
     def arn(self):
@@ -765,6 +782,48 @@ class IAMBackend(BaseBackend):
 
         return users
 
+    def upload_signing_certificate(self, user_name, body):
+        user = self.get_user(user_name)
+        cert_id = random_resource_id(size=32)
+
+        # Validate the signing cert:
+        try:
+            if sys.version_info < (3, 0):
+                data = bytes(body)
+            else:
+                data = bytes(body, 'utf8')
+
+            x509.load_pem_x509_certificate(data, default_backend())
+
+        except Exception:
+            raise MalformedCertificate(body)
+
+        user.signing_certificates[cert_id] = SigningCertificate(cert_id, user_name, body)
+
+        return user.signing_certificates[cert_id]
+
+    def delete_signing_certificate(self, user_name, cert_id):
+        user = self.get_user(user_name)
+
+        try:
+            del user.signing_certificates[cert_id]
+        except KeyError:
+            raise IAMNotFoundException("The Certificate with id {id} cannot be found.".format(id=cert_id))
+
+    def list_signing_certificates(self, user_name):
+        user = self.get_user(user_name)
+
+        return list(user.signing_certificates.values())
+
+    def update_signing_certificate(self, user_name, cert_id, status):
+        user = self.get_user(user_name)
+
+        try:
+            user.signing_certificates[cert_id].status = status
+
+        except KeyError:
+            raise IAMNotFoundException("The Certificate with id {id} cannot be found.".format(id=cert_id))
+
     def create_login_profile(self, user_name, password):
         # This does not currently deal with PasswordPolicyViolation.
         user = self.get_user(user_name)

moto/iam/responses.py

@@ -201,7 +201,7 @@ class IamResponse(BaseResponse):
 
     def create_instance_profile(self):
         profile_name = self._get_param('InstanceProfileName')
-        path = self._get_param('Path')
+        path = self._get_param('Path', '/')
 
         profile = iam_backend.create_instance_profile(
             profile_name, path, role_ids=[])
@@ -552,6 +552,38 @@ class IamResponse(BaseResponse):
             roles=account_details['roles']
         )
 
+    def upload_signing_certificate(self):
+        user_name = self._get_param('UserName')
+        cert_body = self._get_param('CertificateBody')
+
+        cert = iam_backend.upload_signing_certificate(user_name, cert_body)
+        template = self.response_template(UPLOAD_SIGNING_CERTIFICATE_TEMPLATE)
+        return template.render(cert=cert)
+
+    def update_signing_certificate(self):
+        user_name = self._get_param('UserName')
+        cert_id = self._get_param('CertificateId')
+        status = self._get_param('Status')
+
+        iam_backend.update_signing_certificate(user_name, cert_id, status)
+        template = self.response_template(UPDATE_SIGNING_CERTIFICATE_TEMPLATE)
+        return template.render()
+
+    def delete_signing_certificate(self):
+        user_name = self._get_param('UserName')
+        cert_id = self._get_param('CertificateId')
+
+        iam_backend.delete_signing_certificate(user_name, cert_id)
+        template = self.response_template(DELETE_SIGNING_CERTIFICATE_TEMPLATE)
+        return template.render()
+
+    def list_signing_certificates(self):
+        user_name = self._get_param('UserName')
+
+        certs = iam_backend.list_signing_certificates(user_name)
+        template = self.response_template(LIST_SIGNING_CERTIFICATES_TEMPLATE)
+        return template.render(user_name=user_name, certificates=certs)
+
 
 ATTACH_ROLE_POLICY_TEMPLATE = """<AttachRolePolicyResponse>
   <ResponseMetadata>
@@ -734,7 +766,7 @@ CREATE_INSTANCE_PROFILE_TEMPLATE = """<CreateInstanceProfileResponse xmlns="http
       <InstanceProfileName>{{ profile.name }}</InstanceProfileName>
       <Path>{{ profile.path }}</Path>
       <Arn>{{ profile.arn }}</Arn>
-      <CreateDate>2012-05-09T16:11:10.222Z</CreateDate>
+      <CreateDate>{{ profile.create_date }}</CreateDate>
     </InstanceProfile>
   </CreateInstanceProfileResult>
   <ResponseMetadata>
@@ -753,7 +785,7 @@ GET_INSTANCE_PROFILE_TEMPLATE = """<GetInstanceProfileResponse xmlns="https://ia
           <Arn>{{ role.arn }}</Arn>
           <RoleName>{{ role.name }}</RoleName>
           <AssumeRolePolicyDocument>{{ role.assume_role_policy_document }}</AssumeRolePolicyDocument>
-          <CreateDate>2012-05-09T15:45:35Z</CreateDate>
+          <CreateDate>{{ role.create_date }}</CreateDate>
           <RoleId>{{ role.id }}</RoleId>
         </member>
         {% endfor %}
@@ -761,7 +793,7 @@ GET_INSTANCE_PROFILE_TEMPLATE = """<GetInstanceProfileResponse xmlns="https://ia
       <InstanceProfileName>{{ profile.name }}</InstanceProfileName>
       <Path>{{ profile.path }}</Path>
       <Arn>{{ profile.arn }}</Arn>
-      <CreateDate>2012-05-09T16:11:10Z</CreateDate>
+      <CreateDate>{{ profile.create_date }}</CreateDate>
     </InstanceProfile>
   </GetInstanceProfileResult>
   <ResponseMetadata>
@@ -776,7 +808,7 @@ CREATE_ROLE_TEMPLATE = """<CreateRoleResponse xmlns="https://iam.amazonaws.com/d
       <Arn>{{ role.arn }}</Arn>
       <RoleName>{{ role.name }}</RoleName>
       <AssumeRolePolicyDocument>{{ role.assume_role_policy_document }}</AssumeRolePolicyDocument>
-      <CreateDate>2012-05-08T23:34:01.495Z</CreateDate>
+      <CreateDate>{{ role.create_date }}</CreateDate>
       <RoleId>{{ role.id }}</RoleId>
     </Role>
   </CreateRoleResult>
@@ -803,7 +835,7 @@ GET_ROLE_TEMPLATE = """<GetRoleResponse xmlns="https://iam.amazonaws.com/doc/201
       <Arn>{{ role.arn }}</Arn>
       <RoleName>{{ role.name }}</RoleName>
       <AssumeRolePolicyDocument>{{ role.assume_role_policy_document }}</AssumeRolePolicyDocument>
-      <CreateDate>2012-05-08T23:34:01Z</CreateDate>
+      <CreateDate>{{ role.create_date }}</CreateDate>
       <RoleId>{{ role.id }}</RoleId>
     </Role>
   </GetRoleResult>
@@ -834,7 +866,7 @@ LIST_ROLES_TEMPLATE = """<ListRolesResponse xmlns="https://iam.amazonaws.com/doc
         <Arn>{{ role.arn }}</Arn>
         <RoleName>{{ role.name }}</RoleName>
         <AssumeRolePolicyDocument>{{ role.assume_role_policy_document }}</AssumeRolePolicyDocument>
-        <CreateDate>2012-05-09T15:45:35Z</CreateDate>
+        <CreateDate>{{ role.create_date }}</CreateDate>
         <RoleId>{{ role.id }}</RoleId>
       </member>
       {% endfor %}
@@ -865,7 +897,7 @@ CREATE_POLICY_VERSION_TEMPLATE = """<CreatePolicyVersionResponse xmlns="https://
       <Document>{{ policy_version.document }}</Document>
       <VersionId>{{ policy_version.version_id }}</VersionId>
       <IsDefaultVersion>{{ policy_version.is_default }}</IsDefaultVersion>
-      <CreateDate>2012-05-09T15:45:35Z</CreateDate>
+      <CreateDate>{{ policy_version.create_datetime }}</CreateDate>
     </PolicyVersion>
   </CreatePolicyVersionResult>
   <ResponseMetadata>
@@ -879,7 +911,7 @@ GET_POLICY_VERSION_TEMPLATE = """<GetPolicyVersionResponse xmlns="https://iam.am
       <Document>{{ policy_version.document }}</Document>
       <VersionId>{{ policy_version.version_id }}</VersionId>
       <IsDefaultVersion>{{ policy_version.is_default }}</IsDefaultVersion>
-      <CreateDate>2012-05-09T15:45:35Z</CreateDate>
+      <CreateDate>{{ policy_version.create_datetime }}</CreateDate>
     </PolicyVersion>
   </GetPolicyVersionResult>
   <ResponseMetadata>
@@ -896,7 +928,7 @@ LIST_POLICY_VERSIONS_TEMPLATE = """<ListPolicyVersionsResponse xmlns="https://ia
         <Document>{{ policy_version.document }}</Document>
         <VersionId>{{ policy_version.version_id }}</VersionId>
         <IsDefaultVersion>{{ policy_version.is_default }}</IsDefaultVersion>
-        <CreateDate>2012-05-09T15:45:35Z</CreateDate>
+        <CreateDate>{{ policy_version.create_datetime }}</CreateDate>
       </member>
       {% endfor %}
     </Versions>
@@ -912,7 +944,7 @@ LIST_INSTANCE_PROFILES_TEMPLATE = """<ListInstanceProfilesResponse xmlns="https:
     <InstanceProfiles>
       {% for instance in instance_profiles %}
       <member>
-        <Id>{{ instance.id }}</Id>
+        <InstanceProfileId>{{ instance.id }}</InstanceProfileId>
         <Roles>
           {% for role in instance.roles %}
           <member>
@@ -920,7 +952,7 @@ LIST_INSTANCE_PROFILES_TEMPLATE = """<ListInstanceProfilesResponse xmlns="https:
             <Arn>{{ role.arn }}</Arn>
             <RoleName>{{ role.name }}</RoleName>
             <AssumeRolePolicyDocument>{{ role.assume_role_policy_document }}</AssumeRolePolicyDocument>
-            <CreateDate>2012-05-09T15:45:35Z</CreateDate>
+            <CreateDate>{{ role.create_date }}</CreateDate>
             <RoleId>{{ role.id }}</RoleId>
           </member>
           {% endfor %}
@@ -928,7 +960,7 @@ LIST_INSTANCE_PROFILES_TEMPLATE = """<ListInstanceProfilesResponse xmlns="https:
         <InstanceProfileName>{{ instance.name }}</InstanceProfileName>
         <Path>{{ instance.path }}</Path>
         <Arn>{{ instance.arn }}</Arn>
-        <CreateDate>2012-05-09T16:27:03Z</CreateDate>
+        <CreateDate>{{ instance.create_date }}</CreateDate>
       </member>
       {% endfor %}
     </InstanceProfiles>
@@ -1199,8 +1231,8 @@ LIST_USER_POLICIES_TEMPLATE = """<ListUserPoliciesResponse>
          <member>{{ policy }}</member>
         {% endfor %}
       </PolicyNames>
+      <IsTruncated>false</IsTruncated>
    </ListUserPoliciesResult>
-   <IsTruncated>false</IsTruncated>
    <ResponseMetadata>
       <RequestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestId>
    </ResponseMetadata>
@@ -1243,8 +1275,8 @@ LIST_ACCESS_KEYS_TEMPLATE = """<ListAccessKeysResponse>
 CREDENTIAL_REPORT_GENERATING = """
 <GenerateCredentialReportResponse>
     <GenerateCredentialReportResult>
-        <state>STARTED</state>
-        <description>No report exists. Starting a new report generation task</description>
+        <State>STARTED</State>
+        <Description>No report exists. Starting a new report generation task</Description>
     </GenerateCredentialReportResult>
     <ResponseMetadata>
         <RequestId>fa788a82-aa8a-11e4-a278-1786c418872b"</RequestId>
@@ -1253,7 +1285,7 @@ CREDENTIAL_REPORT_GENERATING = """
 
 CREDENTIAL_REPORT_GENERATED = """<GenerateCredentialReportResponse>
     <GenerateCredentialReportResult>
-        <state>COMPLETE</state>
+        <State>COMPLETE</State>
     </GenerateCredentialReportResult>
     <ResponseMetadata>
         <RequestId>fa788a82-aa8a-11e4-a278-1786c418872b"</RequestId>
@@ -1262,7 +1294,7 @@ CREDENTIAL_REPORT_GENERATED = """<GenerateCredentialReportResponse>
 
 CREDENTIAL_REPORT = """<GetCredentialReportResponse>
     <GetCredentialReportResult>
-        <content>{{ report }}</content>
+        <Content>{{ report }}</Content>
         <GeneratedTime>2015-02-02T20:02:02Z</GeneratedTime>
         <ReportFormat>text/csv</ReportFormat>
     </GetCredentialReportResult>
@@ -1277,23 +1309,23 @@ LIST_INSTANCE_PROFILES_FOR_ROLE_TEMPLATE = """<ListInstanceProfilesForRoleRespon
   <InstanceProfiles>
     {% for profile in instance_profiles %}
     <member>
-    <Id>{{ profile.id }}</Id>
-      <Roles>
-        {% for role in profile.roles %}
-        <member>
-          <Path>{{ role.path }}</Path>
-          <Arn>{{ role.arn }}</Arn>
-          <RoleName>{{ role.name }}</RoleName>
-          <AssumeRolePolicyDocument>{{ role.assume_policy_document }}</AssumeRolePolicyDocument>
-          <CreateDate>2012-05-09T15:45:35Z</CreateDate>
-          <RoleId>{{ role.id }}</RoleId>
-        </member>
-        {% endfor %}
-      </Roles>
-      <InstanceProfileName>{{ profile.name }}</InstanceProfileName>
-      <Path>{{ profile.path }}</Path>
-      <Arn>{{ profile.arn }}</Arn>
-      <CreateDate>2012-05-09T16:27:11Z</CreateDate>
+    <InstanceProfileId>{{ profile.id }}</InstanceProfileId>
+    <Roles>
+      {% for role in profile.roles %}
+      <member>
+        <Path>{{ role.path }}</Path>
+        <Arn>{{ role.arn }}</Arn>
+        <RoleName>{{ role.name }}</RoleName>
+        <AssumeRolePolicyDocument>{{ role.assume_policy_document }}</AssumeRolePolicyDocument>
+        <CreateDate>{{ role.create_date }}</CreateDate>
+        <RoleId>{{ role.id }}</RoleId>
+      </member>
+      {% endfor %}
+    </Roles>
+    <InstanceProfileName>{{ profile.name }}</InstanceProfileName>
+    <Path>{{ profile.path }}</Path>
+    <Arn>{{ profile.arn }}</Arn>
+    <CreateDate>{{ profile.create_date }}</CreateDate>
     </member>
     {% endfor %}
   </InstanceProfiles>
@@ -1382,7 +1414,7 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """<GetAccountAuthorizationDetailsR
         <Path>{{ user.path }}</Path>
         <UserName>{{ user.name }}</UserName>
         <Arn>{{ user.arn }}</Arn>
-        <CreateDate>2012-05-09T15:45:35Z</CreateDate>
+        <CreateDate>{{ user.created_iso_8601 }}</CreateDate>
       </member>
     {% endfor %}
     </UserDetailList>
@@ -1401,7 +1433,7 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """<GetAccountAuthorizationDetailsR
         <GroupName>{{ group.name }}</GroupName>
         <Path>{{ group.path }}</Path>
         <Arn>{{ group.arn }}</Arn>
-        <CreateDate>2012-05-09T16:27:11Z</CreateDate>
+        <CreateDate>{{ group.create_date }}</CreateDate>
         <GroupPolicyList/>
       </member>
     {% endfor %}
@@ -1421,23 +1453,23 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """<GetAccountAuthorizationDetailsR
         <InstanceProfileList>
             {% for profile in instance_profiles %}
             <member>
-            <Id>{{ profile.id }}</Id>
-              <Roles>
-                {% for role in profile.roles %}
-                <member>
-                  <Path>{{ role.path }}</Path>
-                  <Arn>{{ role.arn }}</Arn>
-                  <RoleName>{{ role.name }}</RoleName>
-                  <AssumeRolePolicyDocument>{{ role.assume_role_policy_document }}</AssumeRolePolicyDocument>
-                  <CreateDate>2012-05-09T15:45:35Z</CreateDate>
-                  <RoleId>{{ role.id }}</RoleId>
-                </member>
-                {% endfor %}
-              </Roles>
-              <InstanceProfileName>{{ profile.name }}</InstanceProfileName>
-              <Path>{{ profile.path }}</Path>
-              <Arn>{{ profile.arn }}</Arn>
-              <CreateDate>2012-05-09T16:27:11Z</CreateDate>
+            <InstanceProfileId>{{ profile.id }}</InstanceProfileId>
+            <Roles>
+              {% for role in profile.roles %}
+              <member>
+                <Path>{{ role.path }}</Path>
+                <Arn>{{ role.arn }}</Arn>
+                <RoleName>{{ role.name }}</RoleName>
+                <AssumeRolePolicyDocument>{{ role.assume_role_policy_document }}</AssumeRolePolicyDocument>
+                <CreateDate>{{ role.create_date }}</CreateDate>
+                <RoleId>{{ role.id }}</RoleId>
+              </member>
+              {% endfor %}
+            </Roles>
+            <InstanceProfileName>{{ profile.name }}</InstanceProfileName>
+            <Path>{{ profile.path }}</Path>
+            <Arn>{{ profile.arn }}</Arn>
+            <CreateDate>{{ profile.create_date }}</CreateDate>
             </member>
             {% endfor %}
         </InstanceProfileList>
@@ -1445,7 +1477,7 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """<GetAccountAuthorizationDetailsR
         <Arn>{{ role.arn }}</Arn>
         <RoleName>{{ role.name }}</RoleName>
         <AssumeRolePolicyDocument>{{ role.assume_role_policy_document }}</AssumeRolePolicyDocument>
-        <CreateDate>2014-07-30T17:09:20Z</CreateDate>
+        <CreateDate>{{ role.create_date }}</CreateDate>
         <RoleId>{{ role.id }}</RoleId>
       </member>
     {% endfor %}
@@ -1474,9 +1506,9 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """<GetAccountAuthorizationDetailsR
         </PolicyVersionList>
         <Arn>{{ policy.arn }}</Arn>
         <AttachmentCount>1</AttachmentCount>
-        <CreateDate>2012-05-09T16:27:11Z</CreateDate>
+        <CreateDate>{{ policy.create_datetime }}</CreateDate>
         <IsAttachable>true</IsAttachable>
-        <UpdateDate>2012-05-09T16:27:11Z</UpdateDate>
+        <UpdateDate>{{ policy.update_datetime }}</UpdateDate>
       </member>
     {% endfor %}
     </Policies>
@@ -1485,3 +1517,53 @@ GET_ACCOUNT_AUTHORIZATION_DETAILS_TEMPLATE = """<GetAccountAuthorizationDetailsR
     <RequestId>92e79ae7-7399-11e4-8c85-4b53eEXAMPLE</RequestId>
   </ResponseMetadata>
 </GetAccountAuthorizationDetailsResponse>"""
+
+
+UPLOAD_SIGNING_CERTIFICATE_TEMPLATE = """<UploadSigningCertificateResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/">
+  <UploadSigningCertificateResult>
+    <Certificate>
+      <UserName>{{ cert.user_name }}</UserName>
+      <CertificateId>{{ cert.id }}</CertificateId>
+      <CertificateBody>{{ cert.body }}</CertificateBody>
+      <Status>{{ cert.status }}</Status>
+    </Certificate>
+ </UploadSigningCertificateResult>
+ <ResponseMetadata>
+    <RequestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestId>
+ </ResponseMetadata>
+</UploadSigningCertificateResponse>"""
+
+
+UPDATE_SIGNING_CERTIFICATE_TEMPLATE = """<UpdateSigningCertificateResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/">
+ <ResponseMetadata>
+    <RequestId>EXAMPLE8-90ab-cdef-fedc-ba987EXAMPLE</RequestId>
+ </ResponseMetadata>
+</UpdateSigningCertificateResponse>"""
+
+
+DELETE_SIGNING_CERTIFICATE_TEMPLATE = """<DeleteSigningCertificateResponse xmlns="https://iam.amazonaws.com/doc/2010-05-08/">
+  <ResponseMetadata>
+    <RequestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestId>
+  </ResponseMetadata>
+</DeleteSigningCertificateResponse>"""
+
+
+LIST_SIGNING_CERTIFICATES_TEMPLATE = """<ListSigningCertificatesResponse>
+  <ListSigningCertificatesResult>
+    <UserName>{{ user_name }}</UserName>
+    <Certificates>
+       {% for cert in certificates %}
+       <member>
+          <UserName>{{ user_name }}</UserName>
+          <CertificateId>{{ cert.id }}</CertificateId>
+          <CertificateBody>{{ cert.body }}</CertificateBody>
+          <Status>{{ cert.status }}</Status>
+       </member>
+       {% endfor %}
+    </Certificates>
+    <IsTruncated>false</IsTruncated>
+ </ListSigningCertificatesResult>
+ <ResponseMetadata>
+    <RequestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestId>
+ </ResponseMetadata>
+</ListSigningCertificatesResponse>"""

moto/iam/utils.py

@@ -12,8 +12,7 @@ def random_alphanumeric(length):
     )
 
 
-def random_resource_id():
-    size = 20
+def random_resource_id(size=20):
     chars = list(range(10)) + list(string.ascii_lowercase)
 
     return ''.join(six.text_type(random.choice(chars)) for x in range(size))

moto/iot/exceptions.py

@@ -31,3 +31,20 @@ class VersionConflictException(IoTClientError):
             'VersionConflictException',
             'The version for thing %s does not match the expected version.' % name
         )
+
+
+class CertificateStateException(IoTClientError):
+    def __init__(self, msg, cert_id):
+        self.code = 406
+        super(CertificateStateException, self).__init__(
+            'CertificateStateException',
+            '%s Id: %s' % (msg, cert_id)
+        )
+
+
+class DeleteConflictException(IoTClientError):
+    def __init__(self, msg):
+        self.code = 409
+        super(DeleteConflictException, self).__init__(
+            'DeleteConflictException', msg
+        )

moto/iot/models.py

@@ -13,6 +13,8 @@ import boto3
 
 from moto.core import BaseBackend, BaseModel
 from .exceptions import (
+    CertificateStateException,
+    DeleteConflictException,
     ResourceNotFoundException,
     InvalidRequestException,
     VersionConflictException
@@ -378,7 +380,25 @@ class IoTBackend(BaseBackend):
         return certificate, key_pair
 
     def delete_certificate(self, certificate_id):
-        self.describe_certificate(certificate_id)
+        cert = self.describe_certificate(certificate_id)
+        if cert.status == 'ACTIVE':
+            raise CertificateStateException(
+                'Certificate must be deactivated (not ACTIVE) before deletion.', certificate_id)
+
+        certs = [k[0] for k, v in self.principal_things.items()
+                 if self._get_principal(k[0]).certificate_id == certificate_id]
+        if len(certs) > 0:
+            raise DeleteConflictException(
+                'Things must be detached before deletion (arn: %s)' % certs[0]
+            )
+
+        certs = [k[0] for k, v in self.principal_policies.items()
+                 if self._get_principal(k[0]).certificate_id == certificate_id]
+        if len(certs) > 0:
+            raise DeleteConflictException(
+                'Certificate policies must be detached before deletion (arn: %s)' % certs[0]
+            )
+
         del self.certificates[certificate_id]
 
     def describe_certificate(self, certificate_id):
@@ -411,6 +431,14 @@ class IoTBackend(BaseBackend):
         return policies[0]
 
     def delete_policy(self, policy_name):
+
+        policies = [k[1] for k, v in self.principal_policies.items() if k[1] == policy_name]
+        if len(policies) > 0:
+            raise DeleteConflictException(
+                'The policy cannot be deleted as the policy is attached to one or more principals (name=%s)'
+                % policy_name
+            )
+
         policy = self.get_policy(policy_name)
         del self.policies[policy.name]
 
@@ -429,6 +457,14 @@ class IoTBackend(BaseBackend):
             pass
         raise ResourceNotFoundException()
 
+    def attach_policy(self, policy_name, target):
+        principal = self._get_principal(target)
+        policy = self.get_policy(policy_name)
+        k = (target, policy_name)
+        if k in self.principal_policies:
+            return
+        self.principal_policies[k] = (principal, policy)
+
     def attach_principal_policy(self, policy_name, principal_arn):
         principal = self._get_principal(principal_arn)
         policy = self.get_policy(policy_name)
@@ -437,6 +473,15 @@ class IoTBackend(BaseBackend):
             return
         self.principal_policies[k] = (principal, policy)
 
+    def detach_policy(self, policy_name, target):
+        # this may raises ResourceNotFoundException
+        self._get_principal(target)
+        self.get_policy(policy_name)
+        k = (target, policy_name)
+        if k not in self.principal_policies:
+            raise ResourceNotFoundException()
+        del self.principal_policies[k]
+
     def detach_principal_policy(self, policy_name, principal_arn):
         # this may raises ResourceNotFoundException
         self._get_principal(principal_arn)

moto/iot/responses.py

@@ -224,6 +224,15 @@ class IoTResponse(BaseResponse):
         )
         return json.dumps(dict())
 
+    def attach_policy(self):
+        policy_name = self._get_param("policyName")
+        target = self._get_param('target')
+        self.iot_backend.attach_policy(
+            policy_name=policy_name,
+            target=target,
+        )
+        return json.dumps(dict())
+
     def attach_principal_policy(self):
         policy_name = self._get_param("policyName")
         principal = self.headers.get('x-amzn-iot-principal')
@@ -233,6 +242,15 @@ class IoTResponse(BaseResponse):
         )
         return json.dumps(dict())
 
+    def detach_policy(self):
+        policy_name = self._get_param("policyName")
+        target = self._get_param('target')
+        self.iot_backend.detach_policy(
+            policy_name=policy_name,
+            target=target,
+        )
+        return json.dumps(dict())
+
     def detach_principal_policy(self):
         policy_name = self._get_param("policyName")
         principal = self.headers.get('x-amzn-iot-principal')

moto/s3/exceptions.py

@@ -178,3 +178,13 @@ class InvalidStorageClass(S3ClientError):
             "InvalidStorageClass",
             "The storage class you specified is not valid",
             *args, **kwargs)
+
+
+class DuplicateTagKeys(S3ClientError):
+    code = 400
+
+    def __init__(self, *args, **kwargs):
+        super(DuplicateTagKeys, self).__init__(
+            "InvalidTag",
+            "Cannot provide multiple Tags with the same key",
+            *args, **kwargs)

moto/s3/models.py

@@ -15,7 +15,7 @@ from bisect import insort
 from moto.core import BaseBackend, BaseModel
 from moto.core.utils import iso_8601_datetime_with_milliseconds, rfc_1123_datetime
 from .exceptions import BucketAlreadyExists, MissingBucket, InvalidPart, EntityTooSmall, MissingKey, \
-    InvalidNotificationDestination, MalformedXML, InvalidStorageClass
+    InvalidNotificationDestination, MalformedXML, InvalidStorageClass, DuplicateTagKeys
 from .utils import clean_key_name, _VersionedKeyStore
 
 UPLOAD_ID_BYTES = 43
@@ -773,6 +773,9 @@ class S3Backend(BaseBackend):
         return key
 
     def put_bucket_tagging(self, bucket_name, tagging):
+        tag_keys = [tag.key for tag in tagging.tag_set.tags]
+        if len(tag_keys) != len(set(tag_keys)):
+            raise DuplicateTagKeys()
         bucket = self.get_bucket(bucket_name)
         bucket.set_tags(tagging)
 

moto/secretsmanager/models.py

@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import time
 import json
+import uuid
 
 import boto3
 
@@ -18,10 +19,6 @@ class SecretsManager(BaseModel):
 
     def __init__(self, region_name, **kwargs):
         self.region = region_name
-        self.secret_id = kwargs.get('secret_id', '')
-        self.version_id = kwargs.get('version_id', '')
-        self.version_stage = kwargs.get('version_stage', '')
-        self.secret_string = ''
 
 
 class SecretsManagerBackend(BaseBackend):
@@ -29,14 +26,7 @@ class SecretsManagerBackend(BaseBackend):
     def __init__(self, region_name=None, **kwargs):
         super(SecretsManagerBackend, self).__init__()
         self.region = region_name
-        self.secret_id = kwargs.get('secret_id', '')
-        self.name = kwargs.get('name', '')
-        self.createdate = int(time.time())
-        self.secret_string = ''
-        self.rotation_enabled = False
-        self.rotation_lambda_arn = ''
-        self.auto_rotate_after_days = 0
-        self.version_id = ''
+        self.secrets = {}
 
     def reset(self):
         region_name = self.region
@@ -44,36 +34,50 @@ class SecretsManagerBackend(BaseBackend):
         self.__init__(region_name)
 
     def _is_valid_identifier(self, identifier):
-        return identifier in (self.name, self.secret_id)
+        return identifier in self.secrets
 
     def get_secret_value(self, secret_id, version_id, version_stage):
 
         if not self._is_valid_identifier(secret_id):
             raise ResourceNotFoundException()
 
+        secret = self.secrets[secret_id]
+
         response = json.dumps({
-            "ARN": secret_arn(self.region, self.secret_id),
-            "Name": self.name,
-            "VersionId": "A435958A-D821-4193-B719-B7769357AER4",
-            "SecretString": self.secret_string,
+            "ARN": secret_arn(self.region, secret['secret_id']),
+            "Name": secret['name'],
+            "VersionId": secret['version_id'],
+            "SecretString": secret['secret_string'],
             "VersionStages": [
                 "AWSCURRENT",
             ],
-            "CreatedDate": "2018-05-23 13:16:57.198000"
+            "CreatedDate": secret['createdate']
         })
 
         return response
 
-    def create_secret(self, name, secret_string, **kwargs):
+    def create_secret(self, name, secret_string, tags, **kwargs):
 
-        self.secret_string = secret_string
-        self.secret_id = name
-        self.name = name
+        generated_version_id = str(uuid.uuid4())
+
+        secret = {
+            'secret_string': secret_string,
+            'secret_id': name,
+            'name': name,
+            'createdate': int(time.time()),
+            'rotation_enabled': False,
+            'rotation_lambda_arn': '',
+            'auto_rotate_after_days': 0,
+            'version_id': generated_version_id,
+            'tags': tags
+        }
+
+        self.secrets[name] = secret
 
         response = json.dumps({
             "ARN": secret_arn(self.region, name),
-            "Name": self.name,
-            "VersionId": "A435958A-D821-4193-B719-B7769357AER4",
+            "Name": name,
+            "VersionId": generated_version_id,
         })
 
         return response
@@ -82,26 +86,23 @@ class SecretsManagerBackend(BaseBackend):
         if not self._is_valid_identifier(secret_id):
             raise ResourceNotFoundException
 
+        secret = self.secrets[secret_id]
+
         response = json.dumps({
-            "ARN": secret_arn(self.region, self.secret_id),
-            "Name": self.name,
+            "ARN": secret_arn(self.region, secret['secret_id']),
+            "Name": secret['name'],
             "Description": "",
             "KmsKeyId": "",
-            "RotationEnabled": self.rotation_enabled,
-            "RotationLambdaARN": self.rotation_lambda_arn,
+            "RotationEnabled": secret['rotation_enabled'],
+            "RotationLambdaARN": secret['rotation_lambda_arn'],
             "RotationRules": {
-                "AutomaticallyAfterDays": self.auto_rotate_after_days
+                "AutomaticallyAfterDays": secret['auto_rotate_after_days']
             },
             "LastRotatedDate": None,
             "LastChangedDate": None,
             "LastAccessedDate": None,
             "DeletedDate": None,
-            "Tags": [
-                {
-                    "Key": "",
-                    "Value": ""
-                },
-            ]
+            "Tags": secret['tags']
         })
 
         return response
@@ -141,17 +142,19 @@ class SecretsManagerBackend(BaseBackend):
                     )
                     raise InvalidParameterException(msg)
 
-        self.version_id = client_request_token or ''
-        self.rotation_lambda_arn = rotation_lambda_arn or ''
+        secret = self.secrets[secret_id]
+
+        secret['version_id'] = client_request_token or ''
+        secret['rotation_lambda_arn'] = rotation_lambda_arn or ''
         if rotation_rules:
-            self.auto_rotate_after_days = rotation_rules.get(rotation_days, 0)
-        if self.auto_rotate_after_days > 0:
-            self.rotation_enabled = True
+            secret['auto_rotate_after_days'] = rotation_rules.get(rotation_days, 0)
+        if secret['auto_rotate_after_days'] > 0:
+            secret['rotation_enabled'] = True
 
         response = json.dumps({
-            "ARN": secret_arn(self.region, self.secret_id),
-            "Name": self.name,
-            "VersionId": self.version_id
+            "ARN": secret_arn(self.region, secret['secret_id']),
+            "Name": secret['name'],
+            "VersionId": secret['version_id']
         })
 
         return response

moto/secretsmanager/responses.py

@@ -19,9 +19,11 @@ class SecretsManagerResponse(BaseResponse):
     def create_secret(self):
         name = self._get_param('Name')
         secret_string = self._get_param('SecretString')
+        tags = self._get_param('Tags', if_none=[])
         return secretsmanager_backends[self.region].create_secret(
             name=name,
-            secret_string=secret_string
+            secret_string=secret_string,
+            tags=tags
         )
 
     def get_random_password(self):

moto/secretsmanager/utils.py

@@ -52,8 +52,9 @@ def random_password(password_length, exclude_characters, exclude_numbers,
 
 
 def secret_arn(region, secret_id):
-    return "arn:aws:secretsmanager:{0}:1234567890:secret:{1}-rIjad".format(
-        region, secret_id)
+    id_string = ''.join(random.choice(string.ascii_letters) for _ in range(5))
+    return "arn:aws:secretsmanager:{0}:1234567890:secret:{1}-{2}".format(
+        region, secret_id, id_string)
 
 
 def _exclude_characters(password, exclude_characters):

moto/server.py

@@ -80,10 +80,13 @@ class DomainDispatcherApplication(object):
 
                 region = 'us-east-1'
             if service == 'dynamodb':
-                dynamo_api_version = environ['HTTP_X_AMZ_TARGET'].split("_")[1].split(".")[0]
-                # If Newer API version, use dynamodb2
-                if dynamo_api_version > "20111205":
-                    host = "dynamodb2"
+                if environ['HTTP_X_AMZ_TARGET'].startswith('DynamoDBStreams'):
+                    host = 'dynamodbstreams'
+                else:
+                    dynamo_api_version = environ['HTTP_X_AMZ_TARGET'].split("_")[1].split(".")[0]
+                    # If Newer API version, use dynamodb2
+                    if dynamo_api_version > "20111205":
+                        host = "dynamodb2"
             else:
                 host = "{service}.{region}.amazonaws.com".format(
                     service=service, region=region)

moto/sqs/models.py

@@ -534,7 +534,7 @@ class SQSBackend(BaseBackend):
                     break
 
                 import time
-                time.sleep(0.001)
+                time.sleep(0.01)
                 continue
 
             previous_result_count = len(result)