Merge branch 'master' into get-caller-identity

moto/__init__.py

@@ -3,7 +3,7 @@ import logging
 # logging.getLogger('boto').setLevel(logging.CRITICAL)
 
 __title__ = 'moto'
-__version__ = '1.3.11'
+__version__ = '1.3.14.dev'
 
 from .acm import mock_acm  # flake8: noqa
 from .apigateway import mock_apigateway, mock_apigateway_deprecated  # flake8: noqa

moto/apigateway/models.py

@@ -309,6 +309,25 @@ class ApiKey(BaseModel, dict):
         self['createdDate'] = self['lastUpdatedDate'] = int(time.time())
         self['stageKeys'] = stageKeys
 
+    def update_operations(self, patch_operations):
+        for op in patch_operations:
+            if op['op'] == 'replace':
+                if '/name' in op['path']:
+                    self['name'] = op['value']
+                elif '/customerId' in op['path']:
+                    self['customerId'] = op['value']
+                elif '/description' in op['path']:
+                    self['description'] = op['value']
+                elif '/enabled' in op['path']:
+                    self['enabled'] = self._str2bool(op['value'])
+            else:
+                raise Exception(
+                    'Patch operation "%s" not implemented' % op['op'])
+        return self
+
+    def _str2bool(self, v):
+        return v.lower() == "true"
+
 
 class UsagePlan(BaseModel, dict):
 
@@ -599,6 +618,10 @@ class APIGatewayBackend(BaseBackend):
     def get_apikey(self, api_key_id):
         return self.keys[api_key_id]
 
+    def update_apikey(self, api_key_id, patch_operations):
+        key = self.keys[api_key_id]
+        return key.update_operations(patch_operations)
+
     def delete_apikey(self, api_key_id):
         self.keys.pop(api_key_id)
         return {}

moto/apigateway/responses.py

@@ -245,6 +245,9 @@ class APIGatewayResponse(BaseResponse):
 
         if self.method == 'GET':
             apikey_response = self.backend.get_apikey(apikey)
+        elif self.method == 'PATCH':
+            patch_operations = self._get_param('patchOperations')
+            apikey_response = self.backend.update_apikey(apikey, patch_operations)
         elif self.method == 'DELETE':
             apikey_response = self.backend.delete_apikey(apikey)
         return 200, {}, json.dumps(apikey_response)

moto/apigateway/utils.py

@@ -1,9 +1,10 @@
 from __future__ import unicode_literals
 import six
 import random
+import string
 
 
 def create_id():
     size = 10
-    chars = list(range(10)) + ['A-Z']
+    chars = list(range(10)) + list(string.ascii_lowercase)
     return ''.join(six.text_type(random.choice(chars)) for x in range(size))

moto/autoscaling/exceptions.py

@@ -13,3 +13,12 @@ class ResourceContentionError(RESTError):
         super(ResourceContentionError, self).__init__(
             "ResourceContentionError",
             "You already have a pending update to an Auto Scaling resource (for example, a group, instance, or load balancer).")
+
+
+class InvalidInstanceError(AutoscalingClientError):
+
+    def __init__(self, instance_id):
+        super(InvalidInstanceError, self).__init__(
+            "ValidationError",
+            "Instance [{0}] is invalid."
+            .format(instance_id))

moto/autoscaling/models.py

@@ -3,6 +3,8 @@ from __future__ import unicode_literals
 import random
 
 from boto.ec2.blockdevicemapping import BlockDeviceType, BlockDeviceMapping
+from moto.ec2.exceptions import InvalidInstanceIdError
+
 from moto.compat import OrderedDict
 from moto.core import BaseBackend, BaseModel
 from moto.ec2 import ec2_backends
@@ -10,7 +12,7 @@ from moto.elb import elb_backends
 from moto.elbv2 import elbv2_backends
 from moto.elb.exceptions import LoadBalancerNotFoundError
 from .exceptions import (
-    AutoscalingClientError, ResourceContentionError,
+    AutoscalingClientError, ResourceContentionError, InvalidInstanceError
 )
 
 # http://docs.aws.amazon.com/AutoScaling/latest/DeveloperGuide/AS_Concepts.html#Cooldown
@@ -73,6 +75,26 @@ class FakeLaunchConfiguration(BaseModel):
         self.associate_public_ip_address = associate_public_ip_address
         self.block_device_mapping_dict = block_device_mapping_dict
 
+    @classmethod
+    def create_from_instance(cls, name, instance, backend):
+        config = backend.create_launch_configuration(
+            name=name,
+            image_id=instance.image_id,
+            kernel_id='',
+            ramdisk_id='',
+            key_name=instance.key_name,
+            security_groups=instance.security_groups,
+            user_data=instance.user_data,
+            instance_type=instance.instance_type,
+            instance_monitoring=False,
+            instance_profile_name=None,
+            spot_price=None,
+            ebs_optimized=instance.ebs_optimized,
+            associate_public_ip_address=instance.associate_public_ip,
+            block_device_mappings=instance.block_device_mapping
+        )
+        return config
+
     @classmethod
     def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name):
         properties = cloudformation_json['Properties']
@@ -279,6 +301,12 @@ class FakeAutoScalingGroup(BaseModel):
         if min_size is not None:
             self.min_size = min_size
 
+        if desired_capacity is None:
+            if min_size is not None and min_size > len(self.instance_states):
+                desired_capacity = min_size
+            if max_size is not None and max_size < len(self.instance_states):
+                desired_capacity = max_size
+
         if launch_config_name:
             self.launch_config = self.autoscaling_backend.launch_configurations[
                 launch_config_name]
@@ -414,7 +442,8 @@ class AutoScalingBackend(BaseBackend):
                                  health_check_type, load_balancers,
                                  target_group_arns, placement_group,
                                  termination_policies, tags,
-                                 new_instances_protected_from_scale_in=False):
+                                 new_instances_protected_from_scale_in=False,
+                                  instance_id=None):
 
         def make_int(value):
             return int(value) if value is not None else value
@@ -427,6 +456,13 @@ class AutoScalingBackend(BaseBackend):
             health_check_period = 300
         else:
             health_check_period = make_int(health_check_period)
+        if launch_config_name is None and instance_id is not None:
+            try:
+                instance = self.ec2_backend.get_instance(instance_id)
+                launch_config_name = name
+                FakeLaunchConfiguration.create_from_instance(launch_config_name, instance, self)
+            except InvalidInstanceIdError:
+                raise InvalidInstanceError(instance_id)
 
         group = FakeAutoScalingGroup(
             name=name,
@@ -684,6 +720,18 @@ class AutoScalingBackend(BaseBackend):
         for instance in protected_instances:
             instance.protected_from_scale_in = protected_from_scale_in
 
+    def notify_terminate_instances(self, instance_ids):
+        for autoscaling_group_name, autoscaling_group in self.autoscaling_groups.items():
+            original_instance_count = len(autoscaling_group.instance_states)
+            autoscaling_group.instance_states = list(filter(
+                lambda i_state: i_state.instance.id not in instance_ids,
+                autoscaling_group.instance_states
+            ))
+            difference = original_instance_count - len(autoscaling_group.instance_states)
+            if difference > 0:
+                autoscaling_group.replace_autoscaling_group_instances(difference, autoscaling_group.get_propagated_tags())
+                self.update_attached_elbs(autoscaling_group_name)
+
 
 autoscaling_backends = {}
 for region, ec2_backend in ec2_backends.items():

moto/autoscaling/responses.py

@@ -48,7 +48,7 @@ class AutoScalingResponse(BaseResponse):
             start = all_names.index(marker) + 1
         else:
             start = 0
-        max_records = self._get_param('MaxRecords', 50)  # the default is 100, but using 50 to make testing easier
+        max_records = self._get_int_param('MaxRecords', 50)  # the default is 100, but using 50 to make testing easier
         launch_configurations_resp = all_launch_configurations[start:start + max_records]
         next_token = None
         if len(all_launch_configurations) > start + max_records:
@@ -74,6 +74,7 @@ class AutoScalingResponse(BaseResponse):
             desired_capacity=self._get_int_param('DesiredCapacity'),
             max_size=self._get_int_param('MaxSize'),
             min_size=self._get_int_param('MinSize'),
+            instance_id=self._get_param('InstanceId'),
             launch_config_name=self._get_param('LaunchConfigurationName'),
             vpc_zone_identifier=self._get_param('VPCZoneIdentifier'),
             default_cooldown=self._get_int_param('DefaultCooldown'),

moto/batch/models.py

@@ -514,10 +514,13 @@ class BatchBackend(BaseBackend):
         return self._job_definitions.get(arn)
 
     def get_job_definition_by_name(self, name):
-        for comp_env in self._job_definitions.values():
-            if comp_env.name == name:
-                return comp_env
-        return None
+        latest_revision = -1
+        latest_job = None
+        for job_def in self._job_definitions.values():
+            if job_def.name == name and job_def.revision > latest_revision:
+                latest_job = job_def
+                latest_revision = job_def.revision
+        return latest_job
 
     def get_job_definition_by_name_revision(self, name, revision):
         for job_def in self._job_definitions.values():
@@ -534,10 +537,13 @@ class BatchBackend(BaseBackend):
         :return: Job definition or None
         :rtype: JobDefinition or None
         """
-        env = self.get_job_definition_by_arn(identifier)
-        if env is None:
-            env = self.get_job_definition_by_name(identifier)
-        return env
+        job_def = self.get_job_definition_by_arn(identifier)
+        if job_def is None:
+            if ':' in identifier:
+                job_def = self.get_job_definition_by_name_revision(*identifier.split(':', 1))
+            else:
+                job_def = self.get_job_definition_by_name(identifier)
+        return job_def
 
     def get_job_definitions(self, identifier):
         """
@@ -984,9 +990,7 @@ class BatchBackend(BaseBackend):
         # TODO parameters, retries (which is a dict raw from request), job dependancies and container overrides are ignored for now
 
         # Look for job definition
-        job_def = self.get_job_definition_by_arn(job_def_id)
-        if job_def is None and ':' in job_def_id:
-            job_def = self.get_job_definition_by_name_revision(*job_def_id.split(':', 1))
+        job_def = self.get_job_definition(job_def_id)
         if job_def is None:
             raise ClientException('Job definition {0} does not exist'.format(job_def_id))
 

moto/cognitoidentity/models.py

@@ -95,6 +95,15 @@ class CognitoIdentityBackend(BaseBackend):
             })
         return response
 
+    def get_open_id_token(self, identity_id):
+        response = json.dumps(
+            {
+                "IdentityId": identity_id,
+                "Token": get_random_identity_id(self.region)
+            }
+        )
+        return response
+
 
 cognitoidentity_backends = {}
 for region in boto.cognito.identity.regions():

moto/cognitoidentity/responses.py

@@ -35,3 +35,8 @@ class CognitoIdentityResponse(BaseResponse):
         return cognitoidentity_backends[self.region].get_open_id_token_for_developer_identity(
             self._get_param('IdentityId') or get_random_identity_id(self.region)
         )
+
+    def get_open_id_token(self):
+        return cognitoidentity_backends[self.region].get_open_id_token(
+            self._get_param("IdentityId") or get_random_identity_id(self.region)
+        )

moto/cognitoidp/models.py

@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import datetime
 import functools
+import hashlib
 import itertools
 import json
 import os
@@ -154,20 +155,37 @@ class CognitoIdpUserPool(BaseModel):
 
 class CognitoIdpUserPoolDomain(BaseModel):
 
-    def __init__(self, user_pool_id, domain):
+    def __init__(self, user_pool_id, domain, custom_domain_config=None):
         self.user_pool_id = user_pool_id
         self.domain = domain
+        self.custom_domain_config = custom_domain_config or {}
 
-    def to_json(self):
-        return {
-            "UserPoolId": self.user_pool_id,
-            "AWSAccountId": str(uuid.uuid4()),
-            "CloudFrontDistribution": None,
-            "Domain": self.domain,
-            "S3Bucket": None,
-            "Status": "ACTIVE",
-            "Version": None,
-        }
+    def _distribution_name(self):
+        if self.custom_domain_config and \
+                'CertificateArn' in self.custom_domain_config:
+            hash = hashlib.md5(
+                self.custom_domain_config['CertificateArn'].encode('utf-8')
+            ).hexdigest()
+            return "{hash}.cloudfront.net".format(hash=hash[:16])
+        return None
+
+    def to_json(self, extended=True):
+        distribution = self._distribution_name()
+        if extended:
+            return {
+                "UserPoolId": self.user_pool_id,
+                "AWSAccountId": str(uuid.uuid4()),
+                "CloudFrontDistribution": distribution,
+                "Domain": self.domain,
+                "S3Bucket": None,
+                "Status": "ACTIVE",
+                "Version": None,
+            }
+        elif distribution:
+            return {
+                "CloudFrontDomain": distribution,
+            }
+        return None
 
 
 class CognitoIdpUserPoolClient(BaseModel):
@@ -338,11 +356,13 @@ class CognitoIdpBackend(BaseBackend):
         del self.user_pools[user_pool_id]
 
     # User pool domain
-    def create_user_pool_domain(self, user_pool_id, domain):
+    def create_user_pool_domain(self, user_pool_id, domain, custom_domain_config=None):
         if user_pool_id not in self.user_pools:
             raise ResourceNotFoundError(user_pool_id)
 
-        user_pool_domain = CognitoIdpUserPoolDomain(user_pool_id, domain)
+        user_pool_domain = CognitoIdpUserPoolDomain(
+            user_pool_id, domain, custom_domain_config=custom_domain_config
+        )
         self.user_pool_domains[domain] = user_pool_domain
         return user_pool_domain
 
@@ -358,6 +378,14 @@ class CognitoIdpBackend(BaseBackend):
 
         del self.user_pool_domains[domain]
 
+    def update_user_pool_domain(self, domain, custom_domain_config):
+        if domain not in self.user_pool_domains:
+            raise ResourceNotFoundError(domain)
+
+        user_pool_domain = self.user_pool_domains[domain]
+        user_pool_domain.custom_domain_config = custom_domain_config
+        return user_pool_domain
+
     # User pool client
     def create_user_pool_client(self, user_pool_id, extended_config):
         user_pool = self.user_pools.get(user_pool_id)

moto/cognitoidp/responses.py

@@ -50,7 +50,13 @@ class CognitoIdpResponse(BaseResponse):
     def create_user_pool_domain(self):
         domain = self._get_param("Domain")
         user_pool_id = self._get_param("UserPoolId")
-        cognitoidp_backends[self.region].create_user_pool_domain(user_pool_id, domain)
+        custom_domain_config = self._get_param("CustomDomainConfig")
+        user_pool_domain = cognitoidp_backends[self.region].create_user_pool_domain(
+            user_pool_id, domain, custom_domain_config
+        )
+        domain_description = user_pool_domain.to_json(extended=False)
+        if domain_description:
+            return json.dumps(domain_description)
         return ""
 
     def describe_user_pool_domain(self):
@@ -69,6 +75,17 @@ class CognitoIdpResponse(BaseResponse):
         cognitoidp_backends[self.region].delete_user_pool_domain(domain)
         return ""
 
+    def update_user_pool_domain(self):
+        domain = self._get_param("Domain")
+        custom_domain_config = self._get_param("CustomDomainConfig")
+        user_pool_domain = cognitoidp_backends[self.region].update_user_pool_domain(
+            domain, custom_domain_config
+        )
+        domain_description = user_pool_domain.to_json(extended=False)
+        if domain_description:
+            return json.dumps(domain_description)
+        return ""
+
     # User pool client
     def create_user_pool_client(self):
         user_pool_id = self.parameters.pop("UserPoolId")

moto/config/exceptions.py

@@ -52,6 +52,18 @@ class InvalidResourceTypeException(JsonRESTError):
         super(InvalidResourceTypeException, self).__init__("ValidationException", message)
 
 
+class NoSuchConfigurationAggregatorException(JsonRESTError):
+    code = 400
+
+    def __init__(self, number=1):
+        if number == 1:
+            message = 'The configuration aggregator does not exist. Check the configuration aggregator name and try again.'
+        else:
+            message = 'At least one of the configuration aggregators does not exist. Check the configuration aggregator' \
+                      ' names and try again.'
+        super(NoSuchConfigurationAggregatorException, self).__init__("NoSuchConfigurationAggregatorException", message)
+
+
 class NoSuchConfigurationRecorderException(JsonRESTError):
     code = 400
 
@@ -78,6 +90,14 @@ class NoSuchBucketException(JsonRESTError):
         super(NoSuchBucketException, self).__init__("NoSuchBucketException", message)
 
 
+class InvalidNextTokenException(JsonRESTError):
+    code = 400
+
+    def __init__(self):
+        message = 'The nextToken provided is invalid'
+        super(InvalidNextTokenException, self).__init__("InvalidNextTokenException", message)
+
+
 class InvalidS3KeyPrefixException(JsonRESTError):
     code = 400
 
@@ -147,3 +167,66 @@ class LastDeliveryChannelDeleteFailedException(JsonRESTError):
         message = 'Failed to delete last specified delivery channel with name \'{name}\', because there, ' \
                   'because there is a running configuration recorder.'.format(name=name)
         super(LastDeliveryChannelDeleteFailedException, self).__init__("LastDeliveryChannelDeleteFailedException", message)
+
+
+class TooManyAccountSources(JsonRESTError):
+    code = 400
+
+    def __init__(self, length):
+        locations = ['com.amazonaws.xyz'] * length
+
+        message = 'Value \'[{locations}]\' at \'accountAggregationSources\' failed to satisfy constraint: ' \
+                  'Member must have length less than or equal to 1'.format(locations=', '.join(locations))
+        super(TooManyAccountSources, self).__init__("ValidationException", message)
+
+
+class DuplicateTags(JsonRESTError):
+    code = 400
+
+    def __init__(self):
+        super(DuplicateTags, self).__init__(
+            'InvalidInput', 'Duplicate tag keys found. Please note that Tag keys are case insensitive.')
+
+
+class TagKeyTooBig(JsonRESTError):
+    code = 400
+
+    def __init__(self, tag, param='tags.X.member.key'):
+        super(TagKeyTooBig, self).__init__(
+            'ValidationException', "1 validation error detected: Value '{}' at '{}' failed to satisfy "
+                                   "constraint: Member must have length less than or equal to 128".format(tag, param))
+
+
+class TagValueTooBig(JsonRESTError):
+    code = 400
+
+    def __init__(self, tag):
+        super(TagValueTooBig, self).__init__(
+            'ValidationException', "1 validation error detected: Value '{}' at 'tags.X.member.value' failed to satisfy "
+                                   "constraint: Member must have length less than or equal to 256".format(tag))
+
+
+class InvalidParameterValueException(JsonRESTError):
+    code = 400
+
+    def __init__(self, message):
+        super(InvalidParameterValueException, self).__init__('InvalidParameterValueException', message)
+
+
+class InvalidTagCharacters(JsonRESTError):
+    code = 400
+
+    def __init__(self, tag, param='tags.X.member.key'):
+        message = "1 validation error detected: Value '{}' at '{}' failed to satisfy ".format(tag, param)
+        message += 'constraint: Member must satisfy regular expression pattern: [\\\\p{L}\\\\p{Z}\\\\p{N}_.:/=+\\\\-@]+'
+
+        super(InvalidTagCharacters, self).__init__('ValidationException', message)
+
+
+class TooManyTags(JsonRESTError):
+    code = 400
+
+    def __init__(self, tags, param='tags'):
+        super(TooManyTags, self).__init__(
+            'ValidationException', "1 validation error detected: Value '{}' at '{}' failed to satisfy "
+                                   "constraint: Member must have length less than or equal to 50.".format(tags, param))

moto/config/models.py

@@ -1,6 +1,9 @@
 import json
+import re
 import time
 import pkg_resources
+import random
+import string
 
 from datetime import datetime
 
@@ -12,37 +15,125 @@ from moto.config.exceptions import InvalidResourceTypeException, InvalidDelivery
     NoSuchConfigurationRecorderException, NoAvailableConfigurationRecorderException, \
     InvalidDeliveryChannelNameException, NoSuchBucketException, InvalidS3KeyPrefixException, \
     InvalidSNSTopicARNException, MaxNumberOfDeliveryChannelsExceededException, NoAvailableDeliveryChannelException, \
-    NoSuchDeliveryChannelException, LastDeliveryChannelDeleteFailedException
+    NoSuchDeliveryChannelException, LastDeliveryChannelDeleteFailedException, TagKeyTooBig, \
+    TooManyTags, TagValueTooBig, TooManyAccountSources, InvalidParameterValueException, InvalidNextTokenException, \
+    NoSuchConfigurationAggregatorException, InvalidTagCharacters, DuplicateTags
 
 from moto.core import BaseBackend, BaseModel
 
 DEFAULT_ACCOUNT_ID = 123456789012
+POP_STRINGS = [
+    'capitalizeStart',
+    'CapitalizeStart',
+    'capitalizeArn',
+    'CapitalizeArn',
+    'capitalizeARN',
+    'CapitalizeARN'
+]
+DEFAULT_PAGE_SIZE = 100
 
 
 def datetime2int(date):
     return int(time.mktime(date.timetuple()))
 
 
-def snake_to_camels(original):
+def snake_to_camels(original, cap_start, cap_arn):
     parts = original.split('_')
 
     camel_cased = parts[0].lower() + ''.join(p.title() for p in parts[1:])
-    camel_cased = camel_cased.replace('Arn', 'ARN')  # Config uses 'ARN' instead of 'Arn'
+
+    if cap_arn:
+        camel_cased = camel_cased.replace('Arn', 'ARN')  # Some config services use 'ARN' instead of 'Arn'
+
+    if cap_start:
+        camel_cased = camel_cased[0].upper() + camel_cased[1::]
 
     return camel_cased
 
 
+def random_string():
+    """Returns a random set of 8 lowercase letters for the Config Aggregator ARN"""
+    chars = []
+    for x in range(0, 8):
+        chars.append(random.choice(string.ascii_lowercase))
+
+    return "".join(chars)
+
+
+def validate_tag_key(tag_key, exception_param='tags.X.member.key'):
+    """Validates the tag key.
+
+    :param tag_key: The tag key to check against.
+    :param exception_param: The exception parameter to send over to help format the message. This is to reflect
+                            the difference between the tag and untag APIs.
+    :return:
+    """
+    # Validate that the key length is correct:
+    if len(tag_key) > 128:
+        raise TagKeyTooBig(tag_key, param=exception_param)
+
+    # Validate that the tag key fits the proper Regex:
+    # [\w\s_.:/=+\-@]+ SHOULD be the same as the Java regex on the AWS documentation: [\p{L}\p{Z}\p{N}_.:/=+\-@]+
+    match = re.findall(r'[\w\s_.:/=+\-@]+', tag_key)
+    # Kudos if you can come up with a better way of doing a global search :)
+    if not len(match) or len(match[0]) < len(tag_key):
+        raise InvalidTagCharacters(tag_key, param=exception_param)
+
+
+def check_tag_duplicate(all_tags, tag_key):
+    """Validates that a tag key is not a duplicate
+
+    :param all_tags: Dict to check if there is a duplicate tag.
+    :param tag_key: The tag key to check against.
+    :return:
+    """
+    if all_tags.get(tag_key):
+        raise DuplicateTags()
+
+
+def validate_tags(tags):
+    proper_tags = {}
+
+    if len(tags) > 50:
+        raise TooManyTags(tags)
+
+    for tag in tags:
+        # Validate the Key:
+        validate_tag_key(tag['Key'])
+        check_tag_duplicate(proper_tags, tag['Key'])
+
+        # Validate the Value:
+        if len(tag['Value']) > 256:
+            raise TagValueTooBig(tag['Value'])
+
+        proper_tags[tag['Key']] = tag['Value']
+
+    return proper_tags
+
+
 class ConfigEmptyDictable(BaseModel):
     """Base class to make serialization easy. This assumes that the sub-class will NOT return 'None's in the JSON."""
 
+    def __init__(self, capitalize_start=False, capitalize_arn=True):
+        """Assists with the serialization of the config object
+        :param capitalize_start: For some Config services, the first letter is lowercase -- for others it's capital
+        :param capitalize_arn: For some Config services, the API expects 'ARN' and for others, it expects 'Arn'
+        """
+        self.capitalize_start = capitalize_start
+        self.capitalize_arn = capitalize_arn
+
     def to_dict(self):
         data = {}
         for item, value in self.__dict__.items():
             if value is not None:
                 if isinstance(value, ConfigEmptyDictable):
-                    data[snake_to_camels(item)] = value.to_dict()
+                    data[snake_to_camels(item, self.capitalize_start, self.capitalize_arn)] = value.to_dict()
                 else:
-                    data[snake_to_camels(item)] = value
+                    data[snake_to_camels(item, self.capitalize_start, self.capitalize_arn)] = value
+
+        # Cleanse the extra properties:
+        for prop in POP_STRINGS:
+            data.pop(prop, None)
 
         return data
 
@@ -50,8 +141,9 @@ class ConfigEmptyDictable(BaseModel):
 class ConfigRecorderStatus(ConfigEmptyDictable):
 
     def __init__(self, name):
-        self.name = name
+        super(ConfigRecorderStatus, self).__init__()
 
+        self.name = name
         self.recording = False
         self.last_start_time = None
         self.last_stop_time = None
@@ -75,12 +167,16 @@ class ConfigRecorderStatus(ConfigEmptyDictable):
 class ConfigDeliverySnapshotProperties(ConfigEmptyDictable):
 
     def __init__(self, delivery_frequency):
+        super(ConfigDeliverySnapshotProperties, self).__init__()
+
         self.delivery_frequency = delivery_frequency
 
 
 class ConfigDeliveryChannel(ConfigEmptyDictable):
 
     def __init__(self, name, s3_bucket_name, prefix=None, sns_arn=None, snapshot_properties=None):
+        super(ConfigDeliveryChannel, self).__init__()
+
         self.name = name
         self.s3_bucket_name = s3_bucket_name
         self.s3_key_prefix = prefix
@@ -91,6 +187,8 @@ class ConfigDeliveryChannel(ConfigEmptyDictable):
 class RecordingGroup(ConfigEmptyDictable):
 
     def __init__(self, all_supported=True, include_global_resource_types=False, resource_types=None):
+        super(RecordingGroup, self).__init__()
+
         self.all_supported = all_supported
         self.include_global_resource_types = include_global_resource_types
         self.resource_types = resource_types
@@ -99,6 +197,8 @@ class RecordingGroup(ConfigEmptyDictable):
 class ConfigRecorder(ConfigEmptyDictable):
 
     def __init__(self, role_arn, recording_group, name='default', status=None):
+        super(ConfigRecorder, self).__init__()
+
         self.name = name
         self.role_arn = role_arn
         self.recording_group = recording_group
@@ -109,18 +209,118 @@ class ConfigRecorder(ConfigEmptyDictable):
             self.status = status
 
 
+class AccountAggregatorSource(ConfigEmptyDictable):
+
+    def __init__(self, account_ids, aws_regions=None, all_aws_regions=None):
+        super(AccountAggregatorSource, self).__init__(capitalize_start=True)
+
+        # Can't have both the regions and all_regions flag present -- also can't have them both missing:
+        if aws_regions and all_aws_regions:
+            raise InvalidParameterValueException('Your configuration aggregator contains a list of regions and also specifies '
+                                                 'the use of all regions. You must choose one of these options.')
+
+        if not (aws_regions or all_aws_regions):
+            raise InvalidParameterValueException('Your request does not specify any regions. Select AWS Config-supported '
+                                                 'regions and try again.')
+
+        self.account_ids = account_ids
+        self.aws_regions = aws_regions
+
+        if not all_aws_regions:
+            all_aws_regions = False
+
+        self.all_aws_regions = all_aws_regions
+
+
+class OrganizationAggregationSource(ConfigEmptyDictable):
+
+    def __init__(self, role_arn, aws_regions=None, all_aws_regions=None):
+        super(OrganizationAggregationSource, self).__init__(capitalize_start=True, capitalize_arn=False)
+
+        # Can't have both the regions and all_regions flag present -- also can't have them both missing:
+        if aws_regions and all_aws_regions:
+            raise InvalidParameterValueException('Your configuration aggregator contains a list of regions and also specifies '
+                                                 'the use of all regions. You must choose one of these options.')
+
+        if not (aws_regions or all_aws_regions):
+            raise InvalidParameterValueException('Your request does not specify any regions. Select AWS Config-supported '
+                                                 'regions and try again.')
+
+        self.role_arn = role_arn
+        self.aws_regions = aws_regions
+
+        if not all_aws_regions:
+            all_aws_regions = False
+
+        self.all_aws_regions = all_aws_regions
+
+
+class ConfigAggregator(ConfigEmptyDictable):
+
+    def __init__(self, name, region, account_sources=None, org_source=None, tags=None):
+        super(ConfigAggregator, self).__init__(capitalize_start=True, capitalize_arn=False)
+
+        self.configuration_aggregator_name = name
+        self.configuration_aggregator_arn = 'arn:aws:config:{region}:{id}:config-aggregator/config-aggregator-{random}'.format(
+            region=region,
+            id=DEFAULT_ACCOUNT_ID,
+            random=random_string()
+        )
+        self.account_aggregation_sources = account_sources
+        self.organization_aggregation_source = org_source
+        self.creation_time = datetime2int(datetime.utcnow())
+        self.last_updated_time = datetime2int(datetime.utcnow())
+
+        # Tags are listed in the list_tags_for_resource API call ... not implementing yet -- please feel free to!
+        self.tags = tags or {}
+
+    # Override the to_dict so that we can format the tags properly...
+    def to_dict(self):
+        result = super(ConfigAggregator, self).to_dict()
+
+        # Override the account aggregation sources if present:
+        if self.account_aggregation_sources:
+            result['AccountAggregationSources'] = [a.to_dict() for a in self.account_aggregation_sources]
+
+        # Tags are listed in the list_tags_for_resource API call ... not implementing yet -- please feel free to!
+        # if self.tags:
+        #     result['Tags'] = [{'Key': key, 'Value': value} for key, value in self.tags.items()]
+
+        return result
+
+
+class ConfigAggregationAuthorization(ConfigEmptyDictable):
+
+    def __init__(self, current_region, authorized_account_id, authorized_aws_region, tags=None):
+        super(ConfigAggregationAuthorization, self).__init__(capitalize_start=True, capitalize_arn=False)
+
+        self.aggregation_authorization_arn = 'arn:aws:config:{region}:{id}:aggregation-authorization/' \
+                                             '{auth_account}/{auth_region}'.format(region=current_region,
+                                                                                   id=DEFAULT_ACCOUNT_ID,
+                                                                                   auth_account=authorized_account_id,
+                                                                                   auth_region=authorized_aws_region)
+        self.authorized_account_id = authorized_account_id
+        self.authorized_aws_region = authorized_aws_region
+        self.creation_time = datetime2int(datetime.utcnow())
+
+        # Tags are listed in the list_tags_for_resource API call ... not implementing yet -- please feel free to!
+        self.tags = tags or {}
+
+
 class ConfigBackend(BaseBackend):
 
     def __init__(self):
         self.recorders = {}
         self.delivery_channels = {}
+        self.config_aggregators = {}
+        self.aggregation_authorizations = {}
 
     @staticmethod
     def _validate_resource_types(resource_list):
         # Load the service file:
         resource_package = 'botocore'
         resource_path = '/'.join(('data', 'config', '2014-11-12', 'service-2.json'))
-        conifg_schema = json.loads(pkg_resources.resource_string(resource_package, resource_path))
+        config_schema = json.loads(pkg_resources.resource_string(resource_package, resource_path))
 
         # Verify that each entry exists in the supported list:
         bad_list = []
@@ -128,11 +328,11 @@ class ConfigBackend(BaseBackend):
             # For PY2:
             r_str = str(resource)
 
-            if r_str not in conifg_schema['shapes']['ResourceType']['enum']:
+            if r_str not in config_schema['shapes']['ResourceType']['enum']:
                 bad_list.append(r_str)
 
         if bad_list:
-            raise InvalidResourceTypeException(bad_list, conifg_schema['shapes']['ResourceType']['enum'])
+            raise InvalidResourceTypeException(bad_list, config_schema['shapes']['ResourceType']['enum'])
 
     @staticmethod
     def _validate_delivery_snapshot_properties(properties):
@@ -147,6 +347,158 @@ class ConfigBackend(BaseBackend):
             raise InvalidDeliveryFrequency(properties.get('deliveryFrequency', None),
                                            conifg_schema['shapes']['MaximumExecutionFrequency']['enum'])
 
+    def put_configuration_aggregator(self, config_aggregator, region):
+        # Validate the name:
+        if len(config_aggregator['ConfigurationAggregatorName']) > 256:
+            raise NameTooLongException(config_aggregator['ConfigurationAggregatorName'], 'configurationAggregatorName')
+
+        account_sources = None
+        org_source = None
+
+        # Tag validation:
+        tags = validate_tags(config_aggregator.get('Tags', []))
+
+        # Exception if both AccountAggregationSources and OrganizationAggregationSource are supplied:
+        if config_aggregator.get('AccountAggregationSources') and config_aggregator.get('OrganizationAggregationSource'):
+            raise InvalidParameterValueException('The configuration aggregator cannot be created because your request contains both the'
+                                                 ' AccountAggregationSource and the OrganizationAggregationSource. Include only '
+                                                 'one aggregation source and try again.')
+
+        # If neither are supplied:
+        if not config_aggregator.get('AccountAggregationSources') and not config_aggregator.get('OrganizationAggregationSource'):
+            raise InvalidParameterValueException('The configuration aggregator cannot be created because your request is missing either '
+                                                 'the AccountAggregationSource or the OrganizationAggregationSource. Include the '
+                                                 'appropriate aggregation source and try again.')
+
+        if config_aggregator.get('AccountAggregationSources'):
+            # Currently, only 1 account aggregation source can be set:
+            if len(config_aggregator['AccountAggregationSources']) > 1:
+                raise TooManyAccountSources(len(config_aggregator['AccountAggregationSources']))
+
+            account_sources = []
+            for a in config_aggregator['AccountAggregationSources']:
+                account_sources.append(AccountAggregatorSource(a['AccountIds'], aws_regions=a.get('AwsRegions'),
+                                                               all_aws_regions=a.get('AllAwsRegions')))
+
+        else:
+            org_source = OrganizationAggregationSource(config_aggregator['OrganizationAggregationSource']['RoleArn'],
+                                                       aws_regions=config_aggregator['OrganizationAggregationSource'].get('AwsRegions'),
+                                                       all_aws_regions=config_aggregator['OrganizationAggregationSource'].get(
+                                                           'AllAwsRegions'))
+
+        # Grab the existing one if it exists and update it:
+        if not self.config_aggregators.get(config_aggregator['ConfigurationAggregatorName']):
+            aggregator = ConfigAggregator(config_aggregator['ConfigurationAggregatorName'], region, account_sources=account_sources,
+                                          org_source=org_source, tags=tags)
+            self.config_aggregators[config_aggregator['ConfigurationAggregatorName']] = aggregator
+
+        else:
+            aggregator = self.config_aggregators[config_aggregator['ConfigurationAggregatorName']]
+            aggregator.tags = tags
+            aggregator.account_aggregation_sources = account_sources
+            aggregator.organization_aggregation_source = org_source
+            aggregator.last_updated_time = datetime2int(datetime.utcnow())
+
+        return aggregator.to_dict()
+
+    def describe_configuration_aggregators(self, names, token, limit):
+        limit = DEFAULT_PAGE_SIZE if not limit or limit < 0 else limit
+        agg_list = []
+        result = {'ConfigurationAggregators': []}
+
+        if names:
+            for name in names:
+                if not self.config_aggregators.get(name):
+                    raise NoSuchConfigurationAggregatorException(number=len(names))
+
+                agg_list.append(name)
+
+        else:
+            agg_list = list(self.config_aggregators.keys())
+
+        # Empty?
+        if not agg_list:
+            return result
+
+        # Sort by name:
+        sorted_aggregators = sorted(agg_list)
+
+        # Get the start:
+        if not token:
+            start = 0
+        else:
+            # Tokens for this moto feature are just the next names of the items in the list:
+            if not self.config_aggregators.get(token):
+                raise InvalidNextTokenException()
+
+            start = sorted_aggregators.index(token)
+
+        # Get the list of items to collect:
+        agg_list = sorted_aggregators[start:(start + limit)]
+        result['ConfigurationAggregators'] = [self.config_aggregators[agg].to_dict() for agg in agg_list]
+
+        if len(sorted_aggregators) > (start + limit):
+            result['NextToken'] = sorted_aggregators[start + limit]
+
+        return result
+
+    def delete_configuration_aggregator(self, config_aggregator):
+        if not self.config_aggregators.get(config_aggregator):
+            raise NoSuchConfigurationAggregatorException()
+
+        del self.config_aggregators[config_aggregator]
+
+    def put_aggregation_authorization(self, current_region, authorized_account, authorized_region, tags):
+        # Tag validation:
+        tags = validate_tags(tags or [])
+
+        # Does this already exist?
+        key = '{}/{}'.format(authorized_account, authorized_region)
+        agg_auth = self.aggregation_authorizations.get(key)
+        if not agg_auth:
+            agg_auth = ConfigAggregationAuthorization(current_region, authorized_account, authorized_region, tags=tags)
+            self.aggregation_authorizations['{}/{}'.format(authorized_account, authorized_region)] = agg_auth
+        else:
+            # Only update the tags:
+            agg_auth.tags = tags
+
+        return agg_auth.to_dict()
+
+    def describe_aggregation_authorizations(self, token, limit):
+        limit = DEFAULT_PAGE_SIZE if not limit or limit < 0 else limit
+        result = {'AggregationAuthorizations': []}
+
+        if not self.aggregation_authorizations:
+            return result
+
+        # Sort by name:
+        sorted_authorizations = sorted(self.aggregation_authorizations.keys())
+
+        # Get the start:
+        if not token:
+            start = 0
+        else:
+            # Tokens for this moto feature are just the next names of the items in the list:
+            if not self.aggregation_authorizations.get(token):
+                raise InvalidNextTokenException()
+
+            start = sorted_authorizations.index(token)
+
+        # Get the list of items to collect:
+        auth_list = sorted_authorizations[start:(start + limit)]
+        result['AggregationAuthorizations'] = [self.aggregation_authorizations[auth].to_dict() for auth in auth_list]
+
+        if len(sorted_authorizations) > (start + limit):
+            result['NextToken'] = sorted_authorizations[start + limit]
+
+        return result
+
+    def delete_aggregation_authorization(self, authorized_account, authorized_region):
+        # This will always return a 200 -- regardless if there is or isn't an existing
+        # aggregation authorization.
+        key = '{}/{}'.format(authorized_account, authorized_region)
+        self.aggregation_authorizations.pop(key, None)
+
     def put_configuration_recorder(self, config_recorder):
         # Validate the name:
         if not config_recorder.get('name'):

moto/config/responses.py

@@ -13,6 +13,39 @@ class ConfigResponse(BaseResponse):
         self.config_backend.put_configuration_recorder(self._get_param('ConfigurationRecorder'))
         return ""
 
+    def put_configuration_aggregator(self):
+        aggregator = self.config_backend.put_configuration_aggregator(json.loads(self.body), self.region)
+        schema = {'ConfigurationAggregator': aggregator}
+        return json.dumps(schema)
+
+    def describe_configuration_aggregators(self):
+        aggregators = self.config_backend.describe_configuration_aggregators(self._get_param('ConfigurationAggregatorNames'),
+                                                                             self._get_param('NextToken'),
+                                                                             self._get_param('Limit'))
+        return json.dumps(aggregators)
+
+    def delete_configuration_aggregator(self):
+        self.config_backend.delete_configuration_aggregator(self._get_param('ConfigurationAggregatorName'))
+        return ""
+
+    def put_aggregation_authorization(self):
+        agg_auth = self.config_backend.put_aggregation_authorization(self.region,
+                                                                     self._get_param('AuthorizedAccountId'),
+                                                                     self._get_param('AuthorizedAwsRegion'),
+                                                                     self._get_param('Tags'))
+        schema = {'AggregationAuthorization': agg_auth}
+        return json.dumps(schema)
+
+    def describe_aggregation_authorizations(self):
+        authorizations = self.config_backend.describe_aggregation_authorizations(self._get_param('NextToken'), self._get_param('Limit'))
+
+        return json.dumps(authorizations)
+
+    def delete_aggregation_authorization(self):
+        self.config_backend.delete_aggregation_authorization(self._get_param('AuthorizedAccountId'), self._get_param('AuthorizedAwsRegion'))
+
+        return ""
+
     def describe_configuration_recorders(self):
         recorders = self.config_backend.describe_configuration_recorders(self._get_param('ConfigurationRecorderNames'))
         schema = {'ConfigurationRecorders': recorders}

moto/core/models.py

@@ -12,6 +12,7 @@ from collections import defaultdict
 from botocore.handlers import BUILTIN_HANDLERS
 from botocore.awsrequest import AWSResponse
 
+import mock
 from moto import settings
 import responses
 from moto.packages.httpretty import HTTPretty
@@ -22,11 +23,6 @@ from .utils import (
 )
 
 
-# "Mock" the AWS credentials as they can't be mocked in Botocore currently
-os.environ.setdefault("AWS_ACCESS_KEY_ID", "foobar_key")
-os.environ.setdefault("AWS_SECRET_ACCESS_KEY", "foobar_secret")
-
-
 class BaseMockAWS(object):
     nested_count = 0
 
@@ -42,6 +38,10 @@ class BaseMockAWS(object):
         self.backends_for_urls.update(self.backends)
         self.backends_for_urls.update(default_backends)
 
+        # "Mock" the AWS credentials as they can't be mocked in Botocore currently
+        FAKE_KEYS = {"AWS_ACCESS_KEY_ID": "foobar_key", "AWS_SECRET_ACCESS_KEY": "foobar_secret"}
+        self.env_variables_mocks = mock.patch.dict(os.environ, FAKE_KEYS)
+
         if self.__class__.nested_count == 0:
             self.reset()
 
@@ -52,11 +52,14 @@ class BaseMockAWS(object):
 
     def __enter__(self):
         self.start()
+        return self
 
     def __exit__(self, *args):
         self.stop()
 
     def start(self, reset=True):
+        self.env_variables_mocks.start()
+
         self.__class__.nested_count += 1
         if reset:
             for backend in self.backends.values():
@@ -65,6 +68,7 @@ class BaseMockAWS(object):
         self.enable_patching()
 
     def stop(self):
+        self.env_variables_mocks.stop()
         self.__class__.nested_count -= 1
 
         if self.__class__.nested_count < 0:
@@ -465,10 +469,14 @@ class BaseModel(object):
 
 class BaseBackend(object):
 
-    def reset(self):
+    def _reset_model_refs(self):
+        # Remove all references to the models stored
         for service, models in model_data.items():
             for model_name, model in models.items():
                 model.instances = []
+
+    def reset(self):
+        self._reset_model_refs()
         self.__dict__ = {}
         self.__init__()
 

moto/dynamodb2/comparisons.py

@@ -1004,8 +1004,7 @@ class OpOr(Op):
 
     def expr(self, item):
         lhs = self.lhs.expr(item)
-        rhs = self.rhs.expr(item)
-        return lhs or rhs
+        return lhs or self.rhs.expr(item)
 
 
 class Func(object):

moto/dynamodb2/models.py

@@ -298,7 +298,9 @@ class Item(BaseModel):
             new_value = list(update_action['Value'].values())[0]
             if action == 'PUT':
                 # TODO deal with other types
-                if isinstance(new_value, list) or isinstance(new_value, set):
+                if isinstance(new_value, list):
+                    self.attrs[attribute_name] = DynamoType({"L": new_value})
+                elif isinstance(new_value, set):
                     self.attrs[attribute_name] = DynamoType({"SS": new_value})
                 elif isinstance(new_value, dict):
                     self.attrs[attribute_name] = DynamoType({"M": new_value})

moto/dynamodb2/responses.py

@@ -600,7 +600,7 @@ class DynamoHandler(BaseResponse):
         # E.g. `a = b + c` -> `a=b+c`
         if update_expression:
             update_expression = re.sub(
-                '\s*([=\+-])\s*', '\\1', update_expression)
+                r'\s*([=\+-])\s*', '\\1', update_expression)
 
         try:
             item = self.dynamodb_backend.update_item(

moto/ec2/models.py

@@ -142,6 +142,8 @@ AMIS = json.load(
          __name__, 'resources/amis.json'), 'r')
 )
 
+OWNER_ID = "111122223333"
+
 
 def utc_date_and_time():
     return datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S.000Z')
@@ -201,7 +203,7 @@ class TaggedEC2Resource(BaseModel):
 
 class NetworkInterface(TaggedEC2Resource):
     def __init__(self, ec2_backend, subnet, private_ip_address, device_index=0,
-                 public_ip_auto_assign=True, group_ids=None):
+                 public_ip_auto_assign=True, group_ids=None, description=None):
         self.ec2_backend = ec2_backend
         self.id = random_eni_id()
         self.device_index = device_index
@@ -209,6 +211,7 @@ class NetworkInterface(TaggedEC2Resource):
         self.subnet = subnet
         self.instance = None
         self.attachment_id = None
+        self.description = description
 
         self.public_ip = None
         self.public_ip_auto_assign = public_ip_auto_assign
@@ -246,11 +249,13 @@ class NetworkInterface(TaggedEC2Resource):
             subnet = None
 
         private_ip_address = properties.get('PrivateIpAddress', None)
+        description = properties.get('Description', None)
 
         network_interface = ec2_backend.create_network_interface(
             subnet,
             private_ip_address,
-            group_ids=security_group_ids
+            group_ids=security_group_ids,
+            description=description
         )
         return network_interface
 
@@ -298,6 +303,8 @@ class NetworkInterface(TaggedEC2Resource):
             return [group.id for group in self._group_set]
         elif filter_name == 'availability-zone':
             return self.subnet.availability_zone
+        elif filter_name == 'description':
+            return self.description
         else:
             return super(NetworkInterface, self).get_filter_value(
                 filter_name, 'DescribeNetworkInterfaces')
@@ -308,9 +315,9 @@ class NetworkInterfaceBackend(object):
         self.enis = {}
         super(NetworkInterfaceBackend, self).__init__()
 
-    def create_network_interface(self, subnet, private_ip_address, group_ids=None, **kwargs):
+    def create_network_interface(self, subnet, private_ip_address, group_ids=None, description=None, **kwargs):
         eni = NetworkInterface(
-            self, subnet, private_ip_address, group_ids=group_ids, **kwargs)
+            self, subnet, private_ip_address, group_ids=group_ids, description=description, **kwargs)
         self.enis[eni.id] = eni
         return eni
 
@@ -343,6 +350,12 @@ class NetworkInterfaceBackend(object):
                             if group.id in _filter_value:
                                 enis.append(eni)
                                 break
+                elif _filter == 'private-ip-address:':
+                    enis = [eni for eni in enis if eni.private_ip_address in _filter_value]
+                elif _filter == 'subnet-id':
+                    enis = [eni for eni in enis if eni.subnet.id in _filter_value]
+                elif _filter == 'description':
+                    enis = [eni for eni in enis if eni.description in _filter_value]
                 else:
                     self.raise_not_implemented_error(
                         "The filter '{0}' for DescribeNetworkInterfaces".format(_filter))
@@ -413,10 +426,10 @@ class Instance(TaggedEC2Resource, BotoInstance):
         self.instance_initiated_shutdown_behavior = kwargs.get("instance_initiated_shutdown_behavior", "stop")
         self.sriov_net_support = "simple"
         self._spot_fleet_id = kwargs.get("spot_fleet_id", None)
-        associate_public_ip = kwargs.get("associate_public_ip", False)
+        self.associate_public_ip = kwargs.get("associate_public_ip", False)
         if in_ec2_classic:
             # If we are in EC2-Classic, autoassign a public IP
-            associate_public_ip = True
+            self.associate_public_ip = True
 
         amis = self.ec2_backend.describe_images(filters={'image-id': image_id})
         ami = amis[0] if amis else None
@@ -447,9 +460,9 @@ class Instance(TaggedEC2Resource, BotoInstance):
             self.vpc_id = subnet.vpc_id
             self._placement.zone = subnet.availability_zone
 
-            if associate_public_ip is None:
+            if self.associate_public_ip is None:
                 # Mapping public ip hasnt been explicitly enabled or disabled
-                associate_public_ip = subnet.map_public_ip_on_launch == 'true'
+                self.associate_public_ip = subnet.map_public_ip_on_launch == 'true'
         elif placement:
             self._placement.zone = placement
         else:
@@ -461,7 +474,7 @@ class Instance(TaggedEC2Resource, BotoInstance):
         self.prep_nics(
             kwargs.get("nics", {}),
             private_ip=kwargs.get("private_ip"),
-            associate_public_ip=associate_public_ip
+            associate_public_ip=self.associate_public_ip
         )
 
     def __del__(self):
@@ -1076,7 +1089,7 @@ class TagBackend(object):
 
 class Ami(TaggedEC2Resource):
     def __init__(self, ec2_backend, ami_id, instance=None, source_ami=None,
-                 name=None, description=None, owner_id=111122223333,
+                 name=None, description=None, owner_id=OWNER_ID,
                  public=False, virtualization_type=None, architecture=None,
                  state='available', creation_date=None, platform=None,
                  image_type='machine', image_location=None, hypervisor=None,
@@ -1189,7 +1202,7 @@ class AmiBackend(object):
 
         ami = Ami(self, ami_id, instance=instance, source_ami=None,
                   name=name, description=description,
-                  owner_id=context.get_current_user() if context else '111122223333')
+                  owner_id=context.get_current_user() if context else OWNER_ID)
         self.amis[ami_id] = ami
         return ami
 
@@ -1457,7 +1470,7 @@ class SecurityGroup(TaggedEC2Resource):
         self.egress_rules = [SecurityRule(-1, None, None, ['0.0.0.0/0'], [])]
         self.enis = {}
         self.vpc_id = vpc_id
-        self.owner_id = "123456789012"
+        self.owner_id = OWNER_ID
 
     @classmethod
     def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name):
@@ -1978,7 +1991,7 @@ class Volume(TaggedEC2Resource):
 
 
 class Snapshot(TaggedEC2Resource):
-    def __init__(self, ec2_backend, snapshot_id, volume, description, encrypted=False, owner_id='123456789012'):
+    def __init__(self, ec2_backend, snapshot_id, volume, description, encrypted=False, owner_id=OWNER_ID):
         self.id = snapshot_id
         self.volume = volume
         self.description = description
@@ -2480,7 +2493,7 @@ class VPCPeeringConnectionBackend(object):
 
 class Subnet(TaggedEC2Resource):
     def __init__(self, ec2_backend, subnet_id, vpc_id, cidr_block, availability_zone, default_for_az,
-                 map_public_ip_on_launch, owner_id=111122223333, assign_ipv6_address_on_creation=False):
+                 map_public_ip_on_launch, owner_id=OWNER_ID, assign_ipv6_address_on_creation=False):
         self.ec2_backend = ec2_backend
         self.id = subnet_id
         self.vpc_id = vpc_id
@@ -2646,7 +2659,7 @@ class SubnetBackend(object):
             raise InvalidAvailabilityZoneError(availability_zone, ", ".join([zone.name for zones in RegionsAndZonesBackend.zones.values() for zone in zones]))
         subnet = Subnet(self, subnet_id, vpc_id, cidr_block, availability_zone_data,
                         default_for_az, map_public_ip_on_launch,
-                        owner_id=context.get_current_user() if context else '111122223333', assign_ipv6_address_on_creation=False)
+                        owner_id=context.get_current_user() if context else OWNER_ID, assign_ipv6_address_on_creation=False)
 
         # AWS associates a new subnet with the default Network ACL
         self.associate_default_network_acl_with_subnet(subnet_id, vpc_id)

moto/ec2/responses/elastic_network_interfaces.py

@@ -10,9 +10,10 @@ class ElasticNetworkInterfaces(BaseResponse):
         private_ip_address = self._get_param('PrivateIpAddress')
         groups = self._get_multi_param('SecurityGroupId')
         subnet = self.ec2_backend.get_subnet(subnet_id)
+        description = self._get_param('Description')
         if self.is_not_dryrun('CreateNetworkInterface'):
             eni = self.ec2_backend.create_network_interface(
-                subnet, private_ip_address, groups)
+                subnet, private_ip_address, groups, description)
             template = self.response_template(
                 CREATE_NETWORK_INTERFACE_RESPONSE)
             return template.render(eni=eni)
@@ -78,7 +79,11 @@ CREATE_NETWORK_INTERFACE_RESPONSE = """
         <subnetId>{{ eni.subnet.id }}</subnetId>
         <vpcId>{{ eni.subnet.vpc_id }}</vpcId>
         <availabilityZone>us-west-2a</availabilityZone>
+        {% if eni.description %}
+        <description>{{ eni.description }}</description>
+        {% else %}
         <description/>
+        {% endif %}
         <ownerId>498654062920</ownerId>
         <requesterManaged>false</requesterManaged>
         <status>pending</status>
@@ -121,7 +126,7 @@ DESCRIBE_NETWORK_INTERFACES_RESPONSE = """<DescribeNetworkInterfacesResponse xml
            <subnetId>{{ eni.subnet.id }}</subnetId>
            <vpcId>{{ eni.subnet.vpc_id }}</vpcId>
            <availabilityZone>us-west-2a</availabilityZone>
-           <description>Primary network interface</description>
+           <description>{{ eni.description }}</description>
            <ownerId>190610284047</ownerId>
            <requesterManaged>false</requesterManaged>
            {% if eni.attachment_id %}

moto/ec2/responses/instances.py

@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 from boto.ec2.instancetype import InstanceType
+
+from moto.autoscaling import autoscaling_backends
 from moto.core.responses import BaseResponse
 from moto.core.utils import camelcase_to_underscores
 from moto.ec2.utils import filters_from_querystring, \
@@ -65,6 +67,7 @@ class InstanceResponse(BaseResponse):
         instance_ids = self._get_multi_param('InstanceId')
         if self.is_not_dryrun('TerminateInstance'):
             instances = self.ec2_backend.terminate_instances(instance_ids)
+            autoscaling_backends[self.region].notify_terminate_instances(instance_ids)
             template = self.response_template(EC2_TERMINATE_INSTANCES)
             return template.render(instances=instances)
 

moto/ecs/exceptions.py

@@ -1,5 +1,5 @@
 from __future__ import unicode_literals
-from moto.core.exceptions import RESTError
+from moto.core.exceptions import RESTError, JsonRESTError
 
 
 class ServiceNotFoundException(RESTError):
@@ -11,3 +11,13 @@ class ServiceNotFoundException(RESTError):
             message="The service {0} does not exist".format(service_name),
             template='error_json',
         )
+
+
+class TaskDefinitionNotFoundException(JsonRESTError):
+    code = 400
+
+    def __init__(self):
+        super(TaskDefinitionNotFoundException, self).__init__(
+            error_type="ClientException",
+            message="The specified task definition does not exist.",
+        )

moto/ecs/models.py

@@ -1,4 +1,5 @@
 from __future__ import unicode_literals
+import re
 import uuid
 from datetime import datetime
 from random import random, randint
@@ -7,10 +8,14 @@ import boto3
 import pytz
 from moto.core.exceptions import JsonRESTError
 from moto.core import BaseBackend, BaseModel
+from moto.core.utils import unix_time
 from moto.ec2 import ec2_backends
 from copy import copy
 
-from .exceptions import ServiceNotFoundException
+from .exceptions import (
+    ServiceNotFoundException,
+    TaskDefinitionNotFoundException
+)
 
 
 class BaseObject(BaseModel):
@@ -103,12 +108,13 @@ class Cluster(BaseObject):
 
 class TaskDefinition(BaseObject):
 
-    def __init__(self, family, revision, container_definitions, volumes=None):
+    def __init__(self, family, revision, container_definitions, volumes=None, tags=None):
         self.family = family
         self.revision = revision
         self.arn = 'arn:aws:ecs:us-east-1:012345678910:task-definition/{0}:{1}'.format(
             family, revision)
         self.container_definitions = container_definitions
+        self.tags = tags if tags is not None else []
         if volumes is None:
             self.volumes = []
         else:
@@ -119,6 +125,7 @@ class TaskDefinition(BaseObject):
         response_object = self.gen_response_object()
         response_object['taskDefinitionArn'] = response_object['arn']
         del response_object['arn']
+        del response_object['tags']
         return response_object
 
     @property
@@ -225,9 +232,9 @@ class Service(BaseObject):
 
         for deployment in response_object['deployments']:
             if isinstance(deployment['createdAt'], datetime):
-                deployment['createdAt'] = deployment['createdAt'].isoformat()
+                deployment['createdAt'] = unix_time(deployment['createdAt'].replace(tzinfo=None))
             if isinstance(deployment['updatedAt'], datetime):
-                deployment['updatedAt'] = deployment['updatedAt'].isoformat()
+                deployment['updatedAt'] = unix_time(deployment['updatedAt'].replace(tzinfo=None))
 
         return response_object
 
@@ -422,11 +429,9 @@ class EC2ContainerServiceBackend(BaseBackend):
             revision = int(revision)
         else:
             family = task_definition_name
-            revision = len(self.task_definitions.get(family, []))
+            revision = self._get_last_task_definition_revision_id(family)
 
-        if family in self.task_definitions and 0 < revision <= len(self.task_definitions[family]):
-            return self.task_definitions[family][revision - 1]
-        elif family in self.task_definitions and revision == -1:
+        if family in self.task_definitions and revision in self.task_definitions[family]:
             return self.task_definitions[family][revision]
         else:
             raise Exception(
@@ -466,15 +471,16 @@ class EC2ContainerServiceBackend(BaseBackend):
         else:
             raise Exception("{0} is not a cluster".format(cluster_name))
 
-    def register_task_definition(self, family, container_definitions, volumes):
+    def register_task_definition(self, family, container_definitions, volumes, tags=None):
         if family in self.task_definitions:
-            revision = len(self.task_definitions[family]) + 1
+            last_id = self._get_last_task_definition_revision_id(family)
+            revision = (last_id or 0) + 1
         else:
-            self.task_definitions[family] = []
+            self.task_definitions[family] = {}
             revision = 1
         task_definition = TaskDefinition(
-            family, revision, container_definitions, volumes)
-        self.task_definitions[family].append(task_definition)
+            family, revision, container_definitions, volumes, tags)
+        self.task_definitions[family][revision] = task_definition
 
         return task_definition
 
@@ -484,16 +490,18 @@ class EC2ContainerServiceBackend(BaseBackend):
         """
         task_arns = []
         for task_definition_list in self.task_definitions.values():
-            task_arns.extend(
-                [task_definition.arn for task_definition in task_definition_list])
+            task_arns.extend([
+                task_definition.arn
+                for task_definition in task_definition_list.values()
+            ])
         return task_arns
 
     def deregister_task_definition(self, task_definition_str):
         task_definition_name = task_definition_str.split('/')[-1]
         family, revision = task_definition_name.split(':')
         revision = int(revision)
-        if family in self.task_definitions and 0 < revision <= len(self.task_definitions[family]):
-            return self.task_definitions[family].pop(revision - 1)
+        if family in self.task_definitions and revision in self.task_definitions[family]:
+            return self.task_definitions[family].pop(revision)
         else:
             raise Exception(
                 "{0} is not a task_definition".format(task_definition_name))
@@ -950,6 +958,29 @@ class EC2ContainerServiceBackend(BaseBackend):
 
             yield task_fam
 
+    def list_tags_for_resource(self, resource_arn):
+        """Currently only implemented for task definitions"""
+        match = re.match(
+            "^arn:aws:ecs:(?P<region>[^:]+):(?P<account_id>[^:]+):(?P<service>[^:]+)/(?P<id>.*)$",
+            resource_arn)
+        if not match:
+            raise JsonRESTError('InvalidParameterException', 'The ARN provided is invalid.')
+
+        service = match.group("service")
+        if service == "task-definition":
+            for task_definition in self.task_definitions.values():
+                for revision in task_definition.values():
+                    if revision.arn == resource_arn:
+                        return revision.tags
+            else:
+                raise TaskDefinitionNotFoundException()
+        raise NotImplementedError()
+
+    def _get_last_task_definition_revision_id(self, family):
+        definitions = self.task_definitions.get(family, {})
+        if definitions:
+            return max(definitions.keys())
+
 
 available_regions = boto3.session.Session().get_available_regions("ecs")
 ecs_backends = {region: EC2ContainerServiceBackend(region) for region in available_regions}

moto/ecs/responses.py

@@ -62,8 +62,9 @@ class EC2ContainerServiceResponse(BaseResponse):
         family = self._get_param('family')
         container_definitions = self._get_param('containerDefinitions')
         volumes = self._get_param('volumes')
+        tags = self._get_param('tags')
         task_definition = self.ecs_backend.register_task_definition(
-            family, container_definitions, volumes)
+            family, container_definitions, volumes, tags)
         return json.dumps({
             'taskDefinition': task_definition.response_object
         })
@@ -313,3 +314,8 @@ class EC2ContainerServiceResponse(BaseResponse):
         results = self.ecs_backend.list_task_definition_families(family_prefix, status, max_results, next_token)
 
         return json.dumps({'families': list(results)})
+
+    def list_tags_for_resource(self):
+        resource_arn = self._get_param('resourceArn')
+        tags = self.ecs_backend.list_tags_for_resource(resource_arn)
+        return json.dumps({'tags': tags})

moto/elbv2/models.py

@@ -2,9 +2,11 @@ from __future__ import unicode_literals
 
 import datetime
 import re
+from jinja2 import Template
 from moto.compat import OrderedDict
 from moto.core.exceptions import RESTError
 from moto.core import BaseBackend, BaseModel
+from moto.core.utils import camelcase_to_underscores
 from moto.ec2.models import ec2_backends
 from moto.acm.models import acm_backends
 from .utils import make_arn_for_target_group
@@ -35,12 +37,13 @@ from .exceptions import (
 
 class FakeHealthStatus(BaseModel):
 
-    def __init__(self, instance_id, port, health_port, status, reason=None):
+    def __init__(self, instance_id, port, health_port, status, reason=None, description=None):
         self.instance_id = instance_id
         self.port = port
         self.health_port = health_port
         self.status = status
         self.reason = reason
+        self.description = description
 
 
 class FakeTargetGroup(BaseModel):
@@ -69,7 +72,7 @@ class FakeTargetGroup(BaseModel):
         self.protocol = protocol
         self.port = port
         self.healthcheck_protocol = healthcheck_protocol or 'HTTP'
-        self.healthcheck_port = healthcheck_port or 'traffic-port'
+        self.healthcheck_port = healthcheck_port or str(self.port)
         self.healthcheck_path = healthcheck_path or '/'
         self.healthcheck_interval_seconds = healthcheck_interval_seconds or 30
         self.healthcheck_timeout_seconds = healthcheck_timeout_seconds or 5
@@ -112,10 +115,14 @@ class FakeTargetGroup(BaseModel):
             raise TooManyTagsError()
         self.tags[key] = value
 
-    def health_for(self, target):
+    def health_for(self, target, ec2_backend):
         t = self.targets.get(target['id'])
         if t is None:
             raise InvalidTargetError()
+        if t['id'].startswith("i-"):  # EC2 instance ID
+            instance = ec2_backend.get_instance_by_id(t['id'])
+            if instance.state == "stopped":
+                return FakeHealthStatus(t['id'], t['port'], self.healthcheck_port, 'unused', 'Target.InvalidState', 'Target is in the stopped state')
         return FakeHealthStatus(t['id'], t['port'], self.healthcheck_port, 'healthy')
 
     @classmethod
@@ -208,13 +215,12 @@ class FakeListener(BaseModel):
                 action_type = action['Type']
                 if action_type == 'forward':
                     default_actions.append({'type': action_type, 'target_group_arn': action['TargetGroupArn']})
-                elif action_type == 'redirect':
-                    redirect_action = {'type': action_type, }
-                    for redirect_config_key, redirect_config_value in action['RedirectConfig'].items():
+                elif action_type in ['redirect', 'authenticate-cognito']:
+                    redirect_action = {'type': action_type}
+                    key = 'RedirectConfig' if action_type == 'redirect' else 'AuthenticateCognitoConfig'
+                    for redirect_config_key, redirect_config_value in action[key].items():
                         # need to match the output of _get_list_prefix
-                        if redirect_config_key == 'StatusCode':
-                            redirect_config_key = 'status_code'
-                        redirect_action['redirect_config._' + redirect_config_key.lower()] = redirect_config_value
+                        redirect_action[camelcase_to_underscores(key) + '._' + camelcase_to_underscores(redirect_config_key)] = redirect_config_value
                     default_actions.append(redirect_action)
                 else:
                     raise InvalidActionTypeError(action_type, i + 1)
@@ -226,6 +232,32 @@ class FakeListener(BaseModel):
         return listener
 
 
+class FakeAction(BaseModel):
+    def __init__(self, data):
+        self.data = data
+        self.type = data.get("type")
+
+    def to_xml(self):
+        template = Template("""<Type>{{ action.type }}</Type>
+            {% if action.type == "forward" %}
+            <TargetGroupArn>{{ action.data["target_group_arn"] }}</TargetGroupArn>
+            {% elif action.type == "redirect" %}
+            <RedirectConfig>
+                <Protocol>{{ action.data["redirect_config._protocol"] }}</Protocol>
+                <Port>{{ action.data["redirect_config._port"] }}</Port>
+                <StatusCode>{{ action.data["redirect_config._status_code"] }}</StatusCode>
+            </RedirectConfig>
+            {% elif action.type == "authenticate-cognito" %}
+            <AuthenticateCognitoConfig>
+                <UserPoolArn>{{ action.data["authenticate_cognito_config._user_pool_arn"] }}</UserPoolArn>
+                <UserPoolClientId>{{ action.data["authenticate_cognito_config._user_pool_client_id"] }}</UserPoolClientId>
+                <UserPoolDomain>{{ action.data["authenticate_cognito_config._user_pool_domain"] }}</UserPoolDomain>
+            </AuthenticateCognitoConfig>
+            {% endif %}
+            """)
+        return template.render(action=self)
+
+
 class FakeRule(BaseModel):
 
     def __init__(self, listener_arn, conditions, priority, actions, is_default):
@@ -397,6 +429,7 @@ class ELBv2Backend(BaseBackend):
         return new_load_balancer
 
     def create_rule(self, listener_arn, conditions, priority, actions):
+        actions = [FakeAction(action) for action in actions]
         listeners = self.describe_listeners(None, [listener_arn])
         if not listeners:
             raise ListenerNotFoundError()
@@ -424,20 +457,7 @@ class ELBv2Backend(BaseBackend):
             if rule.priority == priority:
                 raise PriorityInUseError()
 
-        # validate Actions
-        target_group_arns = [target_group.arn for target_group in self.target_groups.values()]
-        for i, action in enumerate(actions):
-            index = i + 1
-            action_type = action['type']
-            if action_type == 'forward':
-                action_target_group_arn = action['target_group_arn']
-                if action_target_group_arn not in target_group_arns:
-                    raise ActionTargetGroupNotFoundError(action_target_group_arn)
-            elif action_type == 'redirect':
-                # nothing to do
-                pass
-            else:
-                raise InvalidActionTypeError(action_type, index)
+        self._validate_actions(actions)
 
         # TODO: check for error 'TooManyRegistrationsForTargetId'
         # TODO: check for error 'TooManyRules'
@@ -447,6 +467,21 @@ class ELBv2Backend(BaseBackend):
         listener.register(rule)
         return [rule]
 
+    def _validate_actions(self, actions):
+        # validate Actions
+        target_group_arns = [target_group.arn for target_group in self.target_groups.values()]
+        for i, action in enumerate(actions):
+            index = i + 1
+            action_type = action.type
+            if action_type == 'forward':
+                action_target_group_arn = action.data['target_group_arn']
+                if action_target_group_arn not in target_group_arns:
+                    raise ActionTargetGroupNotFoundError(action_target_group_arn)
+            elif action_type in ['redirect', 'authenticate-cognito']:
+                pass
+            else:
+                raise InvalidActionTypeError(action_type, index)
+
     def create_target_group(self, name, **kwargs):
         if len(name) > 32:
             raise InvalidTargetGroupNameError(
@@ -490,26 +525,22 @@ class ELBv2Backend(BaseBackend):
         return target_group
 
     def create_listener(self, load_balancer_arn, protocol, port, ssl_policy, certificate, default_actions):
+        default_actions = [FakeAction(action) for action in default_actions]
         balancer = self.load_balancers.get(load_balancer_arn)
         if balancer is None:
             raise LoadBalancerNotFoundError()
         if port in balancer.listeners:
             raise DuplicateListenerError()
 
+        self._validate_actions(default_actions)
+
         arn = load_balancer_arn.replace(':loadbalancer/', ':listener/') + "/%s%s" % (port, id(self))
         listener = FakeListener(load_balancer_arn, arn, protocol, port, ssl_policy, certificate, default_actions)
         balancer.listeners[listener.arn] = listener
-        for i, action in enumerate(default_actions):
-            action_type = action['type']
-            if action_type == 'forward':
-                if action['target_group_arn'] in self.target_groups.keys():
-                    target_group = self.target_groups[action['target_group_arn']]
-                    target_group.load_balancer_arns.append(load_balancer_arn)
-            elif action_type == 'redirect':
-                # nothing to do
-                pass
-            else:
-                raise InvalidActionTypeError(action_type, i + 1)
+        for action in default_actions:
+            if action.type == 'forward':
+                target_group = self.target_groups[action.data['target_group_arn']]
+                target_group.load_balancer_arns.append(load_balancer_arn)
 
         return listener
 
@@ -643,6 +674,7 @@ class ELBv2Backend(BaseBackend):
         raise ListenerNotFoundError()
 
     def modify_rule(self, rule_arn, conditions, actions):
+        actions = [FakeAction(action) for action in actions]
         # if conditions or actions is empty list, do not update the attributes
         if not conditions and not actions:
             raise InvalidModifyRuleArgumentsError()
@@ -668,20 +700,7 @@ class ELBv2Backend(BaseBackend):
                 # TODO: check pattern of value for 'path-pattern'
 
         # validate Actions
-        target_group_arns = [target_group.arn for target_group in self.target_groups.values()]
-        if actions:
-            for i, action in enumerate(actions):
-                index = i + 1
-                action_type = action['type']
-                if action_type == 'forward':
-                    action_target_group_arn = action['target_group_arn']
-                    if action_target_group_arn not in target_group_arns:
-                        raise ActionTargetGroupNotFoundError(action_target_group_arn)
-                elif action_type == 'redirect':
-                    # nothing to do
-                    pass
-                else:
-                    raise InvalidActionTypeError(action_type, index)
+        self._validate_actions(actions)
 
         # TODO: check for error 'TooManyRegistrationsForTargetId'
         # TODO: check for error 'TooManyRules'
@@ -712,7 +731,7 @@ class ELBv2Backend(BaseBackend):
 
         if not targets:
             targets = target_group.targets.values()
-        return [target_group.health_for(target) for target in targets]
+        return [target_group.health_for(target, self.ec2_backend) for target in targets]
 
     def set_rule_priorities(self, rule_priorities):
         # validate
@@ -846,6 +865,7 @@ class ELBv2Backend(BaseBackend):
         return target_group
 
     def modify_listener(self, arn, port=None, protocol=None, ssl_policy=None, certificates=None, default_actions=None):
+        default_actions = [FakeAction(action) for action in default_actions]
         for load_balancer in self.load_balancers.values():
             if arn in load_balancer.listeners:
                 break
@@ -912,7 +932,7 @@ class ELBv2Backend(BaseBackend):
             for listener in load_balancer.listeners.values():
                 for rule in listener.rules:
                     for action in rule.actions:
-                        if action.get('target_group_arn') == target_group_arn:
+                        if action.data.get('target_group_arn') == target_group_arn:
                             return True
         return False
 

moto/elbv2/responses.py

@@ -775,16 +775,7 @@ CREATE_LISTENER_TEMPLATE = """<CreateListenerResponse xmlns="http://elasticloadb
         <DefaultActions>
           {% for action in listener.default_actions %}
           <member>
-            <Type>{{ action.type }}</Type>
-            {% if action["type"] == "forward" %}
-            <TargetGroupArn>{{ action["target_group_arn"] }}</TargetGroupArn>
-            {% elif action["type"] == "redirect" %}
-            <RedirectConfig>
-                <Protocol>{{ action["redirect_config._protocol"] }}</Protocol>
-                <Port>{{ action["redirect_config._port"] }}</Port>
-                <StatusCode>{{ action["redirect_config._status_code"] }}</StatusCode>
-            </RedirectConfig>
-            {% endif %}
+            {{ action.to_xml() }}
           </member>
           {% endfor %}
         </DefaultActions>
@@ -888,16 +879,7 @@ DESCRIBE_RULES_TEMPLATE = """<DescribeRulesResponse xmlns="http://elasticloadbal
         <Actions>
           {% for action in rule.actions %}
           <member>
-            <Type>{{ action["type"] }}</Type>
-            {% if action["type"] == "forward" %}
-            <TargetGroupArn>{{ action["target_group_arn"] }}</TargetGroupArn>
-            {% elif action["type"] == "redirect" %}
-            <RedirectConfig>
-                <Protocol>{{ action["redirect_config._protocol"] }}</Protocol>
-                <Port>{{ action["redirect_config._port"] }}</Port>
-                <StatusCode>{{ action["redirect_config._status_code"] }}</StatusCode>
-            </RedirectConfig>
-            {% endif %}
+            {{ action.to_xml() }}
           </member>
           {% endfor %}
         </Actions>
@@ -989,16 +971,7 @@ DESCRIBE_LISTENERS_TEMPLATE = """<DescribeLoadBalancersResponse xmlns="http://el
         <DefaultActions>
           {% for action in listener.default_actions %}
           <member>
-            <Type>{{ action.type }}</Type>
-            {% if action["type"] == "forward" %}
-            <TargetGroupArn>{{ action["target_group_arn"] }}</TargetGroupArn>m
-            {% elif action["type"] == "redirect" %}
-            <RedirectConfig>
-                <Protocol>{{ action["redirect_config._protocol"] }}</Protocol>
-                <Port>{{ action["redirect_config._port"] }}</Port>
-                <StatusCode>{{ action["redirect_config._status_code"] }}</StatusCode>
-            </RedirectConfig>
-            {% endif %}
+            {{ action.to_xml() }}
           </member>
           {% endfor %}
         </DefaultActions>
@@ -1048,8 +1021,7 @@ MODIFY_RULE_TEMPLATE = """<ModifyRuleResponse xmlns="http://elasticloadbalancing
         <Actions>
           {% for action in rule.actions %}
           <member>
-            <Type>{{ action["type"] }}</Type>
-            <TargetGroupArn>{{ action["target_group_arn"] }}</TargetGroupArn>
+            {{ action.to_xml() }}
           </member>
           {% endfor %}
         </Actions>
@@ -1208,6 +1180,12 @@ DESCRIBE_TARGET_HEALTH_TEMPLATE = """<DescribeTargetHealthResponse xmlns="http:/
         <HealthCheckPort>{{ target_health.health_port }}</HealthCheckPort>
         <TargetHealth>
           <State>{{ target_health.status }}</State>
+          {% if target_health.reason %}
+            <Reason>{{ target_health.reason }}</Reason>
+          {% endif %}
+          {% if target_health.description %}
+            <Description>{{ target_health.description }}</Description>
+          {% endif %}
         </TargetHealth>
         <Target>
           <Port>{{ target_health.port }}</Port>
@@ -1426,16 +1404,7 @@ MODIFY_LISTENER_TEMPLATE = """<ModifyListenerResponse xmlns="http://elasticloadb
         <DefaultActions>
           {% for action in listener.default_actions %}
           <member>
-            <Type>{{ action.type }}</Type>
-            {% if action["type"] == "forward" %}
-            <TargetGroupArn>{{ action["target_group_arn"] }}</TargetGroupArn>
-            {% elif action["type"] == "redirect" %}
-            <RedirectConfig>
-                <Protocol>{{ action["redirect_config._protocol"] }}</Protocol>
-                <Port>{{ action["redirect_config._port"] }}</Port>
-                <StatusCode>{{ action["redirect_config._status_code"] }}</StatusCode>
-            </RedirectConfig>
-            {% endif %}
+            {{ action.to_xml() }}
           </member>
           {% endfor %}
         </DefaultActions>

moto/glue/responses.py

@@ -141,6 +141,23 @@ class GlueResponse(BaseResponse):
 
         return json.dumps({'Partition': p.as_dict()})
 
+    def batch_get_partition(self):
+        database_name = self.parameters.get('DatabaseName')
+        table_name = self.parameters.get('TableName')
+        partitions_to_get = self.parameters.get('PartitionsToGet')
+
+        table = self.glue_backend.get_table(database_name, table_name)
+
+        partitions = []
+        for values in partitions_to_get:
+            try:
+                p = table.get_partition(values=values["Values"])
+                partitions.append(p.as_dict())
+            except PartitionNotFoundException:
+                continue
+
+        return json.dumps({'Partitions': partitions})
+
     def create_partition(self):
         database_name = self.parameters.get('DatabaseName')
         table_name = self.parameters.get('TableName')

moto/iam/models.py

@@ -694,7 +694,6 @@ class IAMBackend(BaseBackend):
     def _validate_tag_key(self, tag_key, exception_param='tags.X.member.key'):
         """Validates the tag key.
 
-        :param all_tags: Dict to check if there is a duplicate tag.
         :param tag_key: The tag key to check against.
         :param exception_param: The exception parameter to send over to help format the message. This is to reflect
                                 the difference between the tag and untag APIs.

moto/kms/models.py

@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 import os
 import boto.kms
 from moto.core import BaseBackend, BaseModel
-from moto.core.utils import iso_8601_datetime_without_milliseconds, unix_time
+from moto.core.utils import iso_8601_datetime_without_milliseconds
 from .utils import generate_key_id
 from collections import defaultdict
 from datetime import datetime, timedelta
@@ -11,7 +11,7 @@ from datetime import datetime, timedelta
 
 class Key(BaseModel):
 
-    def __init__(self, policy, key_usage, description, region):
+    def __init__(self, policy, key_usage, description, tags, region):
         self.id = generate_key_id()
         self.policy = policy
         self.key_usage = key_usage
@@ -22,7 +22,7 @@ class Key(BaseModel):
         self.account_id = "0123456789012"
         self.key_rotation_status = False
         self.deletion_date = None
-        self.tags = {}
+        self.tags = tags or {}
 
     @property
     def physical_resource_id(self):
@@ -37,7 +37,7 @@ class Key(BaseModel):
             "KeyMetadata": {
                 "AWSAccountId": self.account_id,
                 "Arn": self.arn,
-                "CreationDate": "%d" % unix_time(),
+                "CreationDate": iso_8601_datetime_without_milliseconds(datetime.now()),
                 "Description": self.description,
                 "Enabled": self.enabled,
                 "KeyId": self.id,
@@ -61,6 +61,7 @@ class Key(BaseModel):
             policy=properties['KeyPolicy'],
             key_usage='ENCRYPT_DECRYPT',
             description=properties['Description'],
+            tags=properties.get('Tags'),
             region=region_name,
         )
         key.key_rotation_status = properties['EnableKeyRotation']
@@ -80,8 +81,8 @@ class KmsBackend(BaseBackend):
         self.keys = {}
         self.key_to_aliases = defaultdict(set)
 
-    def create_key(self, policy, key_usage, description, region):
-        key = Key(policy, key_usage, description, region)
+    def create_key(self, policy, key_usage, description, tags, region):
+        key = Key(policy, key_usage, description, tags, region)
         self.keys[key.id] = key
         return key
 

moto/kms/responses.py

@@ -31,9 +31,10 @@ class KmsResponse(BaseResponse):
         policy = self.parameters.get('Policy')
         key_usage = self.parameters.get('KeyUsage')
         description = self.parameters.get('Description')
+        tags = self.parameters.get('Tags')
 
         key = self.kms_backend.create_key(
-            policy, key_usage, description, self.region)
+            policy, key_usage, description, tags, self.region)
         return json.dumps(key.to_dict())
 
     def update_key_description(self):
@@ -237,7 +238,7 @@ class KmsResponse(BaseResponse):
 
         value = self.parameters.get("CiphertextBlob")
         try:
-            return json.dumps({"Plaintext": base64.b64decode(value).decode("utf-8")})
+            return json.dumps({"Plaintext": base64.b64decode(value).decode("utf-8"), 'KeyId': 'key_id'})
         except UnicodeDecodeError:
             # Generate data key will produce random bytes which when decrypted is still returned as base64
             return json.dumps({"Plaintext": value})

moto/logs/models.py

@@ -98,17 +98,29 @@ class LogStream:
 
             return True
 
+        def get_paging_token_from_index(index, back=False):
+            if index is not None:
+                return "b/{:056d}".format(index) if back else "f/{:056d}".format(index)
+            return 0
+
+        def get_index_from_paging_token(token):
+            if token is not None:
+                return int(token[2:])
+            return 0
+
         events = sorted(filter(filter_func, self.events), key=lambda event: event.timestamp, reverse=start_from_head)
-        back_token = next_token
-        if next_token is None:
-            next_token = 0
+        next_index = get_index_from_paging_token(next_token)
+        back_index = next_index
 
-        events_page = [event.to_response_dict() for event in events[next_token: next_token + limit]]
-        next_token += limit
-        if next_token >= len(self.events):
-            next_token = None
+        events_page = [event.to_response_dict() for event in events[next_index: next_index + limit]]
+        if next_index + limit < len(self.events):
+            next_index += limit
 
-        return events_page, back_token, next_token
+        back_index -= limit
+        if back_index <= 0:
+            back_index = 0
+
+        return events_page, get_paging_token_from_index(back_index, True), get_paging_token_from_index(next_index)
 
     def filter_log_events(self, log_group_name, log_stream_names, start_time, end_time, limit, next_token, filter_pattern, interleaved):
         def filter_func(event):

moto/organizations/models.py

@@ -2,6 +2,7 @@ from __future__ import unicode_literals
 
 import datetime
 import re
+import json
 
 from moto.core import BaseBackend, BaseModel
 from moto.core.exceptions import RESTError
@@ -151,7 +152,6 @@ class FakeRoot(FakeOrganizationalUnit):
 class FakeServiceControlPolicy(BaseModel):
 
     def __init__(self, organization, **kwargs):
-        self.type = 'POLICY'
         self.content = kwargs.get('Content')
         self.description = kwargs.get('Description')
         self.name = kwargs.get('Name')
@@ -197,7 +197,38 @@ class OrganizationsBackend(BaseBackend):
 
     def create_organization(self, **kwargs):
         self.org = FakeOrganization(kwargs['FeatureSet'])
-        self.ou.append(FakeRoot(self.org))
+        root_ou = FakeRoot(self.org)
+        self.ou.append(root_ou)
+        master_account = FakeAccount(
+            self.org,
+            AccountName='master',
+            Email=self.org.master_account_email,
+        )
+        master_account.id = self.org.master_account_id
+        self.accounts.append(master_account)
+        default_policy = FakeServiceControlPolicy(
+            self.org,
+            Name='FullAWSAccess',
+            Description='Allows access to every operation',
+            Type='SERVICE_CONTROL_POLICY',
+            Content=json.dumps(
+                {
+                    "Version": "2012-10-17",
+                    "Statement": [
+                        {
+                            "Effect": "Allow",
+                            "Action": "*",
+                            "Resource": "*"
+                        }
+                    ]
+                }
+            )
+        )
+        default_policy.id = utils.DEFAULT_POLICY_ID
+        default_policy.aws_managed = True
+        self.policies.append(default_policy)
+        self.attach_policy(PolicyId=default_policy.id, TargetId=root_ou.id)
+        self.attach_policy(PolicyId=default_policy.id, TargetId=master_account.id)
         return self.org.describe()
 
     def describe_organization(self):
@@ -216,6 +247,7 @@ class OrganizationsBackend(BaseBackend):
     def create_organizational_unit(self, **kwargs):
         new_ou = FakeOrganizationalUnit(self.org, **kwargs)
         self.ou.append(new_ou)
+        self.attach_policy(PolicyId=utils.DEFAULT_POLICY_ID, TargetId=new_ou.id)
         return new_ou.describe()
 
     def get_organizational_unit_by_id(self, ou_id):
@@ -258,6 +290,7 @@ class OrganizationsBackend(BaseBackend):
     def create_account(self, **kwargs):
         new_account = FakeAccount(self.org, **kwargs)
         self.accounts.append(new_account)
+        self.attach_policy(PolicyId=utils.DEFAULT_POLICY_ID, TargetId=new_account.id)
         return new_account.create_account_status
 
     def get_account_by_id(self, account_id):
@@ -358,8 +391,7 @@ class OrganizationsBackend(BaseBackend):
 
     def attach_policy(self, **kwargs):
         policy = next((p for p in self.policies if p.id == kwargs['PolicyId']), None)
-        if (re.compile(utils.ROOT_ID_REGEX).match(kwargs['TargetId']) or
-                re.compile(utils.OU_ID_REGEX).match(kwargs['TargetId'])):
+        if (re.compile(utils.ROOT_ID_REGEX).match(kwargs['TargetId']) or re.compile(utils.OU_ID_REGEX).match(kwargs['TargetId'])):
             ou = next((ou for ou in self.ou if ou.id == kwargs['TargetId']), None)
             if ou is not None:
                 if ou not in ou.attached_policies:

moto/organizations/utils.py

@@ -4,7 +4,8 @@ import random
 import string
 
 MASTER_ACCOUNT_ID = '123456789012'
-MASTER_ACCOUNT_EMAIL = 'fakeorg@moto-example.com'
+MASTER_ACCOUNT_EMAIL = 'master@example.com'
+DEFAULT_POLICY_ID = 'p-FullAWSAccess'
 ORGANIZATION_ARN_FORMAT = 'arn:aws:organizations::{0}:organization/{1}'
 MASTER_ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{0}'
 ACCOUNT_ARN_FORMAT = 'arn:aws:organizations::{0}:account/{1}/{2}'
@@ -26,7 +27,7 @@ ROOT_ID_REGEX = r'r-[a-z0-9]{%s}' % ROOT_ID_SIZE
 OU_ID_REGEX = r'ou-[a-z0-9]{%s}-[a-z0-9]{%s}' % (ROOT_ID_SIZE, OU_ID_SUFFIX_SIZE)
 ACCOUNT_ID_REGEX = r'[0-9]{%s}' % ACCOUNT_ID_SIZE
 CREATE_ACCOUNT_STATUS_ID_REGEX = r'car-[a-z0-9]{%s}' % CREATE_ACCOUNT_STATUS_ID_SIZE
-SCP_ID_REGEX = r'p-[a-z0-9]{%s}' % SCP_ID_SIZE
+SCP_ID_REGEX = r'%s|p-[a-z0-9]{%s}' % (DEFAULT_POLICY_ID, SCP_ID_SIZE)
 
 
 def make_random_org_id():

moto/rds/responses.py

@@ -95,7 +95,7 @@ class RDSResponse(BaseResponse):
             start = all_ids.index(marker) + 1
         else:
             start = 0
-        page_size = self._get_param('MaxRecords', 50)  # the default is 100, but using 50 to make testing easier
+        page_size = self._get_int_param('MaxRecords', 50)  # the default is 100, but using 50 to make testing easier
         instances_resp = all_instances[start:start + page_size]
         next_marker = None
         if len(all_instances) > start + page_size:

moto/rds2/models.py

@@ -149,7 +149,14 @@ class Database(BaseModel):
               <DBInstanceStatus>{{ database.status }}</DBInstanceStatus>
               {% if database.db_name %}<DBName>{{ database.db_name }}</DBName>{% endif %}
               <MultiAZ>{{ database.multi_az }}</MultiAZ>
-              <VpcSecurityGroups/>
+              <VpcSecurityGroups>
+                {% for vpc_security_group_id in database.vpc_security_group_ids %}
+                <VpcSecurityGroupMembership>
+                  <Status>active</Status>
+                  <VpcSecurityGroupId>{{ vpc_security_group_id }}</VpcSecurityGroupId>
+                </VpcSecurityGroupMembership>
+                {% endfor %}
+              </VpcSecurityGroups>
               <DBInstanceIdentifier>{{ database.db_instance_identifier }}</DBInstanceIdentifier>
               <DbiResourceId>{{ database.dbi_resource_id }}</DbiResourceId>
               <InstanceCreateTime>{{ database.instance_create_time }}</InstanceCreateTime>
@@ -323,6 +330,7 @@ class Database(BaseModel):
             "storage_encrypted": properties.get("StorageEncrypted"),
             "storage_type": properties.get("StorageType"),
             "tags": properties.get("Tags"),
+            "vpc_security_group_ids": properties.get('VpcSecurityGroupIds', []),
         }
 
         rds2_backend = rds2_backends[region_name]
@@ -397,10 +405,12 @@ class Database(BaseModel):
         "SecondaryAvailabilityZone": null,
         "StatusInfos": null,
         "VpcSecurityGroups": [
+            {% for vpc_security_group_id in database.vpc_security_group_ids %}
             {
                 "Status": "active",
-                "VpcSecurityGroupId": "sg-123456"
+                "VpcSecurityGroupId": "{{ vpc_security_group_id }}"
             }
+            {% endfor %}
         ],
         "DBInstanceArn": "{{ database.db_instance_arn }}"
       }""")

moto/rds2/responses.py

@@ -43,7 +43,7 @@ class RDS2Response(BaseResponse):
             "security_groups": self._get_multi_param('DBSecurityGroups.DBSecurityGroupName'),
             "storage_encrypted": self._get_param("StorageEncrypted"),
             "storage_type": self._get_param("StorageType", 'standard'),
-            # VpcSecurityGroupIds.member.N
+            "vpc_security_group_ids": self._get_multi_param("VpcSecurityGroupIds.VpcSecurityGroupId"),
             "tags": list(),
         }
         args['tags'] = self.unpack_complex_list_params(
@@ -280,7 +280,7 @@ class RDS2Response(BaseResponse):
 
     def describe_option_groups(self):
         kwargs = self._get_option_group_kwargs()
-        kwargs['max_records'] = self._get_param('MaxRecords')
+        kwargs['max_records'] = self._get_int_param('MaxRecords')
         kwargs['marker'] = self._get_param('Marker')
         option_groups = self.backend.describe_option_groups(kwargs)
         template = self.response_template(DESCRIBE_OPTION_GROUP_TEMPLATE)
@@ -329,7 +329,7 @@ class RDS2Response(BaseResponse):
 
     def describe_db_parameter_groups(self):
         kwargs = self._get_db_parameter_group_kwargs()
-        kwargs['max_records'] = self._get_param('MaxRecords')
+        kwargs['max_records'] = self._get_int_param('MaxRecords')
         kwargs['marker'] = self._get_param('Marker')
         db_parameter_groups = self.backend.describe_db_parameter_groups(kwargs)
         template = self.response_template(

moto/redshift/models.py

@@ -78,7 +78,7 @@ class Cluster(TaggableResourceMixin, BaseModel):
         super(Cluster, self).__init__(region_name, tags)
         self.redshift_backend = redshift_backend
         self.cluster_identifier = cluster_identifier
-        self.create_time = iso_8601_datetime_with_milliseconds(datetime.datetime.now())
+        self.create_time = iso_8601_datetime_with_milliseconds(datetime.datetime.utcnow())
         self.status = 'available'
         self.node_type = node_type
         self.master_username = master_username

moto/resourcegroupstaggingapi/models.py

@@ -10,6 +10,7 @@ from moto.ec2 import ec2_backends
 from moto.elb import elb_backends
 from moto.elbv2 import elbv2_backends
 from moto.kinesis import kinesis_backends
+from moto.kms import kms_backends
 from moto.rds2 import rds2_backends
 from moto.glacier import glacier_backends
 from moto.redshift import redshift_backends
@@ -71,6 +72,13 @@ class ResourceGroupsTaggingAPIBackend(BaseBackend):
         """
         return kinesis_backends[self.region_name]
 
+    @property
+    def kms_backend(self):
+        """
+        :rtype: moto.kms.models.KmsBackend
+        """
+        return kms_backends[self.region_name]
+
     @property
     def rds_backend(self):
         """
@@ -221,9 +229,6 @@ class ResourceGroupsTaggingAPIBackend(BaseBackend):
         if not resource_type_filters or 'elasticloadbalancer' in resource_type_filters or 'elasticloadbalancer:loadbalancer' in resource_type_filters:
             for elb in self.elbv2_backend.load_balancers.values():
                 tags = get_elbv2_tags(elb.arn)
-                # if 'elasticloadbalancer:loadbalancer' in resource_type_filters:
-                #     from IPython import embed
-                #     embed()
                 if not tag_filter(tags):  # Skip if no tags, or invalid filter
                     continue
 
@@ -235,6 +240,21 @@ class ResourceGroupsTaggingAPIBackend(BaseBackend):
 
         # Kinesis
 
+        # KMS
+        def get_kms_tags(kms_key_id):
+            result = []
+            for tag in self.kms_backend.list_resource_tags(kms_key_id):
+                    result.append({'Key': tag['TagKey'], 'Value': tag['TagValue']})
+            return result
+
+        if not resource_type_filters or 'kms' in resource_type_filters:
+            for kms_key in self.kms_backend.list_keys():
+                tags = get_kms_tags(kms_key.id)
+                if not tag_filter(tags):  # Skip if no tags, or invalid filter
+                    continue
+
+                yield {'ResourceARN': '{0}'.format(kms_key.arn), 'Tags': tags}
+
         # RDS Instance
         # RDS Reserved Database Instance
         # RDS Option Group
@@ -370,7 +390,7 @@ class ResourceGroupsTaggingAPIBackend(BaseBackend):
     def get_resources(self, pagination_token=None,
                       resources_per_page=50, tags_per_page=100,
                       tag_filters=None, resource_type_filters=None):
-        # Simple range checning
+        # Simple range checking
         if 100 >= tags_per_page >= 500:
             raise RESTError('InvalidParameterException', 'TagsPerPage must be between 100 and 500')
         if 1 >= resources_per_page >= 50:

moto/route53/models.py

@@ -198,7 +198,7 @@ class FakeZone(BaseModel):
     def upsert_rrset(self, record_set):
         new_rrset = RecordSet(record_set)
         for i, rrset in enumerate(self.rrsets):
-            if rrset.name == new_rrset.name and rrset.type_ == new_rrset.type_:
+            if rrset.name == new_rrset.name and rrset.type_ == new_rrset.type_ and rrset.set_identifier == new_rrset.set_identifier:
                 self.rrsets[i] = new_rrset
                 break
         else:

moto/s3/exceptions.py

@@ -60,6 +60,17 @@ class MissingKey(S3ClientError):
         )
 
 
+class ObjectNotInActiveTierError(S3ClientError):
+    code = 403
+
+    def __init__(self, key_name):
+        super(ObjectNotInActiveTierError, self).__init__(
+            "ObjectNotInActiveTierError",
+            "The source object of the COPY operation is not in the active tier and is only stored in Amazon Glacier.",
+            Key=key_name,
+        )
+
+
 class InvalidPartOrder(S3ClientError):
     code = 400
 

moto/s3/models.py

@@ -28,7 +28,8 @@ MAX_BUCKET_NAME_LENGTH = 63
 MIN_BUCKET_NAME_LENGTH = 3
 UPLOAD_ID_BYTES = 43
 UPLOAD_PART_MIN_SIZE = 5242880
-STORAGE_CLASS = ["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA"]
+STORAGE_CLASS = ["STANDARD", "REDUCED_REDUNDANCY", "STANDARD_IA", "ONEZONE_IA",
+                 "INTELLIGENT_TIERING", "GLACIER", "DEEP_ARCHIVE"]
 DEFAULT_KEY_BUFFER_SIZE = 16 * 1024 * 1024
 DEFAULT_TEXT_ENCODING = sys.getdefaultencoding()
 
@@ -52,8 +53,17 @@ class FakeDeleteMarker(BaseModel):
 
 class FakeKey(BaseModel):
 
-    def __init__(self, name, value, storage="STANDARD", etag=None, is_versioned=False, version_id=0,
-                 max_buffer_size=DEFAULT_KEY_BUFFER_SIZE):
+    def __init__(
+        self,
+        name,
+        value,
+        storage="STANDARD",
+        etag=None,
+        is_versioned=False,
+        version_id=0,
+        max_buffer_size=DEFAULT_KEY_BUFFER_SIZE,
+        multipart=None
+    ):
         self.name = name
         self.last_modified = datetime.datetime.utcnow()
         self.acl = get_canned_acl('private')
@@ -65,6 +75,7 @@ class FakeKey(BaseModel):
         self._version_id = version_id
         self._is_versioned = is_versioned
         self._tagging = FakeTagging()
+        self.multipart = multipart
 
         self._value_buffer = tempfile.SpooledTemporaryFile(max_size=max_buffer_size)
         self._max_buffer_size = max_buffer_size
@@ -754,7 +765,7 @@ class S3Backend(BaseBackend):
                             prefix=''):
         bucket = self.get_bucket(bucket_name)
 
-        if any((delimiter, encoding_type, key_marker, version_id_marker)):
+        if any((delimiter, key_marker, version_id_marker)):
             raise NotImplementedError(
                 "Called get_bucket_versions with some of delimiter, encoding_type, key_marker, version_id_marker")
 
@@ -782,7 +793,15 @@ class S3Backend(BaseBackend):
         bucket = self.get_bucket(bucket_name)
         return bucket.website_configuration
 
-    def set_key(self, bucket_name, key_name, value, storage=None, etag=None):
+    def set_key(
+        self,
+        bucket_name,
+        key_name,
+        value,
+        storage=None,
+        etag=None,
+        multipart=None,
+    ):
         key_name = clean_key_name(key_name)
         if storage is not None and storage not in STORAGE_CLASS:
             raise InvalidStorageClass(storage=storage)
@@ -795,7 +814,9 @@ class S3Backend(BaseBackend):
             storage=storage,
             etag=etag,
             is_versioned=bucket.is_versioned,
-            version_id=str(uuid.uuid4()) if bucket.is_versioned else None)
+            version_id=str(uuid.uuid4()) if bucket.is_versioned else None,
+            multipart=multipart,
+        )
 
         keys = [
             key for key in bucket.keys.getlist(key_name, [])
@@ -812,7 +833,7 @@ class S3Backend(BaseBackend):
         key.append_to_value(value)
         return key
 
-    def get_key(self, bucket_name, key_name, version_id=None):
+    def get_key(self, bucket_name, key_name, version_id=None, part_number=None):
         key_name = clean_key_name(key_name)
         bucket = self.get_bucket(bucket_name)
         key = None
@@ -827,6 +848,9 @@ class S3Backend(BaseBackend):
                         key = key_version
                         break
 
+            if part_number and key.multipart:
+                key = key.multipart.parts[part_number]
+
         if isinstance(key, FakeKey):
             return key
         else:
@@ -890,7 +914,12 @@ class S3Backend(BaseBackend):
             return
         del bucket.multiparts[multipart_id]
 
-        key = self.set_key(bucket_name, multipart.key_name, value, etag=etag)
+        key = self.set_key(
+            bucket_name,
+            multipart.key_name,
+            value, etag=etag,
+            multipart=multipart
+        )
         key.set_metadata(multipart.metadata)
         return key
 

moto/s3/responses.py

@@ -17,7 +17,7 @@ from moto.s3bucket_path.utils import bucket_name_from_url as bucketpath_bucket_n
     parse_key_name as bucketpath_parse_key_name, is_delete_keys as bucketpath_is_delete_keys
 
 from .exceptions import BucketAlreadyExists, S3ClientError, MissingBucket, MissingKey, InvalidPartOrder, MalformedXML, \
-    MalformedACLError, InvalidNotificationARN, InvalidNotificationEvent
+    MalformedACLError, InvalidNotificationARN, InvalidNotificationEvent, ObjectNotInActiveTierError
 from .models import s3_backend, get_canned_acl, FakeGrantee, FakeGrant, FakeAcl, FakeKey, FakeTagging, FakeTagSet, \
     FakeTag
 from .utils import bucket_name_from_url, clean_key_name, metadata_from_headers, parse_region_from_url
@@ -686,6 +686,8 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
         keys = minidom.parseString(body).getElementsByTagName('Key')
         deleted_names = []
         error_names = []
+        if len(keys) == 0:
+            raise MalformedXML()
 
         for k in keys:
             key_name = k.firstChild.nodeValue
@@ -900,7 +902,11 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
             src_version_id = parse_qs(src_key_parsed.query).get(
                 'versionId', [None])[0]
 
-            if self.backend.get_key(src_bucket, src_key, version_id=src_version_id):
+            key = self.backend.get_key(src_bucket, src_key, version_id=src_version_id)
+
+            if key is not None:
+                if key.storage_class in ["GLACIER", "DEEP_ARCHIVE"]:
+                    raise ObjectNotInActiveTierError(key)
                 self.backend.copy_key(src_bucket, src_key, bucket_name, key_name,
                                       storage=storage_class, acl=acl, src_version_id=src_version_id)
             else:
@@ -940,13 +946,20 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin):
     def _key_response_head(self, bucket_name, query, key_name, headers):
         response_headers = {}
         version_id = query.get('versionId', [None])[0]
+        part_number = query.get('partNumber', [None])[0]
+        if part_number:
+            part_number = int(part_number)
 
         if_modified_since = headers.get('If-Modified-Since', None)
         if if_modified_since:
             if_modified_since = str_to_rfc_1123_datetime(if_modified_since)
 
         key = self.backend.get_key(
-            bucket_name, key_name, version_id=version_id)
+            bucket_name,
+            key_name,
+            version_id=version_id,
+            part_number=part_number
+        )
         if key:
             response_headers.update(key.metadata)
             response_headers.update(key.response_dict)

moto/server.py

@@ -21,6 +21,16 @@ from moto.core.utils import convert_flask_to_httpretty_response
 HTTP_METHODS = ["GET", "POST", "PUT", "DELETE", "HEAD", "PATCH"]
 
 
+DEFAULT_SERVICE_REGION = ('s3', 'us-east-1')
+
+# Map of unsigned calls to service-region as per AWS API docs
+# https://docs.aws.amazon.com/cognito/latest/developerguide/resource-permissions.html#amazon-cognito-signed-versus-unsigned-apis
+UNSIGNED_REQUESTS = {
+    'AWSCognitoIdentityService': ('cognito-identity', 'us-east-1'),
+    'AWSCognitoIdentityProviderService': ('cognito-idp', 'us-east-1'),
+}
+
+
 class DomainDispatcherApplication(object):
     """
     Dispatch requests to different applications based on the "Host:" header
@@ -48,7 +58,45 @@ class DomainDispatcherApplication(object):
                 if re.match(url_base, 'http://%s' % host):
                     return backend_name
 
-        raise RuntimeError('Invalid host: "%s"' % host)
+    def infer_service_region_host(self, environ):
+        auth = environ.get('HTTP_AUTHORIZATION')
+        if auth:
+            # Signed request
+            # Parse auth header to find service assuming a SigV4 request
+            # https://docs.aws.amazon.com/general/latest/gr/sigv4-signed-request-examples.html
+            # ['Credential=sdffdsa', '20170220', 'us-east-1', 'sns', 'aws4_request']
+            try:
+                credential_scope = auth.split(",")[0].split()[1]
+                _, _, region, service, _ = credential_scope.split("/")
+            except ValueError:
+                # Signature format does not match, this is exceptional and we can't
+                # infer a service-region. A reduced set of services still use
+                # the deprecated SigV2, ergo prefer S3 as most likely default.
+                # https://docs.aws.amazon.com/general/latest/gr/signature-version-2.html
+                service, region = DEFAULT_SERVICE_REGION
+        else:
+            # Unsigned request
+            target = environ.get('HTTP_X_AMZ_TARGET')
+            if target:
+                service, _ = target.split('.', 1)
+                service, region = UNSIGNED_REQUESTS.get(service, DEFAULT_SERVICE_REGION)
+            else:
+                # S3 is the last resort when the target is also unknown
+                service, region = DEFAULT_SERVICE_REGION
+
+        if service == 'dynamodb':
+            if environ['HTTP_X_AMZ_TARGET'].startswith('DynamoDBStreams'):
+                host = 'dynamodbstreams'
+            else:
+                dynamo_api_version = environ['HTTP_X_AMZ_TARGET'].split("_")[1].split(".")[0]
+                # If Newer API version, use dynamodb2
+                if dynamo_api_version > "20111205":
+                    host = "dynamodb2"
+        else:
+            host = "{service}.{region}.amazonaws.com".format(
+                service=service, region=region)
+
+        return host
 
     def get_application(self, environ):
         path_info = environ.get('PATH_INFO', '')
@@ -65,34 +113,14 @@ class DomainDispatcherApplication(object):
             host = "instance_metadata"
         else:
             host = environ['HTTP_HOST'].split(':')[0]
-        if host in {'localhost', 'motoserver'} or host.startswith("192.168."):
-            # Fall back to parsing auth header to find service
-            # ['Credential=sdffdsa', '20170220', 'us-east-1', 'sns', 'aws4_request']
-            try:
-                _, _, region, service, _ = environ['HTTP_AUTHORIZATION'].split(",")[0].split()[
-                    1].split("/")
-            except (KeyError, ValueError):
-                # Some cognito-idp endpoints (e.g. change password) do not receive an auth header.
-                if environ.get('HTTP_X_AMZ_TARGET', '').startswith('AWSCognitoIdentityProviderService'):
-                    service = 'cognito-idp'
-                else:
-                    service = 's3'
-
-                region = 'us-east-1'
-            if service == 'dynamodb':
-                if environ['HTTP_X_AMZ_TARGET'].startswith('DynamoDBStreams'):
-                    host = 'dynamodbstreams'
-                else:
-                    dynamo_api_version = environ['HTTP_X_AMZ_TARGET'].split("_")[1].split(".")[0]
-                    # If Newer API version, use dynamodb2
-                    if dynamo_api_version > "20111205":
-                        host = "dynamodb2"
-            else:
-                host = "{service}.{region}.amazonaws.com".format(
-                    service=service, region=region)
 
         with self.lock:
             backend = self.get_backend_for_host(host)
+            if not backend:
+                # No regular backend found; try parsing other headers
+                host = self.infer_service_region_host(environ)
+                backend = self.get_backend_for_host(host)
+
             app = self.app_instances.get(backend, None)
             if app is None:
                 app = self.create_app(backend)

moto/sqs/models.py

@@ -379,6 +379,7 @@ class SQSBackend(BaseBackend):
 
     def reset(self):
         region_name = self.region_name
+        self._reset_model_refs()
         self.__dict__ = {}
         self.__init__(region_name)
 

moto/sts/exceptions.py

@@ -0,0 +1,15 @@
+from __future__ import unicode_literals
+from moto.core.exceptions import RESTError
+
+
+class STSClientError(RESTError):
+    code = 400
+
+
+class STSValidationError(STSClientError):
+
+    def __init__(self, *args, **kwargs):
+        super(STSValidationError, self).__init__(
+            "ValidationError",
+            *args, **kwargs
+        )

moto/sts/models.py

@@ -65,5 +65,8 @@ class STSBackend(BaseBackend):
                 return assumed_role
         return None
 
+    def assume_role_with_web_identity(self, **kwargs):
+        return self.assume_role(**kwargs)
+
 
 sts_backend = STSBackend()

moto/sts/responses.py

@@ -3,8 +3,11 @@ from __future__ import unicode_literals
 from moto.core.responses import BaseResponse
 from moto.iam.models import ACCOUNT_ID
 from moto.iam import iam_backend
+from .exceptions import STSValidationError
 from .models import sts_backend
 
+MAX_FEDERATION_TOKEN_POLICY_LENGTH = 2048
+
 
 class TokenResponse(BaseResponse):
 
@@ -17,6 +20,15 @@ class TokenResponse(BaseResponse):
     def get_federation_token(self):
         duration = int(self.querystring.get('DurationSeconds', [43200])[0])
         policy = self.querystring.get('Policy', [None])[0]
+
+        if policy is not None and len(policy) > MAX_FEDERATION_TOKEN_POLICY_LENGTH:
+            raise STSValidationError(
+                "1 validation error detected: Value "
+                "'{\"Version\": \"2012-10-17\", \"Statement\": [...]}' "
+                "at 'policy' failed to satisfy constraint: Member must have length less than or "
+                " equal to %s" % MAX_FEDERATION_TOKEN_POLICY_LENGTH
+            )
+
         name = self.querystring.get('Name')[0]
         token = sts_backend.get_federation_token(
             duration=duration, name=name, policy=policy)
@@ -41,6 +53,24 @@ class TokenResponse(BaseResponse):
         template = self.response_template(ASSUME_ROLE_RESPONSE)
         return template.render(role=role)
 
+    def assume_role_with_web_identity(self):
+        role_session_name = self.querystring.get('RoleSessionName')[0]
+        role_arn = self.querystring.get('RoleArn')[0]
+
+        policy = self.querystring.get('Policy', [None])[0]
+        duration = int(self.querystring.get('DurationSeconds', [3600])[0])
+        external_id = self.querystring.get('ExternalId', [None])[0]
+
+        role = sts_backend.assume_role_with_web_identity(
+            role_session_name=role_session_name,
+            role_arn=role_arn,
+            policy=policy,
+            duration=duration,
+            external_id=external_id,
+        )
+        template = self.response_template(ASSUME_ROLE_WITH_WEB_IDENTITY_RESPONSE)
+        return template.render(role=role)
+
     def get_caller_identity(self):
         template = self.response_template(GET_CALLER_IDENTITY_RESPONSE)
 
@@ -118,6 +148,27 @@ ASSUME_ROLE_RESPONSE = """<AssumeRoleResponse xmlns="https://sts.amazonaws.com/d
   </ResponseMetadata>
 </AssumeRoleResponse>"""
 
+
+ASSUME_ROLE_WITH_WEB_IDENTITY_RESPONSE = """<AssumeRoleWithWebIdentityResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
+  <AssumeRoleWithWebIdentityResult>
+    <Credentials>
+      <SessionToken>{{ role.session_token }}</SessionToken>
+      <SecretAccessKey>{{ role.secret_access_key }}</SecretAccessKey>
+      <Expiration>{{ role.expiration_ISO8601 }}</Expiration>
+      <AccessKeyId>{{ role.access_key_id }}</AccessKeyId>
+    </Credentials>
+    <AssumedRoleUser>
+      <Arn>{{ role.arn }}</Arn>
+      <AssumedRoleId>ARO123EXAMPLE123:{{ role.session_name }}</AssumedRoleId>
+    </AssumedRoleUser>
+    <PackedPolicySize>6</PackedPolicySize>
+  </AssumeRoleWithWebIdentityResult>
+  <ResponseMetadata>
+    <RequestId>c6104cbe-af31-11e0-8154-cbc7ccf896c7</RequestId>
+  </ResponseMetadata>
+</AssumeRoleWithWebIdentityResponse>"""
+
+
 GET_CALLER_IDENTITY_RESPONSE = """<GetCallerIdentityResponse xmlns="https://sts.amazonaws.com/doc/2011-06-15/">
   <GetCallerIdentityResult>
     <Arn>{{ arn }}</Arn>