Merge branch 'master' into batch

moto/awslambda/models.py

@@ -132,6 +132,7 @@ class LambdaFunction(BaseModel):
         self.logs_backend = logs_backends[self.region]
         self.environment_vars = spec.get('Environment', {}).get('Variables', {})
         self.docker_client = docker.from_env()
+        self.policy = ""
 
         # Unfortunately mocking replaces this method w/o fallback enabled, so we
         # need to replace it if we detect it's been mocked
@@ -527,6 +528,9 @@ class LambdaBackend(BaseBackend):
                 pass
                 # Don't care
 
+    def add_policy(self, function_name, policy):
+        self.get_function(function_name).policy = policy
+
 
 def do_validate_s3():
     return os.environ.get('VALIDATE_LAMBDA_S3', '') in ['', '1', 'true']

moto/awslambda/responses.py

@@ -57,6 +57,35 @@ class LambdaResponse(BaseResponse):
         else:
             raise ValueError("Cannot handle {0} request".format(request.method))
 
+    def policy(self, request, full_url, headers):
+        if request.method == 'GET':
+            return self._get_policy(request, full_url, headers)
+        if request.method == 'POST':
+            return self._add_policy(request, full_url, headers)
+
+    def _add_policy(self, request, full_url, headers):
+        lambda_backend = self.get_lambda_backend(full_url)
+
+        path = request.path if hasattr(request, 'path') else request.path_url
+        function_name = path.split('/')[-2]
+        if lambda_backend.has_function(function_name):
+            policy = request.body.decode('utf8')
+            lambda_backend.add_policy(function_name, policy)
+            return 200, {}, json.dumps(dict(Statement=policy))
+        else:
+            return 404, {}, "{}"
+
+    def _get_policy(self, request, full_url, headers):
+        lambda_backend = self.get_lambda_backend(full_url)
+
+        path = request.path if hasattr(request, 'path') else request.path_url
+        function_name = path.split('/')[-2]
+        if lambda_backend.has_function(function_name):
+            function = lambda_backend.get_function(function_name)
+            return 200, {}, json.dumps(dict(Policy="{\"Statement\":[" + function.policy + "]}"))
+        else:
+            return 404, {}, "{}"
+
     def _invoke(self, request, full_url):
         response_headers = {}
         lambda_backend = self.get_lambda_backend(full_url)

moto/awslambda/urls.py

@@ -12,5 +12,6 @@ url_paths = {
     r'{0}/(?P<api_version>[^/]+)/functions/(?P<function_name>[\w_-]+)/?$': response.function,
     r'{0}/(?P<api_version>[^/]+)/functions/(?P<function_name>[\w_-]+)/invocations/?$': response.invoke,
     r'{0}/(?P<api_version>[^/]+)/functions/(?P<function_name>[\w_-]+)/invoke-async/?$': response.invoke_async,
-    r'{0}/(?P<api_version>[^/]+)/tags/(?P<resource_arn>.+)': response.tag
+    r'{0}/(?P<api_version>[^/]+)/tags/(?P<resource_arn>.+)': response.tag,
+    r'{0}/(?P<api_version>[^/]+)/functions/(?P<function_name>[\w_-]+)/policy/?$': response.policy
 }

moto/elbv2/exceptions.py

@@ -152,6 +152,13 @@ class InvalidDescribeRulesRequest(ELBClientError):
         )
 
 
+class ResourceInUseError(ELBClientError):
+
+    def __init__(self, msg="A specified resource is in use"):
+        super(ResourceInUseError, self).__init__(
+            "ResourceInUse", msg)
+
+
 class RuleNotFoundError(ELBClientError):
 
     def __init__(self):

moto/elbv2/models.py

@@ -21,6 +21,7 @@ from .exceptions import (
     InvalidActionTypeError,
     ActionTargetGroupNotFoundError,
     InvalidDescribeRulesRequest,
+    ResourceInUseError,
     RuleNotFoundError,
     DuplicatePriorityError,
     InvalidTargetGroupNameError,
@@ -426,10 +427,17 @@ class ELBv2Backend(BaseBackend):
         # however, boto3 does't raise error even if rule is not found
 
     def delete_target_group(self, target_group_arn):
-        target_group = self.target_groups.pop(target_group_arn, None)
+        if target_group_arn not in self.target_groups:
+            raise TargetGroupNotFoundError()
+
+        target_group = self.target_groups[target_group_arn]
         if target_group:
+            if self._any_listener_using(target_group_arn):
+                raise ResourceInUseError(
+                    "The target group '{}' is currently in use by a listener or a rule".format(
+                        target_group_arn))
+            del self.target_groups[target_group_arn]
             return target_group
-        raise TargetGroupNotFoundError()
 
     def delete_listener(self, listener_arn):
         for load_balancer in self.load_balancers.values():
@@ -539,6 +547,15 @@ class ELBv2Backend(BaseBackend):
             modified_rules.append(given_rule)
         return modified_rules
 
+    def _any_listener_using(self, target_group_arn):
+        for load_balancer in self.load_balancers.values():
+            for listener in load_balancer.listeners.values():
+                for rule in listener.rules:
+                    for action in rule.actions:
+                        if action.get('target_group_arn') == target_group_arn:
+                            return True
+        return False
+
 
 elbv2_backends = {}
 for region in ec2_backends.keys():

moto/iam/models.py

@@ -74,21 +74,13 @@ class ManagedPolicy(Policy):
 
     is_attachable = True
 
-    def attach_to_role(self, role):
+    def attach_to(self, obj):
         self.attachment_count += 1
-        role.managed_policies[self.name] = self
+        obj.managed_policies[self.name] = self
 
-    def detach_from_role(self, role):
+    def detach_from(self, obj):
         self.attachment_count -= 1
-        del role.managed_policies[self.name]
-
-    def attach_to_user(self, user):
-        self.attachment_count += 1
-        user.managed_policies[self.name] = self
-
-    def detach_from_user(self, user):
-        self.attachment_count -= 1
-        del user.managed_policies[self.name]
+        del obj.managed_policies[self.name]
 
 
 class AWSManagedPolicy(ManagedPolicy):
@@ -249,6 +241,7 @@ class Group(BaseModel):
         )
 
         self.users = []
+        self.managed_policies = {}
         self.policies = {}
 
     def get_cfn_attribute(self, attribute_name):
@@ -423,25 +416,47 @@ class IAMBackend(BaseBackend):
     def attach_role_policy(self, policy_arn, role_name):
         arns = dict((p.arn, p) for p in self.managed_policies.values())
         policy = arns[policy_arn]
-        policy.attach_to_role(self.get_role(role_name))
+        policy.attach_to(self.get_role(role_name))
 
     def detach_role_policy(self, policy_arn, role_name):
         arns = dict((p.arn, p) for p in self.managed_policies.values())
         try:
             policy = arns[policy_arn]
-            policy.detach_from_role(self.get_role(role_name))
+            policy.detach_from(self.get_role(role_name))
         except KeyError:
             raise IAMNotFoundException("Policy {0} was not found.".format(policy_arn))
 
+    def attach_group_policy(self, policy_arn, group_name):
+        arns = dict((p.arn, p) for p in self.managed_policies.values())
+        try:
+            policy = arns[policy_arn]
+        except KeyError:
+            raise IAMNotFoundException("Policy {0} was not found.".format(policy_arn))
+        policy.attach_to(self.get_group(group_name))
+
+    def detach_group_policy(self, policy_arn, group_name):
+        arns = dict((p.arn, p) for p in self.managed_policies.values())
+        try:
+            policy = arns[policy_arn]
+        except KeyError:
+            raise IAMNotFoundException("Policy {0} was not found.".format(policy_arn))
+        policy.detach_from(self.get_group(group_name))
+
     def attach_user_policy(self, policy_arn, user_name):
         arns = dict((p.arn, p) for p in self.managed_policies.values())
-        policy = arns[policy_arn]
-        policy.attach_to_user(self.get_user(user_name))
+        try:
+            policy = arns[policy_arn]
+        except KeyError:
+            raise IAMNotFoundException("Policy {0} was not found.".format(policy_arn))
+        policy.attach_to(self.get_user(user_name))
 
     def detach_user_policy(self, policy_arn, user_name):
         arns = dict((p.arn, p) for p in self.managed_policies.values())
-        policy = arns[policy_arn]
-        policy.detach_from_user(self.get_user(user_name))
+        try:
+            policy = arns[policy_arn]
+        except KeyError:
+            raise IAMNotFoundException("Policy {0} was not found.".format(policy_arn))
+        policy.detach_from(self.get_user(user_name))
 
     def create_policy(self, description, path, policy_document, policy_name):
         policy = ManagedPolicy(
@@ -458,39 +473,15 @@ class IAMBackend(BaseBackend):
 
     def list_attached_role_policies(self, role_name, marker=None, max_items=100, path_prefix='/'):
         policies = self.get_role(role_name).managed_policies.values()
+        return self._filter_attached_policies(policies, marker, max_items, path_prefix)
 
-        if path_prefix:
-            policies = [p for p in policies if p.path.startswith(path_prefix)]
-
-        policies = sorted(policies, key=lambda policy: policy.name)
-        start_idx = int(marker) if marker else 0
-
-        policies = policies[start_idx:start_idx + max_items]
-
-        if len(policies) < max_items:
-            marker = None
-        else:
-            marker = str(start_idx + max_items)
-
-        return policies, marker
+    def list_attached_group_policies(self, group_name, marker=None, max_items=100, path_prefix='/'):
+        policies = self.get_group(group_name).managed_policies.values()
+        return self._filter_attached_policies(policies, marker, max_items, path_prefix)
 
     def list_attached_user_policies(self, user_name, marker=None, max_items=100, path_prefix='/'):
         policies = self.get_user(user_name).managed_policies.values()
-
-        if path_prefix:
-            policies = [p for p in policies if p.path.startswith(path_prefix)]
-
-        policies = sorted(policies, key=lambda policy: policy.name)
-        start_idx = int(marker) if marker else 0
-
-        policies = policies[start_idx:start_idx + max_items]
-
-        if len(policies) < max_items:
-            marker = None
-        else:
-            marker = str(start_idx + max_items)
-
-        return policies, marker
+        return self._filter_attached_policies(policies, marker, max_items, path_prefix)
 
     def list_policies(self, marker, max_items, only_attached, path_prefix, scope):
         policies = self.managed_policies.values()
@@ -504,6 +495,9 @@ class IAMBackend(BaseBackend):
             policies = [p for p in policies if not isinstance(
                 p, AWSManagedPolicy)]
 
+        return self._filter_attached_policies(policies, marker, max_items, path_prefix)
+
+    def _filter_attached_policies(self, policies, marker, max_items, path_prefix):
         if path_prefix:
             policies = [p for p in policies if p.path.startswith(path_prefix)]
 

moto/iam/responses.py

@@ -20,6 +20,20 @@ class IamResponse(BaseResponse):
         template = self.response_template(GENERIC_EMPTY_TEMPLATE)
         return template.render(name="DetachRolePolicyResponse")
 
+    def attach_group_policy(self):
+        policy_arn = self._get_param('PolicyArn')
+        group_name = self._get_param('GroupName')
+        iam_backend.attach_group_policy(policy_arn, group_name)
+        template = self.response_template(ATTACH_GROUP_POLICY_TEMPLATE)
+        return template.render()
+
+    def detach_group_policy(self):
+        policy_arn = self._get_param('PolicyArn')
+        group_name = self._get_param('GroupName')
+        iam_backend.detach_group_policy(policy_arn, group_name)
+        template = self.response_template(DETACH_GROUP_POLICY_TEMPLATE)
+        return template.render()
+
     def attach_user_policy(self):
         policy_arn = self._get_param('PolicyArn')
         user_name = self._get_param('UserName')
@@ -54,6 +68,17 @@ class IamResponse(BaseResponse):
         template = self.response_template(LIST_ATTACHED_ROLE_POLICIES_TEMPLATE)
         return template.render(policies=policies, marker=marker)
 
+    def list_attached_group_policies(self):
+        marker = self._get_param('Marker')
+        max_items = self._get_int_param('MaxItems', 100)
+        path_prefix = self._get_param('PathPrefix', '/')
+        group_name = self._get_param('GroupName')
+        policies, marker = iam_backend.list_attached_group_policies(
+            group_name, marker=marker, max_items=max_items,
+            path_prefix=path_prefix)
+        template = self.response_template(LIST_ATTACHED_GROUP_POLICIES_TEMPLATE)
+        return template.render(policies=policies, marker=marker)
+
     def list_attached_user_policies(self):
         marker = self._get_param('Marker')
         max_items = self._get_int_param('MaxItems', 100)
@@ -520,6 +545,18 @@ DETACH_USER_POLICY_TEMPLATE = """<DetachUserPolicyResponse>
   </ResponseMetadata>
 </DetachUserPolicyResponse>"""
 
+ATTACH_GROUP_POLICY_TEMPLATE = """<AttachGroupPolicyResponse>
+  <ResponseMetadata>
+    <RequestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestId>
+  </ResponseMetadata>
+</AttachGroupPolicyResponse>"""
+
+DETACH_GROUP_POLICY_TEMPLATE = """<DetachGroupPolicyResponse>
+  <ResponseMetadata>
+    <RequestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestId>
+  </ResponseMetadata>
+</DetachGroupPolicyResponse>"""
+
 CREATE_POLICY_TEMPLATE = """<CreatePolicyResponse>
   <CreatePolicyResult>
     <Policy>
@@ -560,6 +597,28 @@ LIST_ATTACHED_ROLE_POLICIES_TEMPLATE = """<ListAttachedRolePoliciesResponse>
   </ResponseMetadata>
 </ListAttachedRolePoliciesResponse>"""
 
+LIST_ATTACHED_GROUP_POLICIES_TEMPLATE = """<ListAttachedGroupPoliciesResponse>
+  <ListAttachedGroupPoliciesResult>
+    {% if marker is none %}
+    <IsTruncated>false</IsTruncated>
+    {% else %}
+    <IsTruncated>true</IsTruncated>
+    <Marker>{{ marker }}</Marker>
+    {% endif %}
+    <AttachedPolicies>
+      {% for policy in policies %}
+      <member>
+        <PolicyName>{{ policy.name }}</PolicyName>
+        <PolicyArn>{{ policy.arn }}</PolicyArn>
+      </member>
+      {% endfor %}
+    </AttachedPolicies>
+  </ListAttachedGroupPoliciesResult>
+  <ResponseMetadata>
+    <RequestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestId>
+  </ResponseMetadata>
+</ListAttachedGroupPoliciesResponse>"""
+
 LIST_ATTACHED_USER_POLICIES_TEMPLATE = """<ListAttachedUserPoliciesResponse>
   <ListAttachedUserPoliciesResult>
     {% if marker is none %}

moto/redshift/responses.py

@@ -2,7 +2,8 @@ from __future__ import unicode_literals
 
 import json
 
-import dicttoxml
+import xmltodict
+
 from jinja2 import Template
 from six import iteritems
 
@@ -26,6 +27,24 @@ def convert_json_error_to_xml(json_error):
     return template.render(code=code, message=message)
 
 
+def itemize(data):
+    """
+    The xmltodict.unparse requires we modify the shape of the input dictionary slightly. Instead of a dict of the form:
+        {'key': ['value1', 'value2']}
+    We must provide:
+        {'key': {'item': ['value1', 'value2']}}
+    """
+    if isinstance(data, dict):
+        ret = {}
+        for key in data:
+            ret[key] = itemize(data[key])
+        return ret
+    elif isinstance(data, list):
+        return {'item': [itemize(value) for value in data]}
+    else:
+        return data
+
+
 class RedshiftResponse(BaseResponse):
 
     @property
@@ -36,8 +55,10 @@ class RedshiftResponse(BaseResponse):
         if self.request_json:
             return json.dumps(response)
         else:
-            xml = dicttoxml.dicttoxml(response, attr_type=False, root=False)
-            return xml.decode("utf-8")
+            xml = xmltodict.unparse(itemize(response), full_document=False)
+            if hasattr(xml, 'decode'):
+                xml = xml.decode('utf-8')
+            return xml
 
     def call_action(self):
         status, headers, body = super(RedshiftResponse, self).call_action()
@@ -66,6 +87,24 @@ class RedshiftResponse(BaseResponse):
             count += 1
         return unpacked_list
 
+    def _get_cluster_security_groups(self):
+        cluster_security_groups = self._get_multi_param('ClusterSecurityGroups.member')
+        if not cluster_security_groups:
+            cluster_security_groups = self._get_multi_param('ClusterSecurityGroups.ClusterSecurityGroupName')
+        return cluster_security_groups
+
+    def _get_vpc_security_group_ids(self):
+        vpc_security_group_ids = self._get_multi_param('VpcSecurityGroupIds.member')
+        if not vpc_security_group_ids:
+            vpc_security_group_ids = self._get_multi_param('VpcSecurityGroupIds.VpcSecurityGroupId')
+        return vpc_security_group_ids
+
+    def _get_subnet_ids(self):
+        subnet_ids = self._get_multi_param('SubnetIds.member')
+        if not subnet_ids:
+            subnet_ids = self._get_multi_param('SubnetIds.SubnetIdentifier')
+        return subnet_ids
+
     def create_cluster(self):
         cluster_kwargs = {
             "cluster_identifier": self._get_param('ClusterIdentifier'),
@@ -74,8 +113,8 @@ class RedshiftResponse(BaseResponse):
             "master_user_password": self._get_param('MasterUserPassword'),
             "db_name": self._get_param('DBName'),
             "cluster_type": self._get_param('ClusterType'),
-            "cluster_security_groups": self._get_multi_param('ClusterSecurityGroups.member'),
-            "vpc_security_group_ids": self._get_multi_param('VpcSecurityGroupIds.member'),
+            "cluster_security_groups": self._get_cluster_security_groups(),
+            "vpc_security_group_ids": self._get_vpc_security_group_ids(),
             "cluster_subnet_group_name": self._get_param('ClusterSubnetGroupName'),
             "availability_zone": self._get_param('AvailabilityZone'),
             "preferred_maintenance_window": self._get_param('PreferredMaintenanceWindow'),
@@ -116,10 +155,8 @@ class RedshiftResponse(BaseResponse):
             "publicly_accessible": self._get_param("PubliclyAccessible"),
             "cluster_parameter_group_name": self._get_param(
                 'ClusterParameterGroupName'),
-            "cluster_security_groups": self._get_multi_param(
-                'ClusterSecurityGroups.member'),
-            "vpc_security_group_ids": self._get_multi_param(
-                'VpcSecurityGroupIds.member'),
+            "cluster_security_groups": self._get_cluster_security_groups(),
+            "vpc_security_group_ids": self._get_vpc_security_group_ids(),
             "preferred_maintenance_window": self._get_param(
                 'PreferredMaintenanceWindow'),
             "automated_snapshot_retention_period": self._get_int_param(
@@ -161,8 +198,8 @@ class RedshiftResponse(BaseResponse):
             "node_type": self._get_param('NodeType'),
             "master_user_password": self._get_param('MasterUserPassword'),
             "cluster_type": self._get_param('ClusterType'),
-            "cluster_security_groups": self._get_multi_param('ClusterSecurityGroups.member'),
-            "vpc_security_group_ids": self._get_multi_param('VpcSecurityGroupIds.member'),
+            "cluster_security_groups": self._get_cluster_security_groups(),
+            "vpc_security_group_ids": self._get_vpc_security_group_ids(),
             "cluster_subnet_group_name": self._get_param('ClusterSubnetGroupName'),
             "preferred_maintenance_window": self._get_param('PreferredMaintenanceWindow'),
             "cluster_parameter_group_name": self._get_param('ClusterParameterGroupName'),
@@ -173,12 +210,6 @@ class RedshiftResponse(BaseResponse):
             "publicly_accessible": self._get_param("PubliclyAccessible"),
             "encrypted": self._get_param("Encrypted"),
         }
-        # There's a bug in boto3 where the security group ids are not passed
-        # according to the AWS documentation
-        if not request_kwargs['vpc_security_group_ids']:
-            request_kwargs['vpc_security_group_ids'] = self._get_multi_param(
-                'VpcSecurityGroupIds.VpcSecurityGroupId')
-
         cluster_kwargs = {}
         # We only want parameters that were actually passed in, otherwise
         # we'll stomp all over our cluster metadata with None values.
@@ -217,11 +248,7 @@ class RedshiftResponse(BaseResponse):
     def create_cluster_subnet_group(self):
         cluster_subnet_group_name = self._get_param('ClusterSubnetGroupName')
         description = self._get_param('Description')
-        subnet_ids = self._get_multi_param('SubnetIds.member')
-        # There's a bug in boto3 where the subnet ids are not passed
-        # according to the AWS documentation
-        if not subnet_ids:
-            subnet_ids = self._get_multi_param('SubnetIds.SubnetIdentifier')
+        subnet_ids = self._get_subnet_ids()
         tags = self.unpack_complex_list_params('Tags.Tag', ('Key', 'Value'))
 
         subnet_group = self.redshift_backend.create_cluster_subnet_group(