From 6ab416724a594e545af42e14a21a4b00db7e38d7 Mon Sep 17 00:00:00 2001 From: andrew Date: Thu, 22 Feb 2018 14:58:19 -0500 Subject: [PATCH 1/4] WIP: add iam roles to redshift --- moto/redshift/models.py | 5 ++++- moto/redshift/responses.py | 11 ++++++++++- tests/test_redshift/test_redshift.py | 18 ++++++++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) diff --git a/moto/redshift/models.py b/moto/redshift/models.py index fa642ef0..7062b521 100644 --- a/moto/redshift/models.py +++ b/moto/redshift/models.py @@ -67,7 +67,7 @@ class Cluster(TaggableResourceMixin, BaseModel): preferred_maintenance_window, cluster_parameter_group_name, automated_snapshot_retention_period, port, cluster_version, allow_version_upgrade, number_of_nodes, publicly_accessible, - encrypted, region_name, tags=None): + encrypted, region_name, tags=None, iam_roles=None): super(Cluster, self).__init__(region_name, tags) self.redshift_backend = redshift_backend self.cluster_identifier = cluster_identifier @@ -112,6 +112,9 @@ class Cluster(TaggableResourceMixin, BaseModel): else: self.number_of_nodes = 1 + if iam_roles: + self.iam_roles = iam_roles + @classmethod def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): redshift_backend = redshift_backends[region_name] diff --git a/moto/redshift/responses.py b/moto/redshift/responses.py index a320f9ca..54cd5174 100644 --- a/moto/redshift/responses.py +++ b/moto/redshift/responses.py @@ -99,6 +99,12 @@ class RedshiftResponse(BaseResponse): vpc_security_group_ids = self._get_multi_param('VpcSecurityGroupIds.VpcSecurityGroupId') return vpc_security_group_ids + def _get_iam_roles(self): + iam_roles = self._get_multi_param('IamRoles.member') + if not iam_roles: + iam_roles = self._get_multi_param('IamRoles.IamRoleArn') + return iam_roles + def _get_subnet_ids(self): subnet_ids = self._get_multi_param('SubnetIds.member') if not subnet_ids: @@ -127,7 +133,8 @@ class RedshiftResponse(BaseResponse): "publicly_accessible": self._get_param("PubliclyAccessible"), "encrypted": self._get_param("Encrypted"), "region_name": self.region, - "tags": self.unpack_complex_list_params('Tags.Tag', ('Key', 'Value')) + "tags": self.unpack_complex_list_params('Tags.Tag', ('Key', 'Value')), + "iam_roles": self._get_iam_roles(), } cluster = self.redshift_backend.create_cluster(**cluster_kwargs).to_json() cluster['ClusterStatus'] = 'creating' @@ -162,6 +169,7 @@ class RedshiftResponse(BaseResponse): "automated_snapshot_retention_period": self._get_int_param( 'AutomatedSnapshotRetentionPeriod'), "region_name": self.region, + "iam_roles": self._get_iam_roles(), } cluster = self.redshift_backend.restore_from_cluster_snapshot(**restore_kwargs).to_json() cluster['ClusterStatus'] = 'creating' @@ -209,6 +217,7 @@ class RedshiftResponse(BaseResponse): "number_of_nodes": self._get_int_param('NumberOfNodes'), "publicly_accessible": self._get_param("PubliclyAccessible"), "encrypted": self._get_param("Encrypted"), + "iam_roles": self._get_iam_roles(), } cluster_kwargs = {} # We only want parameters that were actually passed in, otherwise diff --git a/tests/test_redshift/test_redshift.py b/tests/test_redshift/test_redshift.py index cebaa3ec..3267b3ac 100644 --- a/tests/test_redshift/test_redshift.py +++ b/tests/test_redshift/test_redshift.py @@ -294,6 +294,24 @@ def test_create_cluster_with_vpc_security_groups_boto3(): list(group_ids).should.equal([security_group.id]) +@mock_redshift +def test_create_cluster_with_iam_roles(): + iam_role = 'arn:aws:iam:::role/my-iam-role' + client = boto3.client('redshift', region_name='us-east-1') + cluster_id = 'my_cluster' + client.create_cluster( + ClusterIdentifier=cluster_id, + NodeType="dw.hs1.xlarge", + MasterUsername="username", + MasterUserPassword="password", + IamRoles=[iam_role], + ) + response = client.describe_clusters(ClusterIdentifier=cluster_id) + cluster = response['Clusters'][0] + iam_roles = [role['IamRoleArn'] for role in cluster['IamRoles']] + list(iam_roles).should.equal([iam_role.arn]) + + @mock_redshift_deprecated def test_create_cluster_with_parameter_group(): conn = boto.connect_redshift() From 7d3af65f2233770a4dfb2ca46c127959bb53505d Mon Sep 17 00:00:00 2001 From: andrew Date: Fri, 23 Feb 2018 14:42:49 -0500 Subject: [PATCH 2/4] fix errors --- moto/redshift/models.py | 27 +++++++++++++++++++++------ moto/redshift/responses.py | 6 +++--- tests/test_redshift/test_redshift.py | 6 +++--- 3 files changed, 27 insertions(+), 12 deletions(-) diff --git a/moto/redshift/models.py b/moto/redshift/models.py index 7062b521..a7096d95 100644 --- a/moto/redshift/models.py +++ b/moto/redshift/models.py @@ -8,6 +8,7 @@ from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel from moto.core.utils import iso_8601_datetime_with_milliseconds from moto.ec2 import ec2_backends +from moto.iam import iam_backends from .exceptions import ( ClusterNotFoundError, ClusterParameterGroupNotFoundError, @@ -67,7 +68,7 @@ class Cluster(TaggableResourceMixin, BaseModel): preferred_maintenance_window, cluster_parameter_group_name, automated_snapshot_retention_period, port, cluster_version, allow_version_upgrade, number_of_nodes, publicly_accessible, - encrypted, region_name, tags=None, iam_roles=None): + encrypted, region_name, tags=None, iam_roles_arn=[]): super(Cluster, self).__init__(region_name, tags) self.redshift_backend = redshift_backend self.cluster_identifier = cluster_identifier @@ -112,8 +113,7 @@ class Cluster(TaggableResourceMixin, BaseModel): else: self.number_of_nodes = 1 - if iam_roles: - self.iam_roles = iam_roles + self.iam_roles_arn = iam_roles_arn @classmethod def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): @@ -196,6 +196,14 @@ class Cluster(TaggableResourceMixin, BaseModel): def resource_id(self): return self.cluster_identifier + @property + def iam_roles(self): + return [ + iam_role for iam_role + in self.redshift_backend.iam_backend.get_roles() + if iam_role.arn in self.iam_roles_arn + ] + def to_json(self): return { "MasterUsername": self.master_username, @@ -231,7 +239,11 @@ class Cluster(TaggableResourceMixin, BaseModel): "Port": self.port }, "PendingModifiedValues": [], - "Tags": self.tags + "Tags": self.tags, + "IamRoles": [{ + "ApplyStatus": "in-sync", + "IamRoleArn": iam_role.arn + } for iam_role in self.iam_roles] } @@ -354,7 +366,7 @@ class Snapshot(TaggableResourceMixin, BaseModel): resource_type = 'snapshot' - def __init__(self, cluster, snapshot_identifier, region_name, tags=None): + def __init__(self, cluster, snapshot_identifier, region_name, tags=None, iam_roles_arn=[]): super(Snapshot, self).__init__(region_name, tags) self.cluster = copy.copy(cluster) self.snapshot_identifier = snapshot_identifier @@ -362,6 +374,7 @@ class Snapshot(TaggableResourceMixin, BaseModel): self.status = 'available' self.create_time = iso_8601_datetime_with_milliseconds( datetime.datetime.now()) + self.iam_roles_arn = iam_roles_arn @property def resource_id(self): @@ -383,7 +396,8 @@ class Snapshot(TaggableResourceMixin, BaseModel): 'NodeType': self.cluster.node_type, 'NumberOfNodes': self.cluster.number_of_nodes, 'DBName': self.cluster.db_name, - 'Tags': self.tags + 'Tags': self.tags, + 'IamRoles': self.iam_roles_arn } @@ -413,6 +427,7 @@ class RedshiftBackend(BaseBackend): 'snapshot': self.snapshots, 'subnetgroup': self.subnet_groups } + self.iam_backend = iam_backends['global'] def reset(self): ec2_backend = self.ec2_backend diff --git a/moto/redshift/responses.py b/moto/redshift/responses.py index 54cd5174..34f116bd 100644 --- a/moto/redshift/responses.py +++ b/moto/redshift/responses.py @@ -134,7 +134,7 @@ class RedshiftResponse(BaseResponse): "encrypted": self._get_param("Encrypted"), "region_name": self.region, "tags": self.unpack_complex_list_params('Tags.Tag', ('Key', 'Value')), - "iam_roles": self._get_iam_roles(), + "iam_roles_arn": self._get_iam_roles(), } cluster = self.redshift_backend.create_cluster(**cluster_kwargs).to_json() cluster['ClusterStatus'] = 'creating' @@ -169,7 +169,7 @@ class RedshiftResponse(BaseResponse): "automated_snapshot_retention_period": self._get_int_param( 'AutomatedSnapshotRetentionPeriod'), "region_name": self.region, - "iam_roles": self._get_iam_roles(), + "iam_roles_arn": self._get_iam_roles(), } cluster = self.redshift_backend.restore_from_cluster_snapshot(**restore_kwargs).to_json() cluster['ClusterStatus'] = 'creating' @@ -217,7 +217,7 @@ class RedshiftResponse(BaseResponse): "number_of_nodes": self._get_int_param('NumberOfNodes'), "publicly_accessible": self._get_param("PubliclyAccessible"), "encrypted": self._get_param("Encrypted"), - "iam_roles": self._get_iam_roles(), + "iam_roles_arn": self._get_iam_roles(), } cluster_kwargs = {} # We only want parameters that were actually passed in, otherwise diff --git a/tests/test_redshift/test_redshift.py b/tests/test_redshift/test_redshift.py index 3267b3ac..b617ac79 100644 --- a/tests/test_redshift/test_redshift.py +++ b/tests/test_redshift/test_redshift.py @@ -296,7 +296,7 @@ def test_create_cluster_with_vpc_security_groups_boto3(): @mock_redshift def test_create_cluster_with_iam_roles(): - iam_role = 'arn:aws:iam:::role/my-iam-role' + iam_roles_arn = ['arn:aws:iam:::role/my-iam-role',] client = boto3.client('redshift', region_name='us-east-1') cluster_id = 'my_cluster' client.create_cluster( @@ -304,12 +304,12 @@ def test_create_cluster_with_iam_roles(): NodeType="dw.hs1.xlarge", MasterUsername="username", MasterUserPassword="password", - IamRoles=[iam_role], + IamRoles=iam_roles_arn ) response = client.describe_clusters(ClusterIdentifier=cluster_id) cluster = response['Clusters'][0] iam_roles = [role['IamRoleArn'] for role in cluster['IamRoles']] - list(iam_roles).should.equal([iam_role.arn]) + iam_roles_arn.should.equal(iam_roles) @mock_redshift_deprecated From 894906e0ee8332c6e9dc3d78c5e7da016b5c70ae Mon Sep 17 00:00:00 2001 From: andrew Date: Fri, 23 Feb 2018 14:52:22 -0500 Subject: [PATCH 3/4] remove reliance on iam_backends --- moto/redshift/models.py | 19 ++++++------------- 1 file changed, 6 insertions(+), 13 deletions(-) diff --git a/moto/redshift/models.py b/moto/redshift/models.py index a7096d95..9a5850fa 100644 --- a/moto/redshift/models.py +++ b/moto/redshift/models.py @@ -8,7 +8,6 @@ from moto.compat import OrderedDict from moto.core import BaseBackend, BaseModel from moto.core.utils import iso_8601_datetime_with_milliseconds from moto.ec2 import ec2_backends -from moto.iam import iam_backends from .exceptions import ( ClusterNotFoundError, ClusterParameterGroupNotFoundError, @@ -196,14 +195,6 @@ class Cluster(TaggableResourceMixin, BaseModel): def resource_id(self): return self.cluster_identifier - @property - def iam_roles(self): - return [ - iam_role for iam_role - in self.redshift_backend.iam_backend.get_roles() - if iam_role.arn in self.iam_roles_arn - ] - def to_json(self): return { "MasterUsername": self.master_username, @@ -242,8 +233,8 @@ class Cluster(TaggableResourceMixin, BaseModel): "Tags": self.tags, "IamRoles": [{ "ApplyStatus": "in-sync", - "IamRoleArn": iam_role.arn - } for iam_role in self.iam_roles] + "IamRoleArn": iam_role_arn + } for iam_role_arn in self.iam_roles_arn] } @@ -397,7 +388,10 @@ class Snapshot(TaggableResourceMixin, BaseModel): 'NumberOfNodes': self.cluster.number_of_nodes, 'DBName': self.cluster.db_name, 'Tags': self.tags, - 'IamRoles': self.iam_roles_arn + "IamRoles": [{ + "ApplyStatus": "in-sync", + "IamRoleArn": iam_role_arn + } for iam_role_arn in self.iam_roles_arn] } @@ -427,7 +421,6 @@ class RedshiftBackend(BaseBackend): 'snapshot': self.snapshots, 'subnetgroup': self.subnet_groups } - self.iam_backend = iam_backends['global'] def reset(self): ec2_backend = self.ec2_backend From e2e1c7347b3421adec88e76afbb89b1cad6783ea Mon Sep 17 00:00:00 2001 From: andrew Date: Wed, 7 Mar 2018 08:38:07 -0500 Subject: [PATCH 4/4] default to None --- moto/redshift/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/moto/redshift/models.py b/moto/redshift/models.py index 9a5850fa..c8a4e353 100644 --- a/moto/redshift/models.py +++ b/moto/redshift/models.py @@ -67,7 +67,7 @@ class Cluster(TaggableResourceMixin, BaseModel): preferred_maintenance_window, cluster_parameter_group_name, automated_snapshot_retention_period, port, cluster_version, allow_version_upgrade, number_of_nodes, publicly_accessible, - encrypted, region_name, tags=None, iam_roles_arn=[]): + encrypted, region_name, tags=None, iam_roles_arn=None): super(Cluster, self).__init__(region_name, tags) self.redshift_backend = redshift_backend self.cluster_identifier = cluster_identifier @@ -112,7 +112,7 @@ class Cluster(TaggableResourceMixin, BaseModel): else: self.number_of_nodes = 1 - self.iam_roles_arn = iam_roles_arn + self.iam_roles_arn = iam_roles_arn or [] @classmethod def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name): @@ -357,7 +357,7 @@ class Snapshot(TaggableResourceMixin, BaseModel): resource_type = 'snapshot' - def __init__(self, cluster, snapshot_identifier, region_name, tags=None, iam_roles_arn=[]): + def __init__(self, cluster, snapshot_identifier, region_name, tags=None, iam_roles_arn=None): super(Snapshot, self).__init__(region_name, tags) self.cluster = copy.copy(cluster) self.snapshot_identifier = snapshot_identifier @@ -365,7 +365,7 @@ class Snapshot(TaggableResourceMixin, BaseModel): self.status = 'available' self.create_time = iso_8601_datetime_with_milliseconds( datetime.datetime.now()) - self.iam_roles_arn = iam_roles_arn + self.iam_roles_arn = iam_roles_arn or [] @property def resource_id(self):