From 0ea98233974b86a8ffeaae59eba031ba4b36a9eb Mon Sep 17 00:00:00 2001 From: "Riccardo M. Cefala" Date: Sat, 11 Jun 2016 12:48:01 +0200 Subject: [PATCH 01/10] add generate_instance_identity_document in ec2 utils --- moto/ec2/utils.py | 32 ++++++++++++++++++++++++++++++++ 1 file changed, 32 insertions(+) diff --git a/moto/ec2/utils.py b/moto/ec2/utils.py index 5c3fcac1..e0736708 100644 --- a/moto/ec2/utils.py +++ b/moto/ec2/utils.py @@ -515,3 +515,35 @@ def is_valid_cidr(cird): cidr_pattern = '^(([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])\.){3}([0-9]|[1-9][0-9]|1[0-9]{2}|2[0-4][0-9]|25[0-5])(\/(\d|[1-2]\d|3[0-2]))$' cidr_pattern_re = re.compile(cidr_pattern) return cidr_pattern_re.match(cird) is not None + + +def generate_instance_identity_document(instance): + """ + http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instance-identity-documents.html + + A JSON file that describes an instance. Usually retrieved by URL: + http://169.254.169.254/latest/dynamic/instance-identity/document + Here we just fill a dictionary that represents the document + + Typically, this document is used by the amazon-ecs-agent when registering a + new ContainerInstance + """ + + document = { + 'devPayProductCodes': None, + 'availabilityZone': instance.placement['AvailabilityZone'], + 'privateIp': instance.private_ip_address, + 'version': '2010-8-31', + 'region': instance.placement['AvailabilityZone'][:-1], + 'instanceId': instance.id, + 'billingProducts': None, + 'instanceType': instance.instance_type, + 'accountId': '012345678910', + 'pendingTime': '2015-11-19T16:32:11Z', + 'imageId': instance.image_id, + 'kernelId': instance.kernel_id, + 'ramdiskId': instance.ramdisk_id, + 'architecture': instance.architecture, + } + + return document From 19fab4ca2536cc651e87fd0c5e9ba52b44c9369a Mon Sep 17 00:00:00 2001 From: "Riccardo M. Cefala" Date: Sat, 11 Jun 2016 12:49:08 +0200 Subject: [PATCH 02/10] add ContainerInstance class in ecs models --- moto/ecs/models.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/moto/ecs/models.py b/moto/ecs/models.py index d62d7ffa..f8f8f201 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -85,6 +85,29 @@ class Service(BaseObject): return response_object +class ContainerInstance(BaseObject): + def __init__(self, ec2_instance_id): + self.ec2_instance_id = ec2_instance_id + self.status = 'ACTIVE' + self.registeredResources = [] + self.agentConnected = True + self.containerInstanceArn = "arn:aws:ecs:us-east-1:012345678910:container-instance/{0}".format(str(uuid.uuid1())) + self.pendingTaskCount = 0 + self.remainingResources = [] + self.runningTaskCount = 0 + self.versionInfo = { + 'agentVersion': "1.0.0", + 'agentHash': '4023248', + 'dockerVersion': 'DockerVersion: 1.5.0' + } + + @property + def response_object(self): + response_object = self.gen_response_object() + del response_object['name'], response_object['arn'] + return response_object + + class EC2ContainerServiceBackend(BaseBackend): def __init__(self): self.clusters = {} From 115f9513f6ed1c2d5472493dde7fa66f9a966e7c Mon Sep 17 00:00:00 2001 From: "Riccardo M. Cefala" Date: Sat, 11 Jun 2016 12:52:19 +0200 Subject: [PATCH 03/10] add ECS ContainerInstance register and list actions --- moto/ecs/models.py | 23 +++++++++++++++++++++++ moto/ecs/responses.py | 17 +++++++++++++++++ 2 files changed, 40 insertions(+) diff --git a/moto/ecs/models.py b/moto/ecs/models.py index f8f8f201..45ced304 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -113,6 +113,7 @@ class EC2ContainerServiceBackend(BaseBackend): self.clusters = {} self.task_definitions = {} self.services = {} + self.container_instances = {} def fetch_task_definition(self, task_definition_str): task_definition_components = task_definition_str.split(':') @@ -235,6 +236,28 @@ class EC2ContainerServiceBackend(BaseBackend): else: raise Exception("cluster {0} or service {1} does not exist".format(cluster_name, service_name)) + def register_container_instance(self, cluster_str, ec2_instance_id): + cluster_name = cluster_str.split('/')[-1] + if cluster_name in self.clusters: + cluster = self.clusters[cluster_name] + else: + raise Exception("{0} is not a cluster".format(cluster.name)) + container_instance = ContainerInstance(ec2_instance_id) + if not self.container_instances.get(cluster_name): + self.container_instances[cluster_name] = {} + self.container_instances[cluster_name][container_instance.containerInstanceArn] = container_instance + return container_instance + + def list_container_instances(self, cluster_str): + cluster_name = cluster_str.split('/')[-1] + return sorted(self.container_instances[cluster_name].keys()) + + def describe_container_instances(self, cluster_str, list_container_instances_str): + pass + + def deregister_container_instance(self, cluster_str, container_instance_str): + pass + ecs_backends = {} for region, ec2_backend in ec2_backends.items(): diff --git a/moto/ecs/responses.py b/moto/ecs/responses.py index 1cf88bd4..0adf7813 100644 --- a/moto/ecs/responses.py +++ b/moto/ecs/responses.py @@ -113,3 +113,20 @@ class EC2ContainerServiceResponse(BaseResponse): return json.dumps({ 'service': service.response_object }) + + def register_container_instance(self): + cluster_str = self._get_param('cluster') + instance_identity_document_str = self._get_param('instanceIdentityDocument') + instance_identity_document = json.loads(instance_identity_document_str) + ec2_instance_id = instance_identity_document["instanceId"] + container_instance = self.ecs_backend.register_container_instance(cluster_str, ec2_instance_id) + return json.dumps({ + 'containerInstance' : container_instance.response_object + }) + + def list_container_instances(self): + cluster_str = self._get_param('cluster') + container_instance_arns = self.ecs_backend.list_container_instances(cluster_str) + return json.dumps({ + 'containerInstanceArns': container_instance_arns + }) From 262bf07608e518ef0cf9df5b843b8835c83c504f Mon Sep 17 00:00:00 2001 From: "Riccardo M. Cefala" Date: Sat, 11 Jun 2016 12:52:53 +0200 Subject: [PATCH 04/10] add tests for ECS ContainerInstance list and register actions --- tests/test_ecs/test_ecs_boto3.py | 82 +++++++++++++++++++++++++++++++- 1 file changed, 81 insertions(+), 1 deletion(-) diff --git a/tests/test_ecs/test_ecs_boto3.py b/tests/test_ecs/test_ecs_boto3.py index e902c5d9..d671fde7 100644 --- a/tests/test_ecs/test_ecs_boto3.py +++ b/tests/test_ecs/test_ecs_boto3.py @@ -1,8 +1,12 @@ from __future__ import unicode_literals import boto3 import sure # noqa +import json +from moto.ec2 import utils as ec2_utils +from uuid import UUID from moto import mock_ecs +from moto import mock_ec2 @mock_ecs @@ -333,4 +337,80 @@ def test_delete_service(): response['service']['serviceArn'].should.equal('arn:aws:ecs:us-east-1:012345678910:service/test_ecs_service') response['service']['serviceName'].should.equal('test_ecs_service') response['service']['status'].should.equal('ACTIVE') - response['service']['taskDefinition'].should.equal('arn:aws:ecs:us-east-1:012345678910:task-definition/test_ecs_task:1') \ No newline at end of file + response['service']['taskDefinition'].should.equal('arn:aws:ecs:us-east-1:012345678910:task-definition/test_ecs_task:1') + +@mock_ec2 +@mock_ecs +def test_register_container_instance(): + ecs_client = boto3.client('ecs', region_name='us-east-1') + ec2 = boto3.resource('ec2', region_name='us-east-1') + + test_cluster_name = 'test_ecs_cluster' + + _ = ecs_client.create_cluster( + clusterName=test_cluster_name + ) + + test_instance = ec2.create_instances( + ImageId="ami-1234abcd", + MinCount=1, + MaxCount=1, + )[0] + + instance_id_document = json.dumps( + ec2_utils.generate_instance_identity_document(test_instance) + ) + + response = ecs_client.register_container_instance( + cluster=test_cluster_name, + instanceIdentityDocument=instance_id_document + ) + + response['containerInstance']['ec2InstanceId'].should.equal(test_instance.id) + full_arn = response['containerInstance']['containerInstanceArn'] + arn_part = full_arn.split('/') + arn_part[0].should.equal('arn:aws:ecs:us-east-1:012345678910:container-instance') + arn_part[1].should.equal(str(UUID(arn_part[1]))) + response['containerInstance']['status'].should.equal('ACTIVE') + len(response['containerInstance']['registeredResources']).should.equal(0) + len(response['containerInstance']['remainingResources']).should.equal(0) + response['containerInstance']['agentConnected'].should.equal(True) + response['containerInstance']['versionInfo']['agentVersion'].should.equal('1.0.0') + response['containerInstance']['versionInfo']['agentHash'].should.equal('4023248') + response['containerInstance']['versionInfo']['dockerVersion'].should.equal('DockerVersion: 1.5.0') + +@mock_ec2 +@mock_ecs +def test_list_container_instances(): + ecs_client = boto3.client('ecs', region_name='us-east-1') + ec2 = boto3.resource('ec2', region_name='us-east-1') + + test_cluster_name = 'test_ecs_cluster' + _ = ecs_client.create_cluster( + clusterName=test_cluster_name + ) + + instance_to_create = 3 + test_instance_arns = [] + for i in range(0, instance_to_create): + test_instance = ec2.create_instances( + ImageId="ami-1234abcd", + MinCount=1, + MaxCount=1, + )[0] + + instance_id_document = json.dumps( + ec2_utils.generate_instance_identity_document(test_instance) + ) + + response = ecs_client.register_container_instance( + cluster=test_cluster_name, + instanceIdentityDocument=instance_id_document) + + test_instance_arns.append(response['containerInstance']['containerInstanceArn']) + + response = ecs_client.list_container_instances(cluster=test_cluster_name) + + len(response['containerInstanceArns']).should.equal(instance_to_create) + for arn in test_instance_arns: + response['containerInstanceArns'].should.contain(arn) From 137791e9606b87ef0de31f5cdac0ef016d799e20 Mon Sep 17 00:00:00 2001 From: "Riccardo M. Cefala" Date: Tue, 14 Jun 2016 17:58:11 +0200 Subject: [PATCH 05/10] add ECS describe_container_instances --- moto/ecs/models.py | 30 ++++++++++++++++++++++-- moto/ecs/responses.py | 9 ++++++++ tests/test_ecs/test_ecs_boto3.py | 39 ++++++++++++++++++++++++++++++++ 3 files changed, 76 insertions(+), 2 deletions(-) diff --git a/moto/ecs/models.py b/moto/ecs/models.py index 45ced304..30d7c21e 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -108,6 +108,19 @@ class ContainerInstance(BaseObject): return response_object +class Failure(BaseObject): + def __init__(self, reason, arn): + self.reason = reason + self.arn = arn + + @property + def response_object(self): + response_object = self.gen_response_object() + response_object['reason'] = self.reason + response_object['arn'] = self.arn + return response_object + + class EC2ContainerServiceBackend(BaseBackend): def __init__(self): self.clusters = {} @@ -252,8 +265,21 @@ class EC2ContainerServiceBackend(BaseBackend): cluster_name = cluster_str.split('/')[-1] return sorted(self.container_instances[cluster_name].keys()) - def describe_container_instances(self, cluster_str, list_container_instances_str): - pass + def describe_container_instances(self, cluster_str, list_container_instance_ids): + cluster_name = cluster_str.split('/')[-1] + if cluster_name not in self.clusters: + raise Exception("{0} is not a cluster".format(cluster_name)) + failures = [] + container_instance_objects = [] + for container_instance_id in list_container_instance_ids: + container_instance_arn = 'arn:aws:ecs:us-east-1:012345678910:container-instance/{0}'.format(container_instance_id) + container_instance = self.container_instances[cluster_name].get(container_instance_arn, None) + if container_instance is not None: + container_instance_objects.append(container_instance) + else: + failures.append(Failure(reason='MISSING', arn=container_instance_arn)) + + return container_instance_objects, failures def deregister_container_instance(self, cluster_str, container_instance_str): pass diff --git a/moto/ecs/responses.py b/moto/ecs/responses.py index 0adf7813..b9770a43 100644 --- a/moto/ecs/responses.py +++ b/moto/ecs/responses.py @@ -130,3 +130,12 @@ class EC2ContainerServiceResponse(BaseResponse): return json.dumps({ 'containerInstanceArns': container_instance_arns }) + + def describe_container_instances(self): + cluster_str = self._get_param('cluster') + list_container_instance_arns = self._get_param('containerInstances') + container_instances, failures = self.ecs_backend.describe_container_instances(cluster_str, list_container_instance_arns) + return json.dumps({ + 'failures': [ci.response_object for ci in failures], + 'containerInstances': [ci.response_object for ci in container_instances] + }) diff --git a/tests/test_ecs/test_ecs_boto3.py b/tests/test_ecs/test_ecs_boto3.py index d671fde7..9ca367e0 100644 --- a/tests/test_ecs/test_ecs_boto3.py +++ b/tests/test_ecs/test_ecs_boto3.py @@ -414,3 +414,42 @@ def test_list_container_instances(): len(response['containerInstanceArns']).should.equal(instance_to_create) for arn in test_instance_arns: response['containerInstanceArns'].should.contain(arn) + + +@mock_ec2 +@mock_ecs +def test_describe_container_instances(): + ecs_client = boto3.client('ecs', region_name='us-east-1') + ec2 = boto3.resource('ec2', region_name='us-east-1') + + test_cluster_name = 'test_ecs_cluster' + _ = ecs_client.create_cluster( + clusterName=test_cluster_name + ) + + instance_to_create = 3 + test_instance_arns = [] + for i in range(0, instance_to_create): + test_instance = ec2.create_instances( + ImageId="ami-1234abcd", + MinCount=1, + MaxCount=1, + )[0] + + instance_id_document = json.dumps( + ec2_utils.generate_instance_identity_document(test_instance) + ) + + response = ecs_client.register_container_instance( + cluster=test_cluster_name, + instanceIdentityDocument=instance_id_document) + + test_instance_arns.append(response['containerInstance']['containerInstanceArn']) + + test_instance_ids = map((lambda x: x.split('/')[1]), test_instance_arns) + response = ecs_client.describe_container_instances(cluster=test_cluster_name, containerInstances=test_instance_ids) + len(response['failures']).should.equal(0) + len(response['containerInstances']).should.equal(instance_to_create) + response_arns = [ci['containerInstanceArn'] for ci in response['containerInstances']] + for arn in test_instance_arns: + response_arns.should.contain(arn) From b652256c43d2b046348d93428f0d9677dd6f0152 Mon Sep 17 00:00:00 2001 From: "Riccardo M. Cefala" Date: Tue, 14 Jun 2016 17:58:49 +0200 Subject: [PATCH 06/10] simplify cluster existence check in register_container_instance --- moto/ecs/models.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/moto/ecs/models.py b/moto/ecs/models.py index 30d7c21e..29388024 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -251,10 +251,8 @@ class EC2ContainerServiceBackend(BaseBackend): def register_container_instance(self, cluster_str, ec2_instance_id): cluster_name = cluster_str.split('/')[-1] - if cluster_name in self.clusters: - cluster = self.clusters[cluster_name] - else: - raise Exception("{0} is not a cluster".format(cluster.name)) + if cluster_name not in self.clusters: + raise Exception("{0} is not a cluster".format(cluster_name)) container_instance = ContainerInstance(ec2_instance_id) if not self.container_instances.get(cluster_name): self.container_instances[cluster_name] = {} From dd7ae027cc893e6080d6812f74b8fa47fecae603 Mon Sep 17 00:00:00 2001 From: "Riccardo M. Cefala" Date: Wed, 15 Jun 2016 08:40:31 +0200 Subject: [PATCH 07/10] rename class Failure to ContainerInstanceFailure --- moto/ecs/models.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/moto/ecs/models.py b/moto/ecs/models.py index 29388024..124cde5a 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -108,10 +108,10 @@ class ContainerInstance(BaseObject): return response_object -class Failure(BaseObject): - def __init__(self, reason, arn): +class ContainerInstanceFailure(BaseObject): + def __init__(self, reason, container_instance_id): self.reason = reason - self.arn = arn + self.arn = "arn:aws:ecs:us-east-1:012345678910:container-instance/{0}".format(container_instance_id) @property def response_object(self): @@ -275,7 +275,7 @@ class EC2ContainerServiceBackend(BaseBackend): if container_instance is not None: container_instance_objects.append(container_instance) else: - failures.append(Failure(reason='MISSING', arn=container_instance_arn)) + failures.append(ContainerInstanceFailure('MISSING', container_instance_id)) return container_instance_objects, failures From 5f2255833f3a91089e33f7a8fef8f0f9a1876ce3 Mon Sep 17 00:00:00 2001 From: "Riccardo M. Cefala" Date: Wed, 15 Jun 2016 08:41:26 +0200 Subject: [PATCH 08/10] use ids instead of full arn to index ContainerInstances --- moto/ecs/models.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/moto/ecs/models.py b/moto/ecs/models.py index 124cde5a..d9d7184d 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -256,12 +256,15 @@ class EC2ContainerServiceBackend(BaseBackend): container_instance = ContainerInstance(ec2_instance_id) if not self.container_instances.get(cluster_name): self.container_instances[cluster_name] = {} - self.container_instances[cluster_name][container_instance.containerInstanceArn] = container_instance + container_instance_id = container_instance.containerInstanceArn.split('/')[-1] + self.container_instances[cluster_name][container_instance_id] = container_instance return container_instance def list_container_instances(self, cluster_str): cluster_name = cluster_str.split('/')[-1] - return sorted(self.container_instances[cluster_name].keys()) + container_instances_iter = self.container_instances.get(cluster_name, {}).itervalues() + container_instances = [ci.containerInstanceArn for ci in container_instances_iter] + return sorted(container_instances) def describe_container_instances(self, cluster_str, list_container_instance_ids): cluster_name = cluster_str.split('/')[-1] @@ -270,8 +273,7 @@ class EC2ContainerServiceBackend(BaseBackend): failures = [] container_instance_objects = [] for container_instance_id in list_container_instance_ids: - container_instance_arn = 'arn:aws:ecs:us-east-1:012345678910:container-instance/{0}'.format(container_instance_id) - container_instance = self.container_instances[cluster_name].get(container_instance_arn, None) + container_instance = self.container_instances[cluster_name].get(container_instance_id, None) if container_instance is not None: container_instance_objects.append(container_instance) else: From 57f4f4aa9b053ea32896f4ed47e96d9c53baf68a Mon Sep 17 00:00:00 2001 From: "Riccardo M. Cefala" Date: Wed, 15 Jun 2016 09:23:34 +0200 Subject: [PATCH 09/10] itervalues() has been dropped in python 3 --- moto/ecs/models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/moto/ecs/models.py b/moto/ecs/models.py index d9d7184d..d4e89b57 100644 --- a/moto/ecs/models.py +++ b/moto/ecs/models.py @@ -262,8 +262,8 @@ class EC2ContainerServiceBackend(BaseBackend): def list_container_instances(self, cluster_str): cluster_name = cluster_str.split('/')[-1] - container_instances_iter = self.container_instances.get(cluster_name, {}).itervalues() - container_instances = [ci.containerInstanceArn for ci in container_instances_iter] + container_instances_values = self.container_instances.get(cluster_name, {}).values() + container_instances = [ci.containerInstanceArn for ci in container_instances_values] return sorted(container_instances) def describe_container_instances(self, cluster_str, list_container_instance_ids): From 27095638d9a28a7434f7a956c1907203fd22eac5 Mon Sep 17 00:00:00 2001 From: "Riccardo M. Cefala" Date: Wed, 15 Jun 2016 10:41:34 +0200 Subject: [PATCH 10/10] map() returns a map object iterable instead of a list in python3 --- tests/test_ecs/test_ecs_boto3.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ecs/test_ecs_boto3.py b/tests/test_ecs/test_ecs_boto3.py index 9ca367e0..2c6617d7 100644 --- a/tests/test_ecs/test_ecs_boto3.py +++ b/tests/test_ecs/test_ecs_boto3.py @@ -446,7 +446,7 @@ def test_describe_container_instances(): test_instance_arns.append(response['containerInstance']['containerInstanceArn']) - test_instance_ids = map((lambda x: x.split('/')[1]), test_instance_arns) + test_instance_ids = list(map((lambda x: x.split('/')[1]), test_instance_arns)) response = ecs_client.describe_container_instances(cluster=test_cluster_name, containerInstances=test_instance_ids) len(response['failures']).should.equal(0) len(response['containerInstances']).should.equal(instance_to_create)