From 1ef3094e45dbf87f7359145b3c4c8d01818bb8eb Mon Sep 17 00:00:00 2001 From: Bert Blommers Date: Sun, 24 May 2020 12:12:35 +0100 Subject: [PATCH 01/17] SQS - Return multiple group-messages in the same request --- moto/sqs/models.py | 9 ++--- moto/sqs/responses.py | 8 +++++ tests/test_sqs/test_sqs.py | 73 +++++++++++++++++++++++++++----------- 3 files changed, 65 insertions(+), 25 deletions(-) diff --git a/moto/sqs/models.py b/moto/sqs/models.py index f88d906b..ea3b89f0 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -6,6 +6,7 @@ import json import re import six import struct +from copy import deepcopy from xml.sax.saxutils import escape from boto3 import Session @@ -101,7 +102,6 @@ class Message(BaseModel): if data_type == "String" or data_type == "Number": value = attr["string_value"] elif data_type == "Binary": - print(data_type, attr["binary_value"], type(attr["binary_value"])) value = base64.b64decode(attr["binary_value"]) else: print( @@ -722,6 +722,7 @@ class SQSBackend(BaseBackend): previous_result_count = len(result) polling_end = unix_time() + wait_seconds_timeout + currently_pending_groups = deepcopy(queue.pending_message_groups) # queue.messages only contains visible messages while True: @@ -739,11 +740,11 @@ class SQSBackend(BaseBackend): # The message is pending but is visible again, so the # consumer must have timed out. queue.pending_messages.remove(message) + currently_pending_groups = deepcopy(queue.pending_message_groups) if message.group_id and queue.fifo_queue: - if message.group_id in queue.pending_message_groups: - # There is already one active message with the same - # group, so we cannot deliver this one. + if message.group_id in currently_pending_groups: + # A previous call is still processing messages in this group, so we cannot deliver this one. continue queue.pending_messages.add(message) diff --git a/moto/sqs/responses.py b/moto/sqs/responses.py index f5481cc1..eed50a52 100644 --- a/moto/sqs/responses.py +++ b/moto/sqs/responses.py @@ -232,6 +232,14 @@ class SQSResponse(BaseResponse): queue_name = self._get_queue_name() + if not message_group_id: + queue = self.sqs_backend.get_queue(queue_name) + if queue.attributes.get("FifoQueue", False): + return self._error( + "MissingParameter", + "The request must contain the parameter MessageGroupId.", + ) + message = self.sqs_backend.send_message( queue_name, message, diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index 01e34de0..31bbafff 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -1164,7 +1164,7 @@ def test_send_message_batch_with_empty_list(): @mock_sqs def test_batch_change_message_visibility(): - if os.environ.get("TEST_SERVER_MODE", "false").lower() == "true": + if settings.TEST_SERVER_MODE: raise SkipTest("Cant manipulate time in server mode") with freeze_time("2015-01-01 12:00:00"): @@ -1174,9 +1174,15 @@ def test_batch_change_message_visibility(): ) queue_url = resp["QueueUrl"] - sqs.send_message(QueueUrl=queue_url, MessageBody="msg1") - sqs.send_message(QueueUrl=queue_url, MessageBody="msg2") - sqs.send_message(QueueUrl=queue_url, MessageBody="msg3") + sqs.send_message( + QueueUrl=queue_url, MessageBody="msg1", MessageGroupId="group1" + ) + sqs.send_message( + QueueUrl=queue_url, MessageBody="msg2", MessageGroupId="group2" + ) + sqs.send_message( + QueueUrl=queue_url, MessageBody="msg3", MessageGroupId="group3" + ) with freeze_time("2015-01-01 12:01:00"): receive_resp = sqs.receive_message(QueueUrl=queue_url, MaxNumberOfMessages=2) @@ -1529,7 +1535,7 @@ def test_create_fifo_queue_with_dlq(): @mock_sqs def test_queue_with_dlq(): - if os.environ.get("TEST_SERVER_MODE", "false").lower() == "true": + if settings.TEST_SERVER_MODE: raise SkipTest("Cant manipulate time in server mode") sqs = boto3.client("sqs", region_name="us-east-1") @@ -1554,8 +1560,12 @@ def test_queue_with_dlq(): ) queue_url2 = resp["QueueUrl"] - sqs.send_message(QueueUrl=queue_url2, MessageBody="msg1") - sqs.send_message(QueueUrl=queue_url2, MessageBody="msg2") + sqs.send_message( + QueueUrl=queue_url2, MessageBody="msg1", MessageGroupId="group" + ) + sqs.send_message( + QueueUrl=queue_url2, MessageBody="msg2", MessageGroupId="group" + ) with freeze_time("2015-01-01 13:00:00"): resp = sqs.receive_message( @@ -1686,20 +1696,24 @@ def test_receive_messages_with_message_group_id(): queue.set_attributes(Attributes={"VisibilityTimeout": "3600"}) queue.send_message(MessageBody="message-1", MessageGroupId="group") queue.send_message(MessageBody="message-2", MessageGroupId="group") + queue.send_message(MessageBody="message-3", MessageGroupId="group") + queue.send_message(MessageBody="separate-message", MessageGroupId="anothergroup") - messages = queue.receive_messages() - messages.should.have.length_of(1) - message = messages[0] + messages = queue.receive_messages(MaxNumberOfMessages=2) + messages.should.have.length_of(2) + messages[0].attributes["MessageGroupId"].should.equal("group") - # received message is not deleted! - - messages = queue.receive_messages(WaitTimeSeconds=0) - messages.should.have.length_of(0) + # Different client can not 'see' messages from the group until they are processed + messages_for_client_2 = queue.receive_messages(WaitTimeSeconds=0) + messages_for_client_2.should.have.length_of(1) + messages_for_client_2[0].body.should.equal("separate-message") # message is now processed, next one should be available - message.delete() + for message in messages: + message.delete() messages = queue.receive_messages() messages.should.have.length_of(1) + messages[0].body.should.equal("message-3") @mock_sqs @@ -1730,7 +1744,7 @@ def test_receive_messages_with_message_group_id_on_requeue(): @mock_sqs def test_receive_messages_with_message_group_id_on_visibility_timeout(): - if os.environ.get("TEST_SERVER_MODE", "false").lower() == "true": + if settings.TEST_SERVER_MODE: raise SkipTest("Cant manipulate time in server mode") with freeze_time("2015-01-01 12:00:00"): @@ -1746,12 +1760,12 @@ def test_receive_messages_with_message_group_id_on_visibility_timeout(): messages.should.have.length_of(1) message = messages[0] - # received message is not deleted! + # received message is not processed yet + messages_for_second_client = queue.receive_messages(WaitTimeSeconds=0) + messages_for_second_client.should.have.length_of(0) - messages = queue.receive_messages(WaitTimeSeconds=0) - messages.should.have.length_of(0) - - message.change_visibility(VisibilityTimeout=10) + for message in messages: + message.change_visibility(VisibilityTimeout=10) with freeze_time("2015-01-01 12:00:05"): # no timeout yet @@ -1794,3 +1808,20 @@ def test_list_queues_limits_to_1000_queues(): list(resource.queues.filter(QueueNamePrefix="test-queue")).should.have.length_of( 1000 ) + + +@mock_sqs +def test_send_messages_to_fifo_without_message_group_id(): + sqs = boto3.resource("sqs", region_name="eu-west-3") + queue = sqs.create_queue( + QueueName="blah.fifo", + Attributes={"FifoQueue": "true", "ContentBasedDeduplication": "true"}, + ) + + with assert_raises(Exception) as e: + queue.send_message(MessageBody="message-1") + ex = e.exception + ex.response["Error"]["Code"].should.equal("MissingParameter") + ex.response["Error"]["Message"].should.equal( + "The request must contain the parameter MessageGroupId." + ) From 31ce74a842c3a0d5a82afb431dca0afed7b89fe5 Mon Sep 17 00:00:00 2001 From: Zach Brookler <39153813+zbrookle@users.noreply.github.com> Date: Sun, 24 May 2020 07:21:29 -0400 Subject: [PATCH 02/17] Fix autoscaling tags (#3010) * ENH: Add unit test for propagation tags * BUG: Add missing translation of boolean PropagateAtLaunch tag values to strings * BUG: Should really be checking for "true" and not True * CLN: Black formatting --- moto/autoscaling/models.py | 14 ++++- .../test_cloudformation_stack_integration.py | 55 +++++++++++++++++++ 2 files changed, 67 insertions(+), 2 deletions(-) diff --git a/moto/autoscaling/models.py b/moto/autoscaling/models.py index 1da12a09..f4185da6 100644 --- a/moto/autoscaling/models.py +++ b/moto/autoscaling/models.py @@ -301,6 +301,14 @@ class FakeAutoScalingGroup(BaseModel): self.availability_zones = availability_zones self.vpc_zone_identifier = vpc_zone_identifier + @staticmethod + def __set_string_propagate_at_launch_booleans_on_tags(tags): + bool_to_string = {True: "true", False: "false"} + for tag in tags: + if "PropagateAtLaunch" in tag: + tag["PropagateAtLaunch"] = bool_to_string[tag["PropagateAtLaunch"]] + return tags + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name @@ -329,7 +337,9 @@ class FakeAutoScalingGroup(BaseModel): target_group_arns=target_group_arns, placement_group=None, termination_policies=properties.get("TerminationPolicies", []), - tags=properties.get("Tags", []), + tags=cls.__set_string_propagate_at_launch_booleans_on_tags( + properties.get("Tags", []) + ), new_instances_protected_from_scale_in=properties.get( "NewInstancesProtectedFromScaleIn", False ), @@ -455,7 +465,7 @@ class FakeAutoScalingGroup(BaseModel): # boto3 and cloudformation use PropagateAtLaunch if "propagate_at_launch" in tag and tag["propagate_at_launch"] == "true": propagated_tags[tag["key"]] = tag["value"] - if "PropagateAtLaunch" in tag and tag["PropagateAtLaunch"]: + if "PropagateAtLaunch" in tag and tag["PropagateAtLaunch"] == "true": propagated_tags[tag["Key"]] = tag["Value"] return propagated_tags diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index 27bac5e5..3abb3373 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -21,6 +21,7 @@ import sure # noqa from moto import ( mock_autoscaling_deprecated, + mock_autoscaling, mock_cloudformation, mock_cloudformation_deprecated, mock_datapipeline_deprecated, @@ -2496,3 +2497,57 @@ def test_stack_events_create_rule_as_target(): log_groups["logGroups"][0]["logGroupName"].should.equal(rules["Rules"][0]["Arn"]) log_groups["logGroups"][0]["retentionInDays"].should.equal(3) + + +@mock_cloudformation +@mock_autoscaling +def test_autoscaling_propagate_tags(): + autoscaling_group_with_tags = { + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "AutoScalingGroup": { + "Type": "AWS::AutoScaling::AutoScalingGroup", + "Properties": { + "AutoScalingGroupName": "test-scaling-group", + "DesiredCapacity": 1, + "MinSize": 1, + "MaxSize": 50, + "LaunchConfigurationName": "test-launch-config", + "AvailabilityZones": ["us-east-1a"], + "Tags": [ + { + "Key": "test-key-propagate", + "Value": "test", + "PropagateAtLaunch": True, + }, + { + "Key": "test-key-no-propagate", + "Value": "test", + "PropagateAtLaunch": False, + }, + ], + }, + "DependsOn": "LaunchConfig", + }, + "LaunchConfig": { + "Type": "AWS::AutoScaling::LaunchConfiguration", + "Properties": {"LaunchConfigurationName": "test-launch-config"}, + }, + }, + } + boto3.client("cloudformation", "us-east-1").create_stack( + StackName="propagate_tags_test", + TemplateBody=json.dumps(autoscaling_group_with_tags), + ) + + autoscaling = boto3.client("autoscaling", "us-east-1") + + autoscaling_group_tags = autoscaling.describe_auto_scaling_groups()[ + "AutoScalingGroups" + ][0]["Tags"] + propagation_dict = { + tag["Key"]: tag["PropagateAtLaunch"] for tag in autoscaling_group_tags + } + + assert propagation_dict["test-key-propagate"] + assert not propagation_dict["test-key-no-propagate"] From 2320e8264796c4b1e62382af1a7e176b314ca780 Mon Sep 17 00:00:00 2001 From: Maxim Kirilov Date: Sun, 24 May 2020 14:22:45 +0300 Subject: [PATCH 03/17] Add support for detaching volumes upon instance termination (#2999) --- moto/ec2/exceptions.py | 10 +++ moto/ec2/models.py | 49 +++++++++++--- tests/test_ec2/test_instances.py | 106 ++++++++++++++++++++++++++++++- 3 files changed, 152 insertions(+), 13 deletions(-) diff --git a/moto/ec2/exceptions.py b/moto/ec2/exceptions.py index 5af4690a..4c47adbb 100644 --- a/moto/ec2/exceptions.py +++ b/moto/ec2/exceptions.py @@ -231,6 +231,16 @@ class InvalidVolumeAttachmentError(EC2ClientError): ) +class InvalidVolumeDetachmentError(EC2ClientError): + def __init__(self, volume_id, instance_id, device): + super(InvalidVolumeDetachmentError, self).__init__( + "InvalidAttachment.NotFound", + "The volume {0} is not attached to instance {1} as device {2}".format( + volume_id, instance_id, device + ), + ) + + class VolumeInUseError(EC2ClientError): def __init__(self, volume_id, instance_id): super(VolumeInUseError, self).__init__( diff --git a/moto/ec2/models.py b/moto/ec2/models.py index bab4636a..78e74354 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -72,6 +72,7 @@ from .exceptions import ( InvalidVolumeIdError, VolumeInUseError, InvalidVolumeAttachmentError, + InvalidVolumeDetachmentError, InvalidVpcCidrBlockAssociationIdError, InvalidVPCPeeringConnectionIdError, InvalidVPCPeeringConnectionStateTransitionError, @@ -560,23 +561,34 @@ class Instance(TaggedEC2Resource, BotoInstance): # worst case we'll get IP address exaustion... rarely pass - def add_block_device(self, size, device_path, snapshot_id=None, encrypted=False): + def add_block_device( + self, + size, + device_path, + snapshot_id=None, + encrypted=False, + delete_on_termination=False, + ): volume = self.ec2_backend.create_volume( size, self.region_name, snapshot_id, encrypted ) - self.ec2_backend.attach_volume(volume.id, self.id, device_path) + self.ec2_backend.attach_volume( + volume.id, self.id, device_path, delete_on_termination + ) def setup_defaults(self): # Default have an instance with root volume should you not wish to # override with attach volume cmd. volume = self.ec2_backend.create_volume(8, "us-east-1a") - self.ec2_backend.attach_volume(volume.id, self.id, "/dev/sda1") + self.ec2_backend.attach_volume(volume.id, self.id, "/dev/sda1", True) def teardown_defaults(self): - if "/dev/sda1" in self.block_device_mapping: - volume_id = self.block_device_mapping["/dev/sda1"].volume_id - self.ec2_backend.detach_volume(volume_id, self.id, "/dev/sda1") - self.ec2_backend.delete_volume(volume_id) + for device_path in list(self.block_device_mapping.keys()): + volume = self.block_device_mapping[device_path] + volume_id = volume.volume_id + self.ec2_backend.detach_volume(volume_id, self.id, device_path) + if volume.delete_on_termination: + self.ec2_backend.delete_volume(volume_id) @property def get_block_device_mapping(self): @@ -897,8 +909,15 @@ class InstanceBackend(object): volume_size = block_device["Ebs"].get("VolumeSize") snapshot_id = block_device["Ebs"].get("SnapshotId") encrypted = block_device["Ebs"].get("Encrypted", False) + delete_on_termination = block_device["Ebs"].get( + "DeleteOnTermination", False + ) new_instance.add_block_device( - volume_size, device_name, snapshot_id, encrypted + volume_size, + device_name, + snapshot_id, + encrypted, + delete_on_termination, ) else: new_instance.setup_defaults() @@ -2475,7 +2494,9 @@ class EBSBackend(object): return self.volumes.pop(volume_id) raise InvalidVolumeIdError(volume_id) - def attach_volume(self, volume_id, instance_id, device_path): + def attach_volume( + self, volume_id, instance_id, device_path, delete_on_termination=False + ): volume = self.get_volume(volume_id) instance = self.get_instance(instance_id) @@ -2489,17 +2510,25 @@ class EBSBackend(object): status=volume.status, size=volume.size, attach_time=utc_date_and_time(), + delete_on_termination=delete_on_termination, ) instance.block_device_mapping[device_path] = bdt return volume.attachment def detach_volume(self, volume_id, instance_id, device_path): volume = self.get_volume(volume_id) - self.get_instance(instance_id) + instance = self.get_instance(instance_id) old_attachment = volume.attachment if not old_attachment: raise InvalidVolumeAttachmentError(volume_id, instance_id) + device_path = device_path or old_attachment.device + + try: + del instance.block_device_mapping[device_path] + except KeyError: + raise InvalidVolumeDetachmentError(volume_id, instance_id, device_path) + old_attachment.status = "detached" volume.attachment = None diff --git a/tests/test_ec2/test_instances.py b/tests/test_ec2/test_instances.py index d53bd14a..d2588097 100644 --- a/tests/test_ec2/test_instances.py +++ b/tests/test_ec2/test_instances.py @@ -99,6 +99,106 @@ def test_instance_launch_and_terminate(): instance.state.should.equal("terminated") +@mock_ec2 +def test_instance_terminate_discard_volumes(): + + ec2_resource = boto3.resource("ec2", "us-west-1") + + result = ec2_resource.create_instances( + ImageId="ami-d3adb33f", + MinCount=1, + MaxCount=1, + BlockDeviceMappings=[ + { + "DeviceName": "/dev/sda1", + "Ebs": {"VolumeSize": 50, "DeleteOnTermination": True}, + } + ], + ) + instance = result[0] + + instance_volume_ids = [] + for volume in instance.volumes.all(): + instance_volume_ids.append(volume.volume_id) + + instance.terminate() + instance.wait_until_terminated() + + assert not list(ec2_resource.volumes.all()) + + +@mock_ec2 +def test_instance_terminate_keep_volumes(): + ec2_resource = boto3.resource("ec2", "us-west-1") + + result = ec2_resource.create_instances( + ImageId="ami-d3adb33f", + MinCount=1, + MaxCount=1, + BlockDeviceMappings=[{"DeviceName": "/dev/sda1", "Ebs": {"VolumeSize": 50}}], + ) + instance = result[0] + + instance_volume_ids = [] + for volume in instance.volumes.all(): + instance_volume_ids.append(volume.volume_id) + + instance.terminate() + instance.wait_until_terminated() + + assert len(instance_volume_ids) == 1 + volume = ec2_resource.Volume(instance_volume_ids[0]) + volume.state.should.equal("available") + + +@mock_ec2 +def test_instance_terminate_detach_volumes(): + ec2_resource = boto3.resource("ec2", "us-west-1") + result = ec2_resource.create_instances( + ImageId="ami-d3adb33f", + MinCount=1, + MaxCount=1, + BlockDeviceMappings=[ + {"DeviceName": "/dev/sda1", "Ebs": {"VolumeSize": 50}}, + {"DeviceName": "/dev/sda2", "Ebs": {"VolumeSize": 50}}, + ], + ) + instance = result[0] + for volume in instance.volumes.all(): + response = instance.detach_volume(VolumeId=volume.volume_id) + response["State"].should.equal("detaching") + + instance.terminate() + instance.wait_until_terminated() + + assert len(list(ec2_resource.volumes.all())) == 2 + + +@mock_ec2 +def test_instance_detach_volume_wrong_path(): + ec2_resource = boto3.resource("ec2", "us-west-1") + result = ec2_resource.create_instances( + ImageId="ami-d3adb33f", + MinCount=1, + MaxCount=1, + BlockDeviceMappings=[{"DeviceName": "/dev/sda1", "Ebs": {"VolumeSize": 50}},], + ) + instance = result[0] + for volume in instance.volumes.all(): + with assert_raises(ClientError) as ex: + instance.detach_volume(VolumeId=volume.volume_id, Device="/dev/sdf") + + ex.exception.response["Error"]["Code"].should.equal( + "InvalidAttachment.NotFound" + ) + ex.exception.response["ResponseMetadata"]["HTTPStatusCode"].should.equal(400) + ex.exception.response["Error"]["Message"].should.equal( + "The volume {0} is not attached to instance {1} as device {2}".format( + volume.volume_id, instance.instance_id, "/dev/sdf" + ) + ) + + @mock_ec2_deprecated def test_terminate_empty_instances(): conn = boto.connect_ec2("the_key", "the_secret") @@ -1416,14 +1516,14 @@ def test_modify_delete_on_termination(): result = ec2_client.create_instances(ImageId="ami-12345678", MinCount=1, MaxCount=1) instance = result[0] instance.load() - instance.block_device_mappings[0]["Ebs"]["DeleteOnTermination"].should.be(False) + instance.block_device_mappings[0]["Ebs"]["DeleteOnTermination"].should.be(True) instance.modify_attribute( BlockDeviceMappings=[ - {"DeviceName": "/dev/sda1", "Ebs": {"DeleteOnTermination": True}} + {"DeviceName": "/dev/sda1", "Ebs": {"DeleteOnTermination": False}} ] ) instance.load() - instance.block_device_mappings[0]["Ebs"]["DeleteOnTermination"].should.be(True) + instance.block_device_mappings[0]["Ebs"]["DeleteOnTermination"].should.be(False) @mock_ec2 From 93feeec1b7fa6d71d1507b585c0266f0960d560d Mon Sep 17 00:00:00 2001 From: Aidan Rowe Date: Mon, 25 May 2020 00:06:02 +1000 Subject: [PATCH 04/17] SFN - fix InvalidARN exception on start_execution (#3007) --- moto/stepfunctions/responses.py | 5 ++++- .../test_stepfunctions/test_stepfunctions.py | 20 ++++++++++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/moto/stepfunctions/responses.py b/moto/stepfunctions/responses.py index 689961d5..7083167b 100644 --- a/moto/stepfunctions/responses.py +++ b/moto/stepfunctions/responses.py @@ -95,7 +95,10 @@ class StepFunctionResponse(BaseResponse): def start_execution(self): arn = self._get_param("stateMachineArn") name = self._get_param("name") - execution = self.stepfunction_backend.start_execution(arn, name) + try: + execution = self.stepfunction_backend.start_execution(arn, name) + except AWSError as err: + return err.response() response = { "executionArn": execution.execution_arn, "startDate": execution.start_date, diff --git a/tests/test_stepfunctions/test_stepfunctions.py b/tests/test_stepfunctions/test_stepfunctions.py index eb2ace53..4324964d 100644 --- a/tests/test_stepfunctions/test_stepfunctions.py +++ b/tests/test_stepfunctions/test_stepfunctions.py @@ -253,6 +253,15 @@ def test_state_machine_throws_error_when_describing_unknown_machine(): client.describe_state_machine(stateMachineArn=unknown_state_machine) +@mock_stepfunctions +@mock_sts +def test_state_machine_throws_error_when_describing_bad_arn(): + client = boto3.client("stepfunctions", region_name=region) + # + with assert_raises(ClientError) as exc: + client.describe_state_machine(stateMachineArn="bad") + + @mock_stepfunctions @mock_sts def test_state_machine_throws_error_when_describing_machine_in_different_account(): @@ -362,6 +371,15 @@ def test_state_machine_start_execution(): execution["startDate"].should.be.a(datetime) +@mock_stepfunctions +@mock_sts +def test_state_machine_start_execution_bad_arn_raises_exception(): + client = boto3.client("stepfunctions", region_name=region) + # + with assert_raises(ClientError) as exc: + client.start_execution(stateMachineArn="bad") + + @mock_stepfunctions @mock_sts def test_state_machine_start_execution_with_custom_name(): @@ -446,7 +464,7 @@ def test_state_machine_describe_execution(): @mock_stepfunctions @mock_sts -def test_state_machine_throws_error_when_describing_unknown_machine(): +def test_execution_throws_error_when_describing_unknown_execution(): client = boto3.client("stepfunctions", region_name=region) # with assert_raises(ClientError) as exc: From 97a6e8d9e8635e5ebe34bbfbbd9e75ce37a58e83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Andr=C3=A9=20Nardy?= Date: Tue, 26 May 2020 07:04:59 -0300 Subject: [PATCH 05/17] Enhancement/describe network acls (#3003) * update describe_network_acls and create unit test * add fail test case * adjustment after feedback * fix result test --- IMPLEMENTATION_COVERAGE.md | 2 +- moto/ec2/models.py | 37 ++++++++++++++++------------- moto/ec2/responses/network_acls.py | 2 +- tests/test_ec2/test_network_acls.py | 29 ++++++++++++++++++++++ 4 files changed, 51 insertions(+), 19 deletions(-) diff --git a/IMPLEMENTATION_COVERAGE.md b/IMPLEMENTATION_COVERAGE.md index 1555da1c..bfcdd316 100644 --- a/IMPLEMENTATION_COVERAGE.md +++ b/IMPLEMENTATION_COVERAGE.md @@ -2738,7 +2738,7 @@ - [ ] describe_local_gateways - [ ] describe_moving_addresses - [ ] describe_nat_gateways -- [ ] describe_network_acls +- [X] describe_network_acls - [ ] describe_network_interface_attribute - [ ] describe_network_interface_permissions - [X] describe_network_interfaces diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 78e74354..f8ebd02e 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -4750,23 +4750,7 @@ class NetworkAclBackend(object): ) def get_all_network_acls(self, network_acl_ids=None, filters=None): - network_acls = self.network_acls.values() - - if network_acl_ids: - network_acls = [ - network_acl - for network_acl in network_acls - if network_acl.id in network_acl_ids - ] - if len(network_acls) != len(network_acl_ids): - invalid_id = list( - set(network_acl_ids).difference( - set([network_acl.id for network_acl in network_acls]) - ) - )[0] - raise InvalidRouteTableIdError(invalid_id) - - return generic_filter(filters, network_acls) + self.describe_network_acls(network_acl_ids, filters) def delete_network_acl(self, network_acl_id): deleted = self.network_acls.pop(network_acl_id, None) @@ -4886,6 +4870,25 @@ class NetworkAclBackend(object): self, association_id, subnet_id, acl.id ) + def describe_network_acls(self, network_acl_ids=None, filters=None): + network_acls = self.network_acls.values() + + if network_acl_ids: + network_acls = [ + network_acl + for network_acl in network_acls + if network_acl.id in network_acl_ids + ] + if len(network_acls) != len(network_acl_ids): + invalid_id = list( + set(network_acl_ids).difference( + set([network_acl.id for network_acl in network_acls]) + ) + )[0] + raise InvalidRouteTableIdError(invalid_id) + + return generic_filter(filters, network_acls) + class NetworkAclAssociation(object): def __init__(self, ec2_backend, new_association_id, subnet_id, network_acl_id): diff --git a/moto/ec2/responses/network_acls.py b/moto/ec2/responses/network_acls.py index 8d89e606..c0a9c7c9 100644 --- a/moto/ec2/responses/network_acls.py +++ b/moto/ec2/responses/network_acls.py @@ -83,7 +83,7 @@ class NetworkACLs(BaseResponse): def describe_network_acls(self): network_acl_ids = self._get_multi_param("NetworkAclId") filters = filters_from_querystring(self.querystring) - network_acls = self.ec2_backend.get_all_network_acls(network_acl_ids, filters) + network_acls = self.ec2_backend.describe_network_acls(network_acl_ids, filters) template = self.response_template(DESCRIBE_NETWORK_ACL_RESPONSE) return template.render(network_acls=network_acls) diff --git a/tests/test_ec2/test_network_acls.py b/tests/test_ec2/test_network_acls.py index fb62f717..f255fa67 100644 --- a/tests/test_ec2/test_network_acls.py +++ b/tests/test_ec2/test_network_acls.py @@ -275,3 +275,32 @@ def test_duplicate_network_acl_entry(): rule_number ) ) + + +@mock_ec2 +def test_describe_network_acls(): + conn = boto3.client("ec2", region_name="us-west-2") + + vpc = conn.create_vpc(CidrBlock="10.0.0.0/16") + vpc_id = vpc["Vpc"]["VpcId"] + + network_acl = conn.create_network_acl(VpcId=vpc_id) + + network_acl_id = network_acl["NetworkAcl"]["NetworkAclId"] + + resp = conn.describe_network_acls(NetworkAclIds=[network_acl_id]) + result = resp["NetworkAcls"] + + result.should.have.length_of(1) + result[0]["NetworkAclId"].should.equal(network_acl_id) + + resp2 = conn.describe_network_acls()["NetworkAcls"] + resp2.should.have.length_of(3) + + with assert_raises(ClientError) as ex: + conn.describe_network_acls(NetworkAclIds=["1"]) + + str(ex.exception).should.equal( + "An error occurred (InvalidRouteTableID.NotFound) when calling the " + "DescribeNetworkAcls operation: The routeTable ID '1' does not exist" + ) From b7a1b666a8cb2118b22cfd3508eba10763c1c598 Mon Sep 17 00:00:00 2001 From: jweite Date: Wed, 27 May 2020 12:00:28 -0400 Subject: [PATCH 06/17] =?UTF-8?q?Corrected=20bug=20in=20IAM=20delete=5Frol?= =?UTF-8?q?e()=20due=20to=20overloading=20of=20name=20'role'=20=E2=80=A6?= =?UTF-8?q?=20(#3019)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Corrected bug in IAM delete_role() due to overloading of name 'role' in function * PR-requested fixes: added region to tests boto client create, reformatted with black Co-authored-by: Joseph Weitekamp --- moto/iam/models.py | 4 ++-- tests/test_iam/test_iam.py | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 2 deletions(-) diff --git a/moto/iam/models.py b/moto/iam/models.py index d3907da2..41484add 100755 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -1148,8 +1148,8 @@ class IAMBackend(BaseBackend): def delete_role(self, role_name): role = self.get_role(role_name) for instance_profile in self.get_instance_profiles(): - for role in instance_profile.roles: - if role.name == role_name: + for profile_role in instance_profile.roles: + if profile_role.name == role_name: raise IAMConflictException( code="DeleteConflict", message="Cannot delete entity, must remove roles from instance profile first.", diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 6792d8f5..825e12fe 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -2815,3 +2815,36 @@ def test_list_user_tags(): [{"Key": "Stan", "Value": "The Caddy"}, {"Key": "like-a", "Value": "glove"}] ) response["IsTruncated"].should_not.be.ok + + +@mock_iam() +def test_delete_role_with_instance_profiles_present(): + iam = boto3.client("iam", region_name="us-east-1") + + trust_policy = """ + { + "Version": "2012-10-17", + "Statement": [ + { + "Effect": "Allow", + "Principal": { + "Service": "ec2.amazonaws.com" + }, + "Action": "sts:AssumeRole" + } + ] + } + """ + trust_policy = trust_policy.strip() + + iam.create_role(RoleName="Role1", AssumeRolePolicyDocument=trust_policy) + iam.create_instance_profile(InstanceProfileName="IP1") + iam.add_role_to_instance_profile(InstanceProfileName="IP1", RoleName="Role1") + + iam.create_role(RoleName="Role2", AssumeRolePolicyDocument=trust_policy) + + iam.delete_role(RoleName="Role2") + + role_names = [role["RoleName"] for role in iam.list_roles()["Roles"]] + assert "Role1" in role_names + assert "Role2" not in role_names From 4d3e3c8c5e7737e1f2d050d441d2d2e399de2384 Mon Sep 17 00:00:00 2001 From: jweite Date: Wed, 27 May 2020 12:21:03 -0400 Subject: [PATCH 07/17] implemented s3 default encryption methods (#3022) * implemented s3 default encryption methods * PR adjustments: moved logic for retrieving bucket's encrypted status to the backend. Co-authored-by: Joseph Weitekamp --- moto/s3/models.py | 10 +++++++ moto/s3/responses.py | 62 +++++++++++++++++++++++++++++++++++++++- tests/test_s3/test_s3.py | 33 +++++++++++++++++++++ 3 files changed, 104 insertions(+), 1 deletion(-) diff --git a/moto/s3/models.py b/moto/s3/models.py index 3020fd45..25ead4f5 100644 --- a/moto/s3/models.py +++ b/moto/s3/models.py @@ -778,6 +778,7 @@ class FakeBucket(BaseModel): self.payer = "BucketOwner" self.creation_date = datetime.datetime.utcnow() self.public_access_block = None + self.encryption = None @property def location(self): @@ -1227,6 +1228,9 @@ class S3Backend(BaseBackend): def get_bucket_versioning(self, bucket_name): return self.get_bucket(bucket_name).versioning_status + def get_bucket_encryption(self, bucket_name): + return self.get_bucket(bucket_name).encryption + def get_bucket_latest_versions(self, bucket_name): versions = self.get_bucket_versions(bucket_name) latest_modified_per_key = {} @@ -1275,6 +1279,12 @@ class S3Backend(BaseBackend): bucket = self.get_bucket(bucket_name) bucket.policy = None + def put_bucket_encryption(self, bucket_name, encryption): + self.get_bucket(bucket_name).encryption = encryption + + def delete_bucket_encryption(self, bucket_name): + self.get_bucket(bucket_name).encryption = None + def set_bucket_lifecycle(self, bucket_name, rules): bucket = self.get_bucket(bucket_name) bucket.set_lifecycle(rules) diff --git a/moto/s3/responses.py b/moto/s3/responses.py index 98f28f01..4aaba1fc 100644 --- a/moto/s3/responses.py +++ b/moto/s3/responses.py @@ -466,6 +466,13 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): is_truncated="false", ), ) + elif "encryption" in querystring: + encryption = self.backend.get_bucket_encryption(bucket_name) + if not encryption: + template = self.response_template(S3_NO_ENCRYPTION) + return 404, {}, template.render(bucket_name=bucket_name) + template = self.response_template(S3_ENCRYPTION_CONFIG) + return 200, {}, template.render(encryption=encryption) elif querystring.get("list-type", [None])[0] == "2": return 200, {}, self._handle_list_objects_v2(bucket_name, querystring) @@ -703,7 +710,16 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): bucket_name, pab_config["PublicAccessBlockConfiguration"] ) return "" - + elif "encryption" in querystring: + try: + self.backend.put_bucket_encryption( + bucket_name, self._encryption_config_from_xml(body) + ) + return "" + except KeyError: + raise MalformedXML() + except Exception as e: + raise e else: # us-east-1, the default AWS region behaves a bit differently # - you should not use it as a location constraint --> it fails @@ -768,6 +784,9 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): elif "publicAccessBlock" in querystring: self.backend.delete_bucket_public_access_block(bucket_name) return 204, {}, "" + elif "encryption" in querystring: + bucket = self.backend.delete_bucket_encryption(bucket_name) + return 204, {}, "" removed_bucket = self.backend.delete_bucket(bucket_name) @@ -1427,6 +1446,22 @@ class ResponseObject(_TemplateEnvironmentMixin, ActionAuthenticatorMixin): return [parsed_xml["CORSConfiguration"]["CORSRule"]] + def _encryption_config_from_xml(self, xml): + parsed_xml = xmltodict.parse(xml) + + if ( + not parsed_xml["ServerSideEncryptionConfiguration"].get("Rule") + or not parsed_xml["ServerSideEncryptionConfiguration"]["Rule"].get( + "ApplyServerSideEncryptionByDefault" + ) + or not parsed_xml["ServerSideEncryptionConfiguration"]["Rule"][ + "ApplyServerSideEncryptionByDefault" + ].get("SSEAlgorithm") + ): + raise MalformedXML() + + return [parsed_xml["ServerSideEncryptionConfiguration"]] + def _logging_from_xml(self, xml): parsed_xml = xmltodict.parse(xml) @@ -2130,6 +2165,31 @@ S3_NO_LOGGING_CONFIG = """ """ +S3_ENCRYPTION_CONFIG = """ + + {% for entry in encryption %} + + + {{ entry["Rule"]["ApplyServerSideEncryptionByDefault"]["SSEAlgorithm"] }} + {% if entry["Rule"]["ApplyServerSideEncryptionByDefault"].get("KMSMasterKeyID") %} + {{ entry["Rule"]["ApplyServerSideEncryptionByDefault"]["KMSMasterKeyID"] }} + {% endif %} + + + {% endfor %} + +""" + +S3_NO_ENCRYPTION = """ + + ServerSideEncryptionConfigurationNotFoundError + The server side encryption configuration was not found + {{ bucket_name }} + 0D68A23BB2E2215B + 9Gjjt1m+cjU4OPvX9O9/8RuvnG41MRb/18Oux2o5H5MY7ISNTlXN+Dz9IG62/ILVxhAGI0qyPfg= + +""" + S3_GET_BUCKET_NOTIFICATION_CONFIG = """ {% for topic in bucket.notification_configuration.topic %} diff --git a/tests/test_s3/test_s3.py b/tests/test_s3/test_s3.py index bcb9da87..363ccc02 100644 --- a/tests/test_s3/test_s3.py +++ b/tests/test_s3/test_s3.py @@ -4521,3 +4521,36 @@ def test_creating_presigned_post(): ].read() == fdata ) + + +@mock_s3 +def test_encryption(): + # Create Bucket so that test can run + conn = boto3.client("s3", region_name="us-east-1") + conn.create_bucket(Bucket="mybucket") + + with assert_raises(ClientError) as exc: + conn.get_bucket_encryption(Bucket="mybucket") + + sse_config = { + "Rules": [ + { + "ApplyServerSideEncryptionByDefault": { + "SSEAlgorithm": "aws:kms", + "KMSMasterKeyID": "12345678", + } + } + ] + } + + conn.put_bucket_encryption( + Bucket="mybucket", ServerSideEncryptionConfiguration=sse_config + ) + + resp = conn.get_bucket_encryption(Bucket="mybucket") + assert "ServerSideEncryptionConfiguration" in resp + assert resp["ServerSideEncryptionConfiguration"] == sse_config + + conn.delete_bucket_encryption(Bucket="mybucket") + with assert_raises(ClientError) as exc: + conn.get_bucket_encryption(Bucket="mybucket") From 4303123312e9bd878e08b925f671438874ea4054 Mon Sep 17 00:00:00 2001 From: jweite Date: Wed, 27 May 2020 13:22:06 -0400 Subject: [PATCH 08/17] Implemented IAM delete_instance_profile (#3020) * Implemented IAM delete_instance_profile * PR adjustment: positively verifying instance profile deletion in test case. Co-authored-by: Joseph Weitekamp --- moto/iam/models.py | 9 +++++++++ moto/iam/responses.py | 13 +++++++++++++ tests/test_iam/test_iam.py | 20 ++++++++++++++++++++ 3 files changed, 42 insertions(+) diff --git a/moto/iam/models.py b/moto/iam/models.py index 41484add..82dc84be 100755 --- a/moto/iam/models.py +++ b/moto/iam/models.py @@ -1341,6 +1341,15 @@ class IAMBackend(BaseBackend): self.instance_profiles[name] = instance_profile return instance_profile + def delete_instance_profile(self, name): + instance_profile = self.get_instance_profile(name) + if len(instance_profile.roles) > 0: + raise IAMConflictException( + code="DeleteConflict", + message="Cannot delete entity, must remove roles from instance profile first.", + ) + del self.instance_profiles[name] + def get_instance_profile(self, profile_name): for profile in self.get_instance_profiles(): if profile.name == profile_name: diff --git a/moto/iam/responses.py b/moto/iam/responses.py index 667a6d13..60ab4606 100644 --- a/moto/iam/responses.py +++ b/moto/iam/responses.py @@ -305,6 +305,13 @@ class IamResponse(BaseResponse): template = self.response_template(CREATE_INSTANCE_PROFILE_TEMPLATE) return template.render(profile=profile) + def delete_instance_profile(self): + profile_name = self._get_param("InstanceProfileName") + + profile = iam_backend.delete_instance_profile(profile_name) + template = self.response_template(DELETE_INSTANCE_PROFILE_TEMPLATE) + return template.render(profile=profile) + def get_instance_profile(self): profile_name = self._get_param("InstanceProfileName") profile = iam_backend.get_instance_profile(profile_name) @@ -1180,6 +1187,12 @@ CREATE_INSTANCE_PROFILE_TEMPLATE = """ + + 786dff92-6cfd-4fa4-b1eb-27EXAMPLE804 + +""" + GET_INSTANCE_PROFILE_TEMPLATE = """ diff --git a/tests/test_iam/test_iam.py b/tests/test_iam/test_iam.py index 825e12fe..7b59a572 100644 --- a/tests/test_iam/test_iam.py +++ b/tests/test_iam/test_iam.py @@ -206,6 +206,26 @@ def test_remove_role_from_instance_profile(): dict(profile.roles).should.be.empty +@mock_iam() +def test_delete_instance_profile(): + conn = boto3.client("iam", region_name="us-east-1") + conn.create_role( + RoleName="my-role", AssumeRolePolicyDocument="some policy", Path="/my-path/" + ) + conn.create_instance_profile(InstanceProfileName="my-profile") + conn.add_role_to_instance_profile( + InstanceProfileName="my-profile", RoleName="my-role" + ) + with assert_raises(conn.exceptions.DeleteConflictException): + conn.delete_instance_profile(InstanceProfileName="my-profile") + conn.remove_role_from_instance_profile( + InstanceProfileName="my-profile", RoleName="my-role" + ) + conn.delete_instance_profile(InstanceProfileName="my-profile") + with assert_raises(conn.exceptions.NoSuchEntityException): + profile = conn.get_instance_profile(InstanceProfileName="my-profile") + + @mock_iam() def test_get_login_profile(): conn = boto3.client("iam", region_name="us-east-1") From 162a38bb10e1808d53ca9f88d95b39fe2bfc6249 Mon Sep 17 00:00:00 2001 From: Jeremie Tharaud <46786750+jeremietharaud@users.noreply.github.com> Date: Thu, 28 May 2020 15:14:09 +0200 Subject: [PATCH 09/17] fix missing sure package and region_name (#3031) --- .../test_cloudformation_stack_crud_boto3.py | 8 ++++---- tests/test_cloudformation/test_validate.py | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py index 4df1ff5d..58d505d8 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py @@ -572,7 +572,7 @@ def test_boto3_create_stack_set_with_yaml(): @mock_s3 def test_create_stack_set_from_s3_url(): s3 = boto3.client("s3") - s3_conn = boto3.resource("s3") + s3_conn = boto3.resource("s3", region_name="us-east-1") bucket = s3_conn.create_bucket(Bucket="foobar") key = s3_conn.Object("foobar", "template-key").put(Body=dummy_template_json) @@ -715,7 +715,7 @@ def test_create_stack_with_role_arn(): @mock_s3 def test_create_stack_from_s3_url(): s3 = boto3.client("s3") - s3_conn = boto3.resource("s3") + s3_conn = boto3.resource("s3", region_name="us-east-1") bucket = s3_conn.create_bucket(Bucket="foobar") key = s3_conn.Object("foobar", "template-key").put(Body=dummy_template_json) @@ -770,7 +770,7 @@ def test_update_stack_with_previous_value(): @mock_ec2 def test_update_stack_from_s3_url(): s3 = boto3.client("s3") - s3_conn = boto3.resource("s3") + s3_conn = boto3.resource("s3", region_name="us-east-1") cf_conn = boto3.client("cloudformation", region_name="us-east-1") cf_conn.create_stack( @@ -799,7 +799,7 @@ def test_update_stack_from_s3_url(): @mock_s3 def test_create_change_set_from_s3_url(): s3 = boto3.client("s3") - s3_conn = boto3.resource("s3") + s3_conn = boto3.resource("s3", region_name="us-east-1") bucket = s3_conn.create_bucket(Bucket="foobar") key = s3_conn.Object("foobar", "template-key").put(Body=dummy_template_json) diff --git a/tests/test_cloudformation/test_validate.py b/tests/test_cloudformation/test_validate.py index a4278b55..5ffaeafb 100644 --- a/tests/test_cloudformation/test_validate.py +++ b/tests/test_cloudformation/test_validate.py @@ -5,6 +5,7 @@ import os import boto3 from nose.tools import raises import botocore +import sure # noqa from moto.cloudformation.exceptions import ValidationError From 7a6d78afde02c9dc70c6db73f8729d0d7d6a0883 Mon Sep 17 00:00:00 2001 From: usmangani1 Date: Thu, 28 May 2020 20:22:56 +0530 Subject: [PATCH 10/17] Fix: Cloudwatch delete Alarm status code handling on invalid alarm name (#3028) * CloudWwatch delete Alarm status code handling on invalid alarm Name * Handled cases where a mix of existent and non existent alarms are tried to delete * Linting Co-authored-by: usmankb Co-authored-by: Bert Blommers --- moto/cloudwatch/models.py | 7 +++++ .../test_cloudwatch/test_cloudwatch_boto3.py | 31 +++++++++++++++++++ 2 files changed, 38 insertions(+) diff --git a/moto/cloudwatch/models.py b/moto/cloudwatch/models.py index 19b1efa5..f089acb1 100644 --- a/moto/cloudwatch/models.py +++ b/moto/cloudwatch/models.py @@ -304,6 +304,13 @@ class CloudWatchBackend(BaseBackend): ) def delete_alarms(self, alarm_names): + for alarm_name in alarm_names: + if alarm_name not in self.alarms: + raise RESTError( + "ResourceNotFound", + "Alarm {0} not found".format(alarm_name), + status=404, + ) for alarm_name in alarm_names: self.alarms.pop(alarm_name, None) diff --git a/tests/test_cloudwatch/test_cloudwatch_boto3.py b/tests/test_cloudwatch/test_cloudwatch_boto3.py index 0c814ee4..926c321b 100644 --- a/tests/test_cloudwatch/test_cloudwatch_boto3.py +++ b/tests/test_cloudwatch/test_cloudwatch_boto3.py @@ -92,6 +92,37 @@ def test_get_dashboard_fail(): raise RuntimeError("Should of raised error") +@mock_cloudwatch +def test_delete_invalid_alarm(): + cloudwatch = boto3.client("cloudwatch", "eu-west-1") + + cloudwatch.put_metric_alarm( + AlarmName="testalarm1", + MetricName="cpu", + Namespace="blah", + Period=10, + EvaluationPeriods=5, + Statistic="Average", + Threshold=2, + ComparisonOperator="GreaterThanThreshold", + ActionsEnabled=True, + ) + + # trying to delete an alarm which is not created along with valid alarm. + with assert_raises(ClientError) as e: + cloudwatch.delete_alarms(AlarmNames=["InvalidAlarmName", "testalarm1"]) + e.exception.response["Error"]["Code"].should.equal("ResourceNotFound") + + resp = cloudwatch.describe_alarms(AlarmNames=["testalarm1"]) + # making sure other alarms are not deleted in case of an error. + len(resp["MetricAlarms"]).should.equal(1) + + # test to check if the error raises if only one invalid alarm is tried to delete. + with assert_raises(ClientError) as e: + cloudwatch.delete_alarms(AlarmNames=["InvalidAlarmName"]) + e.exception.response["Error"]["Code"].should.equal("ResourceNotFound") + + @mock_cloudwatch def test_alarm_state(): client = boto3.client("cloudwatch", region_name="eu-central-1") From 8fa625c3def8287882c8193bf72bf778510db3d2 Mon Sep 17 00:00:00 2001 From: Jeremie Tharaud <46786750+jeremietharaud@users.noreply.github.com> Date: Fri, 29 May 2020 08:33:24 +0200 Subject: [PATCH 11/17] Cfn change set fix outputs (#3033) * set creation time of the change set * fix status, execution status, stak id, creation time and update tests --- moto/cloudformation/models.py | 6 ++++-- moto/cloudformation/responses.py | 2 +- .../test_cloudformation_stack_crud_boto3.py | 11 ++++++++--- 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index 281ab5e1..16ceafdb 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -240,7 +240,8 @@ class FakeStack(BaseModel): self.resource_map = self._create_resource_map() self.output_map = self._create_output_map() if create_change_set: - self.status = "REVIEW_IN_PROGRESS" + self.status = "CREATE_COMPLETE" + self.execution_status = "AVAILABLE" else: self.create_resources() self._add_stack_event("CREATE_COMPLETE") @@ -397,6 +398,7 @@ class FakeChangeSet(FakeStack): self.change_set_id = change_set_id self.change_set_name = change_set_name self.changes = self.diff(template=template, parameters=parameters) + self.creation_time = datetime.utcnow() def diff(self, template, parameters=None): self.template = template @@ -587,7 +589,7 @@ class CloudFormationBackend(BaseBackend): if stack is None: raise ValidationError(stack_name) else: - stack_id = generate_stack_id(stack_name) + stack_id = generate_stack_id(stack_name, region_name) stack_template = template change_set_id = generate_changeset_id(change_set_name, region_name) diff --git a/moto/cloudformation/responses.py b/moto/cloudformation/responses.py index 782d6894..c028421c 100644 --- a/moto/cloudformation/responses.py +++ b/moto/cloudformation/responses.py @@ -609,7 +609,7 @@ DESCRIBE_CHANGE_SET_RESPONSE_TEMPLATE = """ {% endfor %} - 2011-05-23T15:47:44Z + {{ change_set.creation_time_iso_8601 }} {{ change_set.execution_status }} {{ change_set.status }} {{ change_set.status_reason }} diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py index 58d505d8..c4fddcad 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py @@ -819,7 +819,7 @@ def test_create_change_set_from_s3_url(): in response["Id"] ) assert ( - "arn:aws:cloudformation:us-east-1:123456789:stack/NewStack" + "arn:aws:cloudformation:us-west-1:123456789:stack/NewStack" in response["StackId"] ) @@ -838,7 +838,12 @@ def test_describe_change_set(): stack["ChangeSetName"].should.equal("NewChangeSet") stack["StackName"].should.equal("NewStack") - stack["Status"].should.equal("REVIEW_IN_PROGRESS") + stack["Status"].should.equal("CREATE_COMPLETE") + stack["ExecutionStatus"].should.equal("AVAILABLE") + two_secs_ago = datetime.now(tz=pytz.UTC) - timedelta(seconds=2) + assert ( + two_secs_ago < stack["CreationTime"] < datetime.now(tz=pytz.UTC) + ), "Change set should have been created recently" cf_conn.create_change_set( StackName="NewStack", @@ -868,7 +873,7 @@ def test_execute_change_set_w_arn(): ) ec2.describe_instances()["Reservations"].should.have.length_of(0) cf_conn.describe_change_set(ChangeSetName="NewChangeSet")["Status"].should.equal( - "REVIEW_IN_PROGRESS" + "CREATE_COMPLETE" ) # Execute change set cf_conn.execute_change_set(ChangeSetName=change_set["Id"]) From 2433d64fe264f41481f6266a06d2ead2bef3dbab Mon Sep 17 00:00:00 2001 From: usmangani1 Date: Fri, 29 May 2020 17:01:41 +0530 Subject: [PATCH 12/17] Fix: SecretsManager Added VersionIdsToStages key in describe_secret function (#3029) * Fix: SecretsManager Added VersionIdsToStages key in describe_secret function * Added more assertions * Linting Co-authored-by: usmankb Co-authored-by: Bert Blommers --- moto/secretsmanager/models.py | 10 ++++++++++ tests/test_secretsmanager/test_secretsmanager.py | 12 ++++++++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/moto/secretsmanager/models.py b/moto/secretsmanager/models.py index 29bd6c96..01acf2db 100644 --- a/moto/secretsmanager/models.py +++ b/moto/secretsmanager/models.py @@ -274,6 +274,7 @@ class SecretsManagerBackend(BaseBackend): raise SecretNotFoundException() secret = self.secrets[secret_id] + version_id_to_stages = self.form_version_ids_to_stages(secret["versions"]) response = json.dumps( { @@ -291,6 +292,7 @@ class SecretsManagerBackend(BaseBackend): "LastAccessedDate": None, "DeletedDate": secret.get("deleted_date", None), "Tags": secret["tags"], + "VersionIdsToStages": version_id_to_stages, } ) @@ -552,6 +554,14 @@ class SecretsManagerBackend(BaseBackend): } ) + @staticmethod + def form_version_ids_to_stages(secret): + version_id_to_stages = {} + for key, value in secret.items(): + version_id_to_stages[key] = value["version_stages"] + + return version_id_to_stages + secretsmanager_backends = {} for region in Session().get_available_regions("secretsmanager"): diff --git a/tests/test_secretsmanager/test_secretsmanager.py b/tests/test_secretsmanager/test_secretsmanager.py index 6ec53460..0fe23fd7 100644 --- a/tests/test_secretsmanager/test_secretsmanager.py +++ b/tests/test_secretsmanager/test_secretsmanager.py @@ -733,25 +733,33 @@ def test_put_secret_value_versions_differ_if_same_secret_put_twice(): def test_put_secret_value_maintains_description_and_tags(): conn = boto3.client("secretsmanager", region_name="us-west-2") - conn.create_secret( + previous_response = conn.create_secret( Name=DEFAULT_SECRET_NAME, SecretString="foosecret", Description="desc", Tags=[{"Key": "Foo", "Value": "Bar"}, {"Key": "Mykey", "Value": "Myvalue"}], ) + previous_version_id = previous_response["VersionId"] conn = boto3.client("secretsmanager", region_name="us-west-2") - conn.put_secret_value( + current_response = conn.put_secret_value( SecretId=DEFAULT_SECRET_NAME, SecretString="dupe_secret", VersionStages=["AWSCURRENT"], ) + current_version_id = current_response["VersionId"] + secret_details = conn.describe_secret(SecretId=DEFAULT_SECRET_NAME) assert secret_details["Tags"] == [ {"Key": "Foo", "Value": "Bar"}, {"Key": "Mykey", "Value": "Myvalue"}, ] assert secret_details["Description"] == "desc" + assert secret_details["VersionIdsToStages"] is not None + assert previous_version_id in secret_details["VersionIdsToStages"] + assert current_version_id in secret_details["VersionIdsToStages"] + assert secret_details["VersionIdsToStages"][previous_version_id] == ["AWSPREVIOUS"] + assert secret_details["VersionIdsToStages"][current_version_id] == ["AWSCURRENT"] @mock_secretsmanager From cb600377b48ca676fc6a29a6690aeb51e702da43 Mon Sep 17 00:00:00 2001 From: Victor Le Fichant Date: Tue, 2 Jun 2020 17:31:42 +0200 Subject: [PATCH 13/17] Fix incorrect response for put-targets action (#3037) --- moto/events/responses.py | 5 ++++- tests/test_events/test_events.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/moto/events/responses.py b/moto/events/responses.py index 55a664b2..73db00bd 100644 --- a/moto/events/responses.py +++ b/moto/events/responses.py @@ -217,7 +217,10 @@ class EventsHandler(BaseResponse): "ResourceNotFoundException", "Rule " + rule_name + " does not exist." ) - return "", self.response_headers + return ( + json.dumps({"FailedEntryCount": 0, "FailedEntries": []}), + self.response_headers, + ) def remove_targets(self): rule_name = self._get_param("Rule") diff --git a/tests/test_events/test_events.py b/tests/test_events/test_events.py index 5b4e958d..f83c6076 100644 --- a/tests/test_events/test_events.py +++ b/tests/test_events/test_events.py @@ -4,6 +4,7 @@ import unittest import boto3 import sure # noqa + from botocore.exceptions import ClientError from nose.tools import assert_raises @@ -201,6 +202,35 @@ def test_remove_targets(): assert targets_before - 1 == targets_after +@mock_events +def test_put_targets(): + client = boto3.client("events", "us-west-2") + rule_name = "my-event" + rule_data = { + "Name": rule_name, + "ScheduleExpression": "rate(5 minutes)", + "EventPattern": '{"source": ["test-source"]}', + } + + client.put_rule(**rule_data) + + targets = client.list_targets_by_rule(Rule=rule_name)["Targets"] + targets_before = len(targets) + assert targets_before == 0 + + targets_data = [{"Arn": "test_arn", "Id": "test_id"}] + resp = client.put_targets(Rule=rule_name, Targets=targets_data) + assert resp["FailedEntryCount"] == 0 + assert len(resp["FailedEntries"]) == 0 + + targets = client.list_targets_by_rule(Rule=rule_name)["Targets"] + targets_after = len(targets) + assert targets_before + 1 == targets_after + + assert targets[0]["Arn"] == "test_arn" + assert targets[0]["Id"] == "test_id" + + @mock_events def test_permissions(): client = boto3.client("events", "eu-central-1") From 90e200f0f6e0936c0ccb59c33990831a4eb1cda7 Mon Sep 17 00:00:00 2001 From: Jeremie Tharaud <46786750+jeremietharaud@users.noreply.github.com> Date: Wed, 3 Jun 2020 07:08:35 +0200 Subject: [PATCH 14/17] Add missing changes when creating a change set (#3039) * Display changes when creating a change set * add change set id and description when describing stack * fix lint with flake8 and black --- moto/cloudformation/models.py | 19 +++++++-- moto/cloudformation/parsing.py | 6 ++- moto/cloudformation/responses.py | 4 ++ .../test_cloudformation_stack_crud_boto3.py | 41 +++++++++++++------ .../test_cloudformation_stack_integration.py | 6 --- 5 files changed, 51 insertions(+), 25 deletions(-) diff --git a/moto/cloudformation/models.py b/moto/cloudformation/models.py index 16ceafdb..8c14f55b 100644 --- a/moto/cloudformation/models.py +++ b/moto/cloudformation/models.py @@ -219,7 +219,12 @@ class FakeStack(BaseModel): self.stack_id = stack_id self.name = name self.template = template - self._parse_template() + if template != {}: + self._parse_template() + self.description = self.template_dict.get("Description") + else: + self.template_dict = {} + self.description = None self.parameters = parameters self.region_name = region_name self.notification_arns = notification_arns if notification_arns else [] @@ -235,7 +240,6 @@ class FakeStack(BaseModel): "CREATE_IN_PROGRESS", resource_status_reason="User Initiated" ) - self.description = self.template_dict.get("Description") self.cross_stack_resources = cross_stack_resources or {} self.resource_map = self._create_resource_map() self.output_map = self._create_output_map() @@ -331,7 +335,9 @@ class FakeStack(BaseModel): return self.output_map.exports def create_resources(self): - self.resource_map.create() + self.resource_map.create(self.template_dict) + # Set the description of the stack + self.description = self.template_dict.get("Description") self.status = "CREATE_COMPLETE" def update(self, template, role_arn=None, parameters=None, tags=None): @@ -398,6 +404,8 @@ class FakeChangeSet(FakeStack): self.change_set_id = change_set_id self.change_set_name = change_set_name self.changes = self.diff(template=template, parameters=parameters) + if self.description is None: + self.description = self.template_dict.get("Description") self.creation_time = datetime.utcnow() def diff(self, template, parameters=None): @@ -590,7 +598,7 @@ class CloudFormationBackend(BaseBackend): raise ValidationError(stack_name) else: stack_id = generate_stack_id(stack_name, region_name) - stack_template = template + stack_template = {} change_set_id = generate_changeset_id(change_set_name, region_name) new_change_set = FakeChangeSet( @@ -645,6 +653,9 @@ class CloudFormationBackend(BaseBackend): if stack is None: raise ValidationError(stack_name) if stack.events[-1].resource_status == "REVIEW_IN_PROGRESS": + stack._add_stack_event( + "CREATE_IN_PROGRESS", resource_status_reason="User Initiated" + ) stack._add_stack_event("CREATE_COMPLETE") else: stack._add_stack_event("UPDATE_IN_PROGRESS") diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index d59b21b8..81d4d1c7 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -456,7 +456,7 @@ class ResourceMap(collections_abc.Mapping): cross_stack_resources, ): self._template = template - self._resource_json_map = template["Resources"] + self._resource_json_map = template["Resources"] if template != {} else {} self._region_name = region_name self.input_parameters = parameters self.tags = copy.deepcopy(tags) @@ -592,10 +592,12 @@ class ResourceMap(collections_abc.Mapping): self.load_parameters() self.load_conditions() - def create(self): + def create(self, template): # Since this is a lazy map, to create every object we just need to # iterate through self. # Assumes that self.load() has been called before + self._template = template + self._resource_json_map = template["Resources"] self.tags.update( { "aws:cloudformation:stack-name": self.get("AWS::StackName"), diff --git a/moto/cloudformation/responses.py b/moto/cloudformation/responses.py index c028421c..30284948 100644 --- a/moto/cloudformation/responses.py +++ b/moto/cloudformation/responses.py @@ -662,6 +662,10 @@ DESCRIBE_STACKS_TEMPLATE = """ {{ stack.name }} {{ stack.stack_id }} + {% if stack.change_set_id %} + {{ stack.change_set_id }} + {% endif %} + {{ stack.description }} {{ stack.creation_time_iso_8601 }} {{ stack.status }} {% if stack.notification_arns %} diff --git a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py index c4fddcad..cd76743d 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py +++ b/tests/test_cloudformation/test_cloudformation_stack_crud_boto3.py @@ -573,9 +573,9 @@ def test_boto3_create_stack_set_with_yaml(): def test_create_stack_set_from_s3_url(): s3 = boto3.client("s3") s3_conn = boto3.resource("s3", region_name="us-east-1") - bucket = s3_conn.create_bucket(Bucket="foobar") + s3_conn.create_bucket(Bucket="foobar") - key = s3_conn.Object("foobar", "template-key").put(Body=dummy_template_json) + s3_conn.Object("foobar", "template-key").put(Body=dummy_template_json) key_url = s3.generate_presigned_url( ClientMethod="get_object", Params={"Bucket": "foobar", "Key": "template-key"} ) @@ -716,9 +716,9 @@ def test_create_stack_with_role_arn(): def test_create_stack_from_s3_url(): s3 = boto3.client("s3") s3_conn = boto3.resource("s3", region_name="us-east-1") - bucket = s3_conn.create_bucket(Bucket="foobar") + s3_conn.create_bucket(Bucket="foobar") - key = s3_conn.Object("foobar", "template-key").put(Body=dummy_template_json) + s3_conn.Object("foobar", "template-key").put(Body=dummy_template_json) key_url = s3.generate_presigned_url( ClientMethod="get_object", Params={"Bucket": "foobar", "Key": "template-key"} ) @@ -800,9 +800,9 @@ def test_update_stack_from_s3_url(): def test_create_change_set_from_s3_url(): s3 = boto3.client("s3") s3_conn = boto3.resource("s3", region_name="us-east-1") - bucket = s3_conn.create_bucket(Bucket="foobar") + s3_conn.create_bucket(Bucket="foobar") - key = s3_conn.Object("foobar", "template-key").put(Body=dummy_template_json) + s3_conn.Object("foobar", "template-key").put(Body=dummy_template_json) key_url = s3.generate_presigned_url( ClientMethod="get_object", Params={"Bucket": "foobar", "Key": "template-key"} ) @@ -844,6 +844,25 @@ def test_describe_change_set(): assert ( two_secs_ago < stack["CreationTime"] < datetime.now(tz=pytz.UTC) ), "Change set should have been created recently" + stack["Changes"].should.have.length_of(1) + stack["Changes"][0].should.equal( + dict( + { + "Type": "Resource", + "ResourceChange": { + "Action": "Add", + "LogicalResourceId": "EC2Instance1", + "ResourceType": "AWS::EC2::Instance", + }, + } + ) + ) + + # Execute change set + cf_conn.execute_change_set(ChangeSetName="NewChangeSet") + # Verify that the changes have been applied + stack = cf_conn.describe_change_set(ChangeSetName="NewChangeSet") + stack["Changes"].should.have.length_of(1) cf_conn.create_change_set( StackName="NewStack", @@ -887,7 +906,7 @@ def test_execute_change_set_w_arn(): @mock_cloudformation def test_execute_change_set_w_name(): cf_conn = boto3.client("cloudformation", region_name="us-east-1") - change_set = cf_conn.create_change_set( + cf_conn.create_change_set( StackName="NewStack", TemplateBody=dummy_template_json, ChangeSetName="NewChangeSet", @@ -1221,9 +1240,7 @@ def test_delete_stack_with_export(): @mock_cloudformation def test_export_names_must_be_unique(): cf = boto3.resource("cloudformation", region_name="us-east-1") - first_stack = cf.create_stack( - StackName="test_stack", TemplateBody=dummy_output_template_json - ) + cf.create_stack(StackName="test_stack", TemplateBody=dummy_output_template_json) with assert_raises(ClientError): cf.create_stack(StackName="test_stack", TemplateBody=dummy_output_template_json) @@ -1237,9 +1254,7 @@ def test_stack_with_imports(): output_stack = cf.create_stack( StackName="test_stack1", TemplateBody=dummy_output_template_json ) - import_stack = cf.create_stack( - StackName="test_stack2", TemplateBody=dummy_import_template_json - ) + cf.create_stack(StackName="test_stack2", TemplateBody=dummy_import_template_json) output_stack.outputs.should.have.length_of(1) output = output_stack.outputs[0]["OutputValue"] diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index 3abb3373..a49c4a1f 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -1,7 +1,6 @@ from __future__ import unicode_literals import json -import base64 from decimal import Decimal import boto @@ -28,7 +27,6 @@ from moto import ( mock_dynamodb2, mock_ec2, mock_ec2_deprecated, - mock_elb, mock_elb_deprecated, mock_events, mock_iam_deprecated, @@ -37,18 +35,14 @@ from moto import ( mock_logs, mock_rds_deprecated, mock_rds2, - mock_rds2_deprecated, - mock_redshift, mock_redshift_deprecated, mock_route53_deprecated, mock_s3, mock_sns_deprecated, - mock_sqs, mock_sqs_deprecated, mock_elbv2, ) from moto.core import ACCOUNT_ID -from moto.dynamodb2.models import Table from tests.test_cloudformation.fixtures import ( ec2_classic_eip, From 149e307bc9421de5780febf9ab4a734e72cbee9d Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Wed, 3 Jun 2020 02:54:01 -0300 Subject: [PATCH 15/17] Rule's cloudformation support for updates (#3043) * add support to update stack using cloudformation * blacked test file --- moto/events/models.py | 9 +++++ .../test_cloudformation_stack_integration.py | 40 +++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/moto/events/models.py b/moto/events/models.py index e1224242..5397f28c 100644 --- a/moto/events/models.py +++ b/moto/events/models.py @@ -80,6 +80,15 @@ class Rule(BaseModel): event_name = properties.get("Name") or resource_name return event_backend.put_rule(name=event_name, **properties) + @classmethod + def update_from_cloudformation_json( + cls, original_resource, new_resource_name, cloudformation_json, region_name + ): + original_resource.delete(region_name) + return cls.create_from_cloudformation_json( + new_resource_name, cloudformation_json, region_name + ) + @classmethod def delete_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index a49c4a1f..082a20e0 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -17,6 +17,7 @@ import boto.sqs import boto.vpc import boto3 import sure # noqa +from string import Template from moto import ( mock_autoscaling_deprecated, @@ -2493,6 +2494,45 @@ def test_stack_events_create_rule_as_target(): log_groups["logGroups"][0]["retentionInDays"].should.equal(3) +@mock_cloudformation +@mock_events +def test_stack_events_update_rule_integration(): + events_template = Template( + """{ + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "Event": { + "Type": "AWS::Events::Rule", + "Properties": { + "Name": "$Name", + "State": "$State", + "ScheduleExpression": "rate(5 minutes)", + }, + } + }, + } """ + ) + + cf_conn = boto3.client("cloudformation", "us-west-2") + + original_template = events_template.substitute(Name="Foo", State="ENABLED") + cf_conn.create_stack(StackName="test_stack", TemplateBody=original_template) + + rules = boto3.client("events", "us-west-2").list_rules() + rules["Rules"].should.have.length_of(1) + rules["Rules"][0]["Name"].should.equal("Foo") + rules["Rules"][0]["State"].should.equal("ENABLED") + + update_template = events_template.substitute(Name="Bar", State="DISABLED") + cf_conn.update_stack(StackName="test_stack", TemplateBody=update_template) + + rules = boto3.client("events", "us-west-2").list_rules() + + rules["Rules"].should.have.length_of(1) + rules["Rules"][0]["Name"].should.equal("Bar") + rules["Rules"][0]["State"].should.equal("DISABLED") + + @mock_cloudformation @mock_autoscaling def test_autoscaling_propagate_tags(): From 029b2a9751817ed34bb43027c196a76cc97edadc Mon Sep 17 00:00:00 2001 From: Matt Bullock Date: Thu, 4 Jun 2020 06:04:53 -0700 Subject: [PATCH 16/17] chore: refine python-jose dependency (#3049) --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 994e5530..707a5621 100755 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ install_requires = [ "PyYAML>=5.1", "pytz", "python-dateutil<3.0.0,>=2.1", - "python-jose<4.0.0", + "python-jose[cryptography]>=3.1.0,<4.0.0", "docker>=2.5.1", "jsondiff>=1.1.2", "aws-xray-sdk!=0.96,>=0.93", From e32a60185fb7369d8916ee077fbabbc1bf9747ed Mon Sep 17 00:00:00 2001 From: Guilherme Martins Crocetti Date: Sat, 6 Jun 2020 07:31:14 -0300 Subject: [PATCH 17/17] Cloudformation - EventBus support (#3052) * add EventBus to model's map * add support for creation of EventBus through cloudformation's api * add cloudformation's delete * add cloudformation's update * add cloudformation's attribute --- moto/cloudformation/parsing.py | 1 + moto/events/models.py | 49 ++++- .../test_cloudformation_stack_integration.py | 175 ++++++++++++++++++ 3 files changed, 223 insertions(+), 2 deletions(-) diff --git a/moto/cloudformation/parsing.py b/moto/cloudformation/parsing.py index 81d4d1c7..05ebdace 100644 --- a/moto/cloudformation/parsing.py +++ b/moto/cloudformation/parsing.py @@ -96,6 +96,7 @@ MODEL_MAP = { "AWS::S3::Bucket": s3_models.FakeBucket, "AWS::SQS::Queue": sqs_models.Queue, "AWS::Events::Rule": events_models.Rule, + "AWS::Events::EventBus": events_models.EventBus, } UNDOCUMENTED_NAME_TYPE_MAP = { diff --git a/moto/events/models.py b/moto/events/models.py index 5397f28c..360c8d63 100644 --- a/moto/events/models.py +++ b/moto/events/models.py @@ -134,6 +134,52 @@ class EventBus(BaseModel): return json.dumps(policy) + def delete(self, region_name): + event_backend = events_backends[region_name] + event_backend.delete_event_bus(name=self.name) + + def get_cfn_attribute(self, attribute_name): + from moto.cloudformation.exceptions import UnformattedGetAttTemplateException + + if attribute_name == "Arn": + return self.arn + elif attribute_name == "Name": + return self.name + elif attribute_name == "Policy": + return self.policy + + raise UnformattedGetAttTemplateException() + + @classmethod + def create_from_cloudformation_json( + cls, resource_name, cloudformation_json, region_name + ): + properties = cloudformation_json["Properties"] + event_backend = events_backends[region_name] + event_name = properties["Name"] + event_source_name = properties.get("EventSourceName") + return event_backend.create_event_bus( + name=event_name, event_source_name=event_source_name + ) + + @classmethod + def update_from_cloudformation_json( + cls, original_resource, new_resource_name, cloudformation_json, region_name + ): + original_resource.delete(region_name) + return cls.create_from_cloudformation_json( + new_resource_name, cloudformation_json, region_name + ) + + @classmethod + def delete_from_cloudformation_json( + cls, resource_name, cloudformation_json, region_name + ): + properties = cloudformation_json["Properties"] + event_backend = events_backends[region_name] + event_bus_name = properties["Name"] + event_backend.delete_event_bus(event_bus_name) + class EventsBackend(BaseBackend): ACCOUNT_ID = re.compile(r"^(\d{1,12}|\*)$") @@ -369,7 +415,7 @@ class EventsBackend(BaseBackend): return event_bus - def create_event_bus(self, name, event_source_name): + def create_event_bus(self, name, event_source_name=None): if name in self.event_buses: raise JsonRESTError( "ResourceAlreadyExistsException", @@ -406,7 +452,6 @@ class EventsBackend(BaseBackend): raise JsonRESTError( "ValidationException", "Cannot delete event bus default." ) - self.event_buses.pop(name, None) def list_tags_for_resource(self, arn): diff --git a/tests/test_cloudformation/test_cloudformation_stack_integration.py b/tests/test_cloudformation/test_cloudformation_stack_integration.py index 082a20e0..fec9891a 100644 --- a/tests/test_cloudformation/test_cloudformation_stack_integration.py +++ b/tests/test_cloudformation/test_cloudformation_stack_integration.py @@ -2585,3 +2585,178 @@ def test_autoscaling_propagate_tags(): assert propagation_dict["test-key-propagate"] assert not propagation_dict["test-key-no-propagate"] + + +@mock_cloudformation +@mock_events +def test_stack_eventbus_create_from_cfn_integration(): + eventbus_template = """{ + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "EventBus": { + "Type": "AWS::Events::EventBus", + "Properties": { + "Name": "MyCustomEventBus" + }, + } + }, + }""" + + cf_conn = boto3.client("cloudformation", "us-west-2") + cf_conn.create_stack(StackName="test_stack", TemplateBody=eventbus_template) + + event_buses = boto3.client("events", "us-west-2").list_event_buses( + NamePrefix="MyCustom" + ) + + event_buses["EventBuses"].should.have.length_of(1) + event_buses["EventBuses"][0]["Name"].should.equal("MyCustomEventBus") + + +@mock_cloudformation +@mock_events +def test_stack_events_delete_eventbus_integration(): + eventbus_template = """{ + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "EventBus": { + "Type": "AWS::Events::EventBus", + "Properties": { + "Name": "MyCustomEventBus" + }, + } + }, + }""" + cf_conn = boto3.client("cloudformation", "us-west-2") + cf_conn.create_stack(StackName="test_stack", TemplateBody=eventbus_template) + + event_buses = boto3.client("events", "us-west-2").list_event_buses( + NamePrefix="MyCustom" + ) + event_buses["EventBuses"].should.have.length_of(1) + + cf_conn.delete_stack(StackName="test_stack") + + event_buses = boto3.client("events", "us-west-2").list_event_buses( + NamePrefix="MyCustom" + ) + event_buses["EventBuses"].should.have.length_of(0) + + +@mock_cloudformation +@mock_events +def test_stack_events_delete_from_cfn_integration(): + eventbus_template = Template( + """{ + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "$resource_name": { + "Type": "AWS::Events::EventBus", + "Properties": { + "Name": "$name" + }, + } + }, + }""" + ) + + cf_conn = boto3.client("cloudformation", "us-west-2") + + original_template = eventbus_template.substitute( + {"resource_name": "original", "name": "MyCustomEventBus"} + ) + cf_conn.create_stack(StackName="test_stack", TemplateBody=original_template) + + original_event_buses = boto3.client("events", "us-west-2").list_event_buses( + NamePrefix="MyCustom" + ) + original_event_buses["EventBuses"].should.have.length_of(1) + + original_eventbus = original_event_buses["EventBuses"][0] + + updated_template = eventbus_template.substitute( + {"resource_name": "updated", "name": "AnotherEventBus"} + ) + cf_conn.update_stack(StackName="test_stack", TemplateBody=updated_template) + + update_event_buses = boto3.client("events", "us-west-2").list_event_buses( + NamePrefix="AnotherEventBus" + ) + update_event_buses["EventBuses"].should.have.length_of(1) + update_event_buses["EventBuses"][0]["Arn"].shouldnt.equal(original_eventbus["Arn"]) + + +@mock_cloudformation +@mock_events +def test_stack_events_update_from_cfn_integration(): + eventbus_template = Template( + """{ + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "EventBus": { + "Type": "AWS::Events::EventBus", + "Properties": { + "Name": "$name" + }, + } + }, + }""" + ) + + cf_conn = boto3.client("cloudformation", "us-west-2") + + original_template = eventbus_template.substitute({"name": "MyCustomEventBus"}) + cf_conn.create_stack(StackName="test_stack", TemplateBody=original_template) + + original_event_buses = boto3.client("events", "us-west-2").list_event_buses( + NamePrefix="MyCustom" + ) + original_event_buses["EventBuses"].should.have.length_of(1) + + original_eventbus = original_event_buses["EventBuses"][0] + + updated_template = eventbus_template.substitute({"name": "NewEventBus"}) + cf_conn.update_stack(StackName="test_stack", TemplateBody=updated_template) + + update_event_buses = boto3.client("events", "us-west-2").list_event_buses( + NamePrefix="NewEventBus" + ) + update_event_buses["EventBuses"].should.have.length_of(1) + update_event_buses["EventBuses"][0]["Name"].should.equal("NewEventBus") + update_event_buses["EventBuses"][0]["Arn"].shouldnt.equal(original_eventbus["Arn"]) + + +@mock_cloudformation +@mock_events +def test_stack_events_get_attribute_integration(): + eventbus_template = """{ + "AWSTemplateFormatVersion": "2010-09-09", + "Resources": { + "EventBus": { + "Type": "AWS::Events::EventBus", + "Properties": { + "Name": "MyEventBus" + }, + } + }, + "Outputs": { + "bus_arn": {"Value": {"Fn::GetAtt": ["EventBus", "Arn"]}}, + "bus_name": {"Value": {"Fn::GetAtt": ["EventBus", "Name"]}}, + } + }""" + + cf = boto3.client("cloudformation", "us-west-2") + events = boto3.client("events", "us-west-2") + + cf.create_stack(StackName="test_stack", TemplateBody=eventbus_template) + + stack = cf.describe_stacks(StackName="test_stack")["Stacks"][0] + outputs = stack["Outputs"] + + output_arn = list(filter(lambda item: item["OutputKey"] == "bus_arn", outputs))[0] + output_name = list(filter(lambda item: item["OutputKey"] == "bus_name", outputs))[0] + + event_bus = events.list_event_buses(NamePrefix="MyEventBus")["EventBuses"][0] + + output_arn["OutputValue"].should.equal(event_bus["Arn"]) + output_name["OutputValue"].should.equal(event_bus["Name"])