diff --git a/moto/awslambda/models.py b/moto/awslambda/models.py index 939952d5..9cdf2397 100644 --- a/moto/awslambda/models.py +++ b/moto/awslambda/models.py @@ -981,7 +981,7 @@ class LambdaBackend(BaseBackend): ] } func = self._lambdas.get_arn(function_arn) - func.invoke(json.dumps(event), {}, {}) + return func.invoke(json.dumps(event), {}, {}) def list_tags(self, resource): return self.get_function_by_arn(resource).tags diff --git a/moto/config/exceptions.py b/moto/config/exceptions.py index 4a0dc0d7..6b6498d3 100644 --- a/moto/config/exceptions.py +++ b/moto/config/exceptions.py @@ -366,3 +366,13 @@ class TooManyResourceKeys(JsonRESTError): message = str(message) super(TooManyResourceKeys, self).__init__("ValidationException", message) + + +class InvalidResultTokenException(JsonRESTError): + code = 400 + + def __init__(self): + message = "The resultToken provided is invalid" + super(InvalidResultTokenException, self).__init__( + "InvalidResultTokenException", message + ) diff --git a/moto/config/models.py b/moto/config/models.py index a6657697..242a219e 100644 --- a/moto/config/models.py +++ b/moto/config/models.py @@ -40,6 +40,7 @@ from moto.config.exceptions import ( TooManyResourceIds, ResourceNotDiscoveredException, TooManyResourceKeys, + InvalidResultTokenException, ) from moto.core import BaseBackend, BaseModel @@ -1089,6 +1090,26 @@ class ConfigBackend(BaseBackend): "UnprocessedResourceIdentifiers": not_found, } + def put_evaluations(self, evaluations=None, result_token=None, test_mode=False): + if not evaluations: + raise InvalidParameterValueException( + "The Evaluations object in your request cannot be null." + "Add the required parameters and try again." + ) + + if not result_token: + raise InvalidResultTokenException() + + # Moto only supports PutEvaluations with test mode currently (missing rule and token support) + if not test_mode: + raise NotImplementedError( + "PutEvaluations without TestMode is not yet implemented" + ) + + return { + "FailedEvaluations": [], + } # At this time, moto is not adding failed evaluations. + config_backends = {} for region in Session().get_available_regions("config"): diff --git a/moto/config/responses.py b/moto/config/responses.py index e977945c..3b647b5b 100644 --- a/moto/config/responses.py +++ b/moto/config/responses.py @@ -151,3 +151,11 @@ class ConfigResponse(BaseResponse): self._get_param("ResourceIdentifiers"), ) return json.dumps(schema) + + def put_evaluations(self): + evaluations = self.config_backend.put_evaluations( + self._get_param("Evaluations"), + self._get_param("ResultToken"), + self._get_param("TestMode"), + ) + return json.dumps(evaluations) diff --git a/moto/core/models.py b/moto/core/models.py index ffb2ffd9..8ca74d5b 100644 --- a/moto/core/models.py +++ b/moto/core/models.py @@ -7,6 +7,7 @@ import inspect import os import re import six +import types from io import BytesIO from collections import defaultdict from botocore.handlers import BUILTIN_HANDLERS @@ -217,12 +218,29 @@ botocore_mock = responses.RequestsMock( assert_all_requests_are_fired=False, target="botocore.vendored.requests.adapters.HTTPAdapter.send", ) + responses_mock = responses._default_mock # Add passthrough to allow any other requests to work # Since this uses .startswith, it applies to http and https requests. responses_mock.add_passthru("http") +def _find_first_match(self, request): + for i, match in enumerate(self._matches): + if match.matches(request): + return match + + return None + + +# Modify behaviour of the matcher to only/always return the first match +# Default behaviour is to return subsequent matches for subsequent requests, which leads to https://github.com/spulec/moto/issues/2567 +# - First request matches on the appropriate S3 URL +# - Same request, executed again, will be matched on the subsequent match, which happens to be the catch-all, not-yet-implemented, callback +# Fix: Always return the first match +responses_mock._find_match = types.MethodType(_find_first_match, responses_mock) + + BOTOCORE_HTTP_METHODS = ["GET", "DELETE", "HEAD", "OPTIONS", "PATCH", "POST", "PUT"] diff --git a/moto/ec2/models.py b/moto/ec2/models.py index 166d8e64..be39bab2 100644 --- a/moto/ec2/models.py +++ b/moto/ec2/models.py @@ -822,6 +822,21 @@ class Instance(TaggedEC2Resource, BotoInstance): return self.public_ip raise UnformattedGetAttTemplateException() + def applies(self, filters): + if filters: + applicable = False + for f in filters: + acceptable_values = f["values"] + if f["name"] == "instance-state-name": + if self._state.name in acceptable_values: + applicable = True + if f["name"] == "instance-state-code": + if str(self._state.code) in acceptable_values: + applicable = True + return applicable + # If there are no filters, all instances are valid + return True + class InstanceBackend(object): def __init__(self): @@ -921,22 +936,23 @@ class InstanceBackend(object): value = getattr(instance, key) return instance, value - def all_instances(self): + def all_instances(self, filters=None): instances = [] for reservation in self.all_reservations(): for instance in reservation.instances: - instances.append(instance) - return instances - - def all_running_instances(self): - instances = [] - for reservation in self.all_reservations(): - for instance in reservation.instances: - if instance.state_code == 16: + if instance.applies(filters): instances.append(instance) return instances - def get_multi_instances_by_id(self, instance_ids): + def all_running_instances(self, filters=None): + instances = [] + for reservation in self.all_reservations(): + for instance in reservation.instances: + if instance.state_code == 16 and instance.applies(filters): + instances.append(instance) + return instances + + def get_multi_instances_by_id(self, instance_ids, filters=None): """ :param instance_ids: A string list with instance ids :return: A list with instance objects @@ -946,7 +962,8 @@ class InstanceBackend(object): for reservation in self.all_reservations(): for instance in reservation.instances: if instance.id in instance_ids: - result.append(instance) + if instance.applies(filters): + result.append(instance) # TODO: Trim error message down to specific invalid id. if instance_ids and len(instance_ids) > len(result): @@ -1722,6 +1739,12 @@ class SecurityGroup(TaggedEC2Resource): self.vpc_id = vpc_id self.owner_id = OWNER_ID + # Append default IPv6 egress rule for VPCs with IPv6 support + if vpc_id: + vpc = self.ec2_backend.vpcs.get(vpc_id) + if vpc and len(vpc.get_cidr_block_association_set(ipv6=True)) > 0: + self.egress_rules.append(SecurityRule("-1", None, None, [], [])) + @classmethod def create_from_cloudformation_json( cls, resource_name, cloudformation_json, region_name diff --git a/moto/ec2/responses/instances.py b/moto/ec2/responses/instances.py index b9e572d2..29c346f8 100644 --- a/moto/ec2/responses/instances.py +++ b/moto/ec2/responses/instances.py @@ -113,16 +113,34 @@ class InstanceResponse(BaseResponse): template = self.response_template(EC2_START_INSTANCES) return template.render(instances=instances) + def _get_list_of_dict_params(self, param_prefix, _dct): + """ + Simplified version of _get_dict_param + Allows you to pass in a custom dict instead of using self.querystring by default + """ + params = [] + for key, value in _dct.items(): + if key.startswith(param_prefix): + params.append(value) + return params + def describe_instance_status(self): instance_ids = self._get_multi_param("InstanceId") include_all_instances = self._get_param("IncludeAllInstances") == "true" + filters = self._get_list_prefix("Filter") + filters = [ + {"name": f["name"], "values": self._get_list_of_dict_params("value.", f)} + for f in filters + ] if instance_ids: - instances = self.ec2_backend.get_multi_instances_by_id(instance_ids) + instances = self.ec2_backend.get_multi_instances_by_id( + instance_ids, filters + ) elif include_all_instances: - instances = self.ec2_backend.all_instances() + instances = self.ec2_backend.all_instances(filters) else: - instances = self.ec2_backend.all_running_instances() + instances = self.ec2_backend.all_running_instances(filters) template = self.response_template(EC2_INSTANCE_STATUS) return template.render(instances=instances) diff --git a/moto/ecr/exceptions.py b/moto/ecr/exceptions.py index 9b55f058..6d1713a6 100644 --- a/moto/ecr/exceptions.py +++ b/moto/ecr/exceptions.py @@ -1,5 +1,5 @@ from __future__ import unicode_literals -from moto.core.exceptions import RESTError +from moto.core.exceptions import RESTError, JsonRESTError class RepositoryNotFoundException(RESTError): @@ -13,7 +13,7 @@ class RepositoryNotFoundException(RESTError): ) -class ImageNotFoundException(RESTError): +class ImageNotFoundException(JsonRESTError): code = 400 def __init__(self, image_id, repository_name, registry_id): diff --git a/moto/emr/models.py b/moto/emr/models.py index d9ec2fd6..72c58816 100644 --- a/moto/emr/models.py +++ b/moto/emr/models.py @@ -35,6 +35,7 @@ class FakeInstanceGroup(BaseModel): name=None, id=None, bid_price=None, + ebs_configuration=None, ): self.id = id or random_instance_group_id() @@ -51,6 +52,7 @@ class FakeInstanceGroup(BaseModel): self.num_instances = instance_count self.role = instance_role self.type = instance_type + self.ebs_configuration = ebs_configuration self.creation_datetime = datetime.now(pytz.utc) self.start_datetime = datetime.now(pytz.utc) diff --git a/moto/emr/responses.py b/moto/emr/responses.py index 38b9774e..3708db0e 100644 --- a/moto/emr/responses.py +++ b/moto/emr/responses.py @@ -73,6 +73,8 @@ class ElasticMapReduceResponse(BaseResponse): instance_groups = self._get_list_prefix("InstanceGroups.member") for item in instance_groups: item["instance_count"] = int(item["instance_count"]) + # Adding support to EbsConfiguration + self._parse_ebs_configuration(item) instance_groups = self.backend.add_instance_groups(jobflow_id, instance_groups) template = self.response_template(ADD_INSTANCE_GROUPS_TEMPLATE) return template.render(instance_groups=instance_groups) @@ -324,6 +326,8 @@ class ElasticMapReduceResponse(BaseResponse): if instance_groups: for ig in instance_groups: ig["instance_count"] = int(ig["instance_count"]) + # Adding support to EbsConfiguration + self._parse_ebs_configuration(ig) self.backend.add_instance_groups(cluster.id, instance_groups) tags = self._get_list_prefix("Tags.member") @@ -335,6 +339,85 @@ class ElasticMapReduceResponse(BaseResponse): template = self.response_template(RUN_JOB_FLOW_TEMPLATE) return template.render(cluster=cluster) + def _has_key_prefix(self, key_prefix, value): + for key in value: # iter on both keys and values + if key.startswith(key_prefix): + return True + return False + + def _parse_ebs_configuration(self, instance_group): + key_ebs_config = "ebs_configuration" + ebs_configuration = dict() + # Filter only EBS config keys + for key in instance_group: + if key.startswith(key_ebs_config): + ebs_configuration[key] = instance_group[key] + + if len(ebs_configuration) > 0: + # Key that should be extracted + ebs_optimized = "ebs_optimized" + ebs_block_device_configs = "ebs_block_device_configs" + volume_specification = "volume_specification" + size_in_gb = "size_in_gb" + volume_type = "volume_type" + iops = "iops" + volumes_per_instance = "volumes_per_instance" + + key_ebs_optimized = "{0}._{1}".format(key_ebs_config, ebs_optimized) + # EbsOptimized config + if key_ebs_optimized in ebs_configuration: + instance_group.pop(key_ebs_optimized) + ebs_configuration[ebs_optimized] = ebs_configuration.pop( + key_ebs_optimized + ) + + # Ebs Blocks + ebs_blocks = [] + idx = 1 + keyfmt = "{0}._{1}.member.{{}}".format( + key_ebs_config, ebs_block_device_configs + ) + key = keyfmt.format(idx) + while self._has_key_prefix(key, ebs_configuration): + vlespc_keyfmt = "{0}._{1}._{{}}".format(key, volume_specification) + vol_size = vlespc_keyfmt.format(size_in_gb) + vol_iops = vlespc_keyfmt.format(iops) + vol_type = vlespc_keyfmt.format(volume_type) + + ebs_block = dict() + ebs_block[volume_specification] = dict() + if vol_size in ebs_configuration: + instance_group.pop(vol_size) + ebs_block[volume_specification][size_in_gb] = int( + ebs_configuration.pop(vol_size) + ) + if vol_iops in ebs_configuration: + instance_group.pop(vol_iops) + ebs_block[volume_specification][iops] = ebs_configuration.pop( + vol_iops + ) + if vol_type in ebs_configuration: + instance_group.pop(vol_type) + ebs_block[volume_specification][ + volume_type + ] = ebs_configuration.pop(vol_type) + + per_instance = "{0}._{1}".format(key, volumes_per_instance) + if per_instance in ebs_configuration: + instance_group.pop(per_instance) + ebs_block[volumes_per_instance] = int( + ebs_configuration.pop(per_instance) + ) + + if len(ebs_block) > 0: + ebs_blocks.append(ebs_block) + idx += 1 + key = keyfmt.format(idx) + + if len(ebs_blocks) > 0: + ebs_configuration[ebs_block_device_configs] = ebs_blocks + instance_group[key_ebs_config] = ebs_configuration + @generate_boto3_response("SetTerminationProtection") def set_termination_protection(self): termination_protection = self._get_param("TerminationProtected") @@ -754,7 +837,22 @@ LIST_INSTANCE_GROUPS_TEMPLATE = """