Run black on moto & test directories.

This commit is contained in:
Asher Foa 2019-10-31 08:44:26 -07:00
commit 96e5b1993d
507 changed files with 52541 additions and 47814 deletions

View file

@ -28,7 +28,12 @@ from moto.s3 import models as s3_models
from moto.sns import models as sns_models
from moto.sqs import models as sqs_models
from .utils import random_suffix
from .exceptions import ExportNotFound, MissingParameterError, UnformattedGetAttTemplateException, ValidationError
from .exceptions import (
ExportNotFound,
MissingParameterError,
UnformattedGetAttTemplateException,
ValidationError,
)
from boto.cloudformation.stack import Output
MODEL_MAP = {
@ -100,7 +105,7 @@ NAME_TYPE_MAP = {
"AWS::RDS::DBInstance": "DBInstanceIdentifier",
"AWS::S3::Bucket": "BucketName",
"AWS::SNS::Topic": "TopicName",
"AWS::SQS::Queue": "QueueName"
"AWS::SQS::Queue": "QueueName",
}
# Just ignore these models types for now
@ -109,13 +114,12 @@ NULL_MODELS = [
"AWS::CloudFormation::WaitConditionHandle",
]
DEFAULT_REGION = 'us-east-1'
DEFAULT_REGION = "us-east-1"
logger = logging.getLogger("moto")
class LazyDict(dict):
def __getitem__(self, key):
val = dict.__getitem__(self, key)
if callable(val):
@ -132,10 +136,10 @@ def clean_json(resource_json, resources_map):
Eventually, this is where we would add things like function parsing (fn::)
"""
if isinstance(resource_json, dict):
if 'Ref' in resource_json:
if "Ref" in resource_json:
# Parse resource reference
resource = resources_map[resource_json['Ref']]
if hasattr(resource, 'physical_resource_id'):
resource = resources_map[resource_json["Ref"]]
if hasattr(resource, "physical_resource_id"):
return resource.physical_resource_id
else:
return resource
@ -148,74 +152,92 @@ def clean_json(resource_json, resources_map):
result = result[clean_json(path, resources_map)]
return result
if 'Fn::GetAtt' in resource_json:
resource = resources_map.get(resource_json['Fn::GetAtt'][0])
if "Fn::GetAtt" in resource_json:
resource = resources_map.get(resource_json["Fn::GetAtt"][0])
if resource is None:
return resource_json
try:
return resource.get_cfn_attribute(resource_json['Fn::GetAtt'][1])
return resource.get_cfn_attribute(resource_json["Fn::GetAtt"][1])
except NotImplementedError as n:
logger.warning(str(n).format(
resource_json['Fn::GetAtt'][0]))
logger.warning(str(n).format(resource_json["Fn::GetAtt"][0]))
except UnformattedGetAttTemplateException:
raise ValidationError(
'Bad Request',
"Bad Request",
UnformattedGetAttTemplateException.description.format(
resource_json['Fn::GetAtt'][0], resource_json['Fn::GetAtt'][1]))
resource_json["Fn::GetAtt"][0], resource_json["Fn::GetAtt"][1]
),
)
if 'Fn::If' in resource_json:
condition_name, true_value, false_value = resource_json['Fn::If']
if "Fn::If" in resource_json:
condition_name, true_value, false_value = resource_json["Fn::If"]
if resources_map.lazy_condition_map[condition_name]:
return clean_json(true_value, resources_map)
else:
return clean_json(false_value, resources_map)
if 'Fn::Join' in resource_json:
join_list = clean_json(resource_json['Fn::Join'][1], resources_map)
return resource_json['Fn::Join'][0].join([str(x) for x in join_list])
if "Fn::Join" in resource_json:
join_list = clean_json(resource_json["Fn::Join"][1], resources_map)
return resource_json["Fn::Join"][0].join([str(x) for x in join_list])
if 'Fn::Split' in resource_json:
to_split = clean_json(resource_json['Fn::Split'][1], resources_map)
return to_split.split(resource_json['Fn::Split'][0])
if "Fn::Split" in resource_json:
to_split = clean_json(resource_json["Fn::Split"][1], resources_map)
return to_split.split(resource_json["Fn::Split"][0])
if 'Fn::Select' in resource_json:
select_index = int(resource_json['Fn::Select'][0])
select_list = clean_json(resource_json['Fn::Select'][1], resources_map)
if "Fn::Select" in resource_json:
select_index = int(resource_json["Fn::Select"][0])
select_list = clean_json(resource_json["Fn::Select"][1], resources_map)
return select_list[select_index]
if 'Fn::Sub' in resource_json:
if isinstance(resource_json['Fn::Sub'], list):
if "Fn::Sub" in resource_json:
if isinstance(resource_json["Fn::Sub"], list):
warnings.warn(
"Tried to parse Fn::Sub with variable mapping but it's not supported by moto's CloudFormation implementation")
"Tried to parse Fn::Sub with variable mapping but it's not supported by moto's CloudFormation implementation"
)
else:
fn_sub_value = clean_json(resource_json['Fn::Sub'], resources_map)
fn_sub_value = clean_json(resource_json["Fn::Sub"], resources_map)
to_sub = re.findall('(?=\${)[^!^"]*?}', fn_sub_value)
literals = re.findall('(?=\${!)[^"]*?}', fn_sub_value)
for sub in to_sub:
if '.' in sub:
cleaned_ref = clean_json({'Fn::GetAtt': re.findall('(?<=\${)[^"]*?(?=})', sub)[0].split('.')}, resources_map)
if "." in sub:
cleaned_ref = clean_json(
{
"Fn::GetAtt": re.findall('(?<=\${)[^"]*?(?=})', sub)[
0
].split(".")
},
resources_map,
)
else:
cleaned_ref = clean_json({'Ref': re.findall('(?<=\${)[^"]*?(?=})', sub)[0]}, resources_map)
cleaned_ref = clean_json(
{"Ref": re.findall('(?<=\${)[^"]*?(?=})', sub)[0]},
resources_map,
)
fn_sub_value = fn_sub_value.replace(sub, cleaned_ref)
for literal in literals:
fn_sub_value = fn_sub_value.replace(literal, literal.replace('!', ''))
fn_sub_value = fn_sub_value.replace(
literal, literal.replace("!", "")
)
return fn_sub_value
pass
if 'Fn::ImportValue' in resource_json:
cleaned_val = clean_json(resource_json['Fn::ImportValue'], resources_map)
values = [x.value for x in resources_map.cross_stack_resources.values() if x.name == cleaned_val]
if "Fn::ImportValue" in resource_json:
cleaned_val = clean_json(resource_json["Fn::ImportValue"], resources_map)
values = [
x.value
for x in resources_map.cross_stack_resources.values()
if x.name == cleaned_val
]
if any(values):
return values[0]
else:
raise ExportNotFound(cleaned_val)
if 'Fn::GetAZs' in resource_json:
region = resource_json.get('Fn::GetAZs') or DEFAULT_REGION
if "Fn::GetAZs" in resource_json:
region = resource_json.get("Fn::GetAZs") or DEFAULT_REGION
result = []
# TODO: make this configurable, to reflect the real AWS AZs
for az in ('a', 'b', 'c', 'd'):
result.append('%s%s' % (region, az))
for az in ("a", "b", "c", "d"):
result.append("%s%s" % (region, az))
return result
cleaned_json = {}
@ -246,58 +268,69 @@ def resource_name_property_from_type(resource_type):
def generate_resource_name(resource_type, stack_name, logical_id):
if resource_type in ["AWS::ElasticLoadBalancingV2::TargetGroup",
"AWS::ElasticLoadBalancingV2::LoadBalancer"]:
if resource_type in [
"AWS::ElasticLoadBalancingV2::TargetGroup",
"AWS::ElasticLoadBalancingV2::LoadBalancer",
]:
# Target group names need to be less than 32 characters, so when cloudformation creates a name for you
# it makes sure to stay under that limit
name_prefix = '{0}-{1}'.format(stack_name, logical_id)
name_prefix = "{0}-{1}".format(stack_name, logical_id)
my_random_suffix = random_suffix()
truncated_name_prefix = name_prefix[0:32 - (len(my_random_suffix) + 1)]
truncated_name_prefix = name_prefix[0 : 32 - (len(my_random_suffix) + 1)]
# if the truncated name ends in a dash, we'll end up with a double dash in the final name, which is
# not allowed
if truncated_name_prefix.endswith('-'):
if truncated_name_prefix.endswith("-"):
truncated_name_prefix = truncated_name_prefix[:-1]
return '{0}-{1}'.format(truncated_name_prefix, my_random_suffix)
return "{0}-{1}".format(truncated_name_prefix, my_random_suffix)
else:
return '{0}-{1}-{2}'.format(stack_name, logical_id, random_suffix())
return "{0}-{1}-{2}".format(stack_name, logical_id, random_suffix())
def parse_resource(logical_id, resource_json, resources_map):
resource_type = resource_json['Type']
resource_type = resource_json["Type"]
resource_class = resource_class_from_type(resource_type)
if not resource_class:
warnings.warn(
"Tried to parse {0} but it's not supported by moto's CloudFormation implementation".format(resource_type))
"Tried to parse {0} but it's not supported by moto's CloudFormation implementation".format(
resource_type
)
)
return None
resource_json = clean_json(resource_json, resources_map)
resource_name_property = resource_name_property_from_type(resource_type)
if resource_name_property:
if 'Properties' not in resource_json:
resource_json['Properties'] = dict()
if resource_name_property not in resource_json['Properties']:
resource_json['Properties'][resource_name_property] = generate_resource_name(
resource_type, resources_map.get('AWS::StackName'), logical_id)
resource_name = resource_json['Properties'][resource_name_property]
if "Properties" not in resource_json:
resource_json["Properties"] = dict()
if resource_name_property not in resource_json["Properties"]:
resource_json["Properties"][
resource_name_property
] = generate_resource_name(
resource_type, resources_map.get("AWS::StackName"), logical_id
)
resource_name = resource_json["Properties"][resource_name_property]
else:
resource_name = generate_resource_name(resource_type, resources_map.get('AWS::StackName'), logical_id)
resource_name = generate_resource_name(
resource_type, resources_map.get("AWS::StackName"), logical_id
)
return resource_class, resource_json, resource_name
def parse_and_create_resource(logical_id, resource_json, resources_map, region_name):
condition = resource_json.get('Condition')
condition = resource_json.get("Condition")
if condition and not resources_map.lazy_condition_map[condition]:
# If this has a False condition, don't create the resource
return None
resource_type = resource_json['Type']
resource_type = resource_json["Type"]
resource_tuple = parse_resource(logical_id, resource_json, resources_map)
if not resource_tuple:
return None
resource_class, resource_json, resource_name = resource_tuple
resource = resource_class.create_from_cloudformation_json(
resource_name, resource_json, region_name)
resource_name, resource_json, region_name
)
resource.type = resource_type
resource.logical_resource_id = logical_id
return resource
@ -305,24 +338,27 @@ def parse_and_create_resource(logical_id, resource_json, resources_map, region_n
def parse_and_update_resource(logical_id, resource_json, resources_map, region_name):
resource_class, new_resource_json, new_resource_name = parse_resource(
logical_id, resource_json, resources_map)
logical_id, resource_json, resources_map
)
original_resource = resources_map[logical_id]
new_resource = resource_class.update_from_cloudformation_json(
original_resource=original_resource,
new_resource_name=new_resource_name,
cloudformation_json=new_resource_json,
region_name=region_name
region_name=region_name,
)
new_resource.type = resource_json['Type']
new_resource.type = resource_json["Type"]
new_resource.logical_resource_id = logical_id
return new_resource
def parse_and_delete_resource(logical_id, resource_json, resources_map, region_name):
resource_class, resource_json, resource_name = parse_resource(
logical_id, resource_json, resources_map)
logical_id, resource_json, resources_map
)
resource_class.delete_from_cloudformation_json(
resource_name, resource_json, region_name)
resource_name, resource_json, region_name
)
def parse_condition(condition, resources_map, condition_map):
@ -334,8 +370,8 @@ def parse_condition(condition, resources_map, condition_map):
condition_values = []
for value in list(condition.values())[0]:
# Check if we are referencing another Condition
if 'Condition' in value:
condition_values.append(condition_map[value['Condition']])
if "Condition" in value:
condition_values.append(condition_map[value["Condition"]])
else:
condition_values.append(clean_json(value, resources_map))
@ -344,23 +380,27 @@ def parse_condition(condition, resources_map, condition_map):
elif condition_operator == "Fn::Not":
return not parse_condition(condition_values[0], resources_map, condition_map)
elif condition_operator == "Fn::And":
return all([
parse_condition(condition_value, resources_map, condition_map)
for condition_value
in condition_values])
return all(
[
parse_condition(condition_value, resources_map, condition_map)
for condition_value in condition_values
]
)
elif condition_operator == "Fn::Or":
return any([
parse_condition(condition_value, resources_map, condition_map)
for condition_value
in condition_values])
return any(
[
parse_condition(condition_value, resources_map, condition_map)
for condition_value in condition_values
]
)
def parse_output(output_logical_id, output_json, resources_map):
output_json = clean_json(output_json, resources_map)
output = Output()
output.key = output_logical_id
output.value = clean_json(output_json['Value'], resources_map)
output.description = output_json.get('Description')
output.value = clean_json(output_json["Value"], resources_map)
output.description = output_json.get("Description")
return output
@ -371,9 +411,18 @@ class ResourceMap(collections.Mapping):
each resources is passed this lazy map that it can grab dependencies from.
"""
def __init__(self, stack_id, stack_name, parameters, tags, region_name, template, cross_stack_resources):
def __init__(
self,
stack_id,
stack_name,
parameters,
tags,
region_name,
template,
cross_stack_resources,
):
self._template = template
self._resource_json_map = template['Resources']
self._resource_json_map = template["Resources"]
self._region_name = region_name
self.input_parameters = parameters
self.tags = copy.deepcopy(tags)
@ -401,7 +450,8 @@ class ResourceMap(collections.Mapping):
if not resource_json:
raise KeyError(resource_logical_id)
new_resource = parse_and_create_resource(
resource_logical_id, resource_json, self, self._region_name)
resource_logical_id, resource_json, self, self._region_name
)
if new_resource is not None:
self._parsed_resources[resource_logical_id] = new_resource
return new_resource
@ -417,13 +467,13 @@ class ResourceMap(collections.Mapping):
return self._resource_json_map.keys()
def load_mapping(self):
self._parsed_resources.update(self._template.get('Mappings', {}))
self._parsed_resources.update(self._template.get("Mappings", {}))
def load_parameters(self):
parameter_slots = self._template.get('Parameters', {})
parameter_slots = self._template.get("Parameters", {})
for parameter_name, parameter in parameter_slots.items():
# Set the default values.
self.resolved_parameters[parameter_name] = parameter.get('Default')
self.resolved_parameters[parameter_name] = parameter.get("Default")
# Set any input parameters that were passed
self.no_echo_parameter_keys = []
@ -431,11 +481,11 @@ class ResourceMap(collections.Mapping):
if key in self.resolved_parameters:
parameter_slot = parameter_slots[key]
value_type = parameter_slot.get('Type', 'String')
if value_type == 'CommaDelimitedList' or value_type.startswith("List"):
value = value.split(',')
value_type = parameter_slot.get("Type", "String")
if value_type == "CommaDelimitedList" or value_type.startswith("List"):
value = value.split(",")
if parameter_slot.get('NoEcho'):
if parameter_slot.get("NoEcho"):
self.no_echo_parameter_keys.append(key)
self.resolved_parameters[key] = value
@ -449,11 +499,15 @@ class ResourceMap(collections.Mapping):
self._parsed_resources.update(self.resolved_parameters)
def load_conditions(self):
conditions = self._template.get('Conditions', {})
conditions = self._template.get("Conditions", {})
self.lazy_condition_map = LazyDict()
for condition_name, condition in conditions.items():
self.lazy_condition_map[condition_name] = functools.partial(parse_condition,
condition, self._parsed_resources, self.lazy_condition_map)
self.lazy_condition_map[condition_name] = functools.partial(
parse_condition,
condition,
self._parsed_resources,
self.lazy_condition_map,
)
for condition_name in self.lazy_condition_map:
self.lazy_condition_map[condition_name]
@ -465,13 +519,18 @@ class ResourceMap(collections.Mapping):
# Since this is a lazy map, to create every object we just need to
# iterate through self.
self.tags.update({'aws:cloudformation:stack-name': self.get('AWS::StackName'),
'aws:cloudformation:stack-id': self.get('AWS::StackId')})
self.tags.update(
{
"aws:cloudformation:stack-name": self.get("AWS::StackName"),
"aws:cloudformation:stack-id": self.get("AWS::StackId"),
}
)
for resource in self.resources:
if isinstance(self[resource], ec2_models.TaggedEC2Resource):
self.tags['aws:cloudformation:logical-id'] = resource
self.tags["aws:cloudformation:logical-id"] = resource
ec2_models.ec2_backends[self._region_name].create_tags(
[self[resource].physical_resource_id], self.tags)
[self[resource].physical_resource_id], self.tags
)
def diff(self, template, parameters=None):
if parameters:
@ -481,36 +540,35 @@ class ResourceMap(collections.Mapping):
self.load_conditions()
old_template = self._resource_json_map
new_template = template['Resources']
new_template = template["Resources"]
resource_names_by_action = {
'Add': set(new_template) - set(old_template),
'Modify': set(name for name in new_template if name in old_template and new_template[
name] != old_template[name]),
'Remove': set(old_template) - set(new_template)
}
resources_by_action = {
'Add': {},
'Modify': {},
'Remove': {},
"Add": set(new_template) - set(old_template),
"Modify": set(
name
for name in new_template
if name in old_template and new_template[name] != old_template[name]
),
"Remove": set(old_template) - set(new_template),
}
resources_by_action = {"Add": {}, "Modify": {}, "Remove": {}}
for resource_name in resource_names_by_action['Add']:
resources_by_action['Add'][resource_name] = {
'LogicalResourceId': resource_name,
'ResourceType': new_template[resource_name]['Type']
for resource_name in resource_names_by_action["Add"]:
resources_by_action["Add"][resource_name] = {
"LogicalResourceId": resource_name,
"ResourceType": new_template[resource_name]["Type"],
}
for resource_name in resource_names_by_action['Modify']:
resources_by_action['Modify'][resource_name] = {
'LogicalResourceId': resource_name,
'ResourceType': new_template[resource_name]['Type']
for resource_name in resource_names_by_action["Modify"]:
resources_by_action["Modify"][resource_name] = {
"LogicalResourceId": resource_name,
"ResourceType": new_template[resource_name]["Type"],
}
for resource_name in resource_names_by_action['Remove']:
resources_by_action['Remove'][resource_name] = {
'LogicalResourceId': resource_name,
'ResourceType': old_template[resource_name]['Type']
for resource_name in resource_names_by_action["Remove"]:
resources_by_action["Remove"][resource_name] = {
"LogicalResourceId": resource_name,
"ResourceType": old_template[resource_name]["Type"],
}
return resources_by_action
@ -519,35 +577,38 @@ class ResourceMap(collections.Mapping):
resources_by_action = self.diff(template, parameters)
old_template = self._resource_json_map
new_template = template['Resources']
new_template = template["Resources"]
self._resource_json_map = new_template
for resource_name, resource in resources_by_action['Add'].items():
for resource_name, resource in resources_by_action["Add"].items():
resource_json = new_template[resource_name]
new_resource = parse_and_create_resource(
resource_name, resource_json, self, self._region_name)
resource_name, resource_json, self, self._region_name
)
self._parsed_resources[resource_name] = new_resource
for resource_name, resource in resources_by_action['Remove'].items():
for resource_name, resource in resources_by_action["Remove"].items():
resource_json = old_template[resource_name]
parse_and_delete_resource(
resource_name, resource_json, self, self._region_name)
resource_name, resource_json, self, self._region_name
)
self._parsed_resources.pop(resource_name)
tries = 1
while resources_by_action['Modify'] and tries < 5:
for resource_name, resource in resources_by_action['Modify'].copy().items():
while resources_by_action["Modify"] and tries < 5:
for resource_name, resource in resources_by_action["Modify"].copy().items():
resource_json = new_template[resource_name]
try:
changed_resource = parse_and_update_resource(
resource_name, resource_json, self, self._region_name)
resource_name, resource_json, self, self._region_name
)
except Exception as e:
# skip over dependency violations, and try again in a
# second pass
last_exception = e
else:
self._parsed_resources[resource_name] = changed_resource
del resources_by_action['Modify'][resource_name]
del resources_by_action["Modify"][resource_name]
tries += 1
if tries == 5:
raise last_exception
@ -559,7 +620,7 @@ class ResourceMap(collections.Mapping):
for resource in remaining_resources.copy():
parsed_resource = self._parsed_resources.get(resource)
try:
if parsed_resource and hasattr(parsed_resource, 'delete'):
if parsed_resource and hasattr(parsed_resource, "delete"):
parsed_resource.delete(self._region_name)
except Exception as e:
# skip over dependency violations, and try again in a
@ -573,11 +634,10 @@ class ResourceMap(collections.Mapping):
class OutputMap(collections.Mapping):
def __init__(self, resources, template, stack_id):
self._template = template
self._stack_id = stack_id
self._output_json_map = template.get('Outputs')
self._output_json_map = template.get("Outputs")
# Create the default resources
self._resource_map = resources
@ -591,7 +651,8 @@ class OutputMap(collections.Mapping):
else:
output_json = self._output_json_map.get(output_logical_id)
new_output = parse_output(
output_logical_id, output_json, self._resource_map)
output_logical_id, output_json, self._resource_map
)
self._parsed_outputs[output_logical_id] = new_output
return new_output
@ -610,9 +671,11 @@ class OutputMap(collections.Mapping):
exports = []
if self.outputs:
for key, value in self._output_json_map.items():
if value.get('Export'):
cleaned_name = clean_json(value['Export'].get('Name'), self._resource_map)
cleaned_value = clean_json(value.get('Value'), self._resource_map)
if value.get("Export"):
cleaned_name = clean_json(
value["Export"].get("Name"), self._resource_map
)
cleaned_value = clean_json(value.get("Value"), self._resource_map)
exports.append(Export(self._stack_id, cleaned_name, cleaned_value))
return exports
@ -622,7 +685,6 @@ class OutputMap(collections.Mapping):
class Export(object):
def __init__(self, exporting_stack_id, name, value):
self._exporting_stack_id = exporting_stack_id
self._name = name