catalin/moto
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Run black on moto & test directories.

Browse source
This commit is contained in:
Asher Foa 2019-10-31 08:44:26 -07:00
commit 96e5b1993d
507 changed files with 52541 additions and 47814 deletions
Download patch file Download diff file Expand all files Collapse all files

2
moto/emr/__init__.py
View file

@ -2,6 +2,6 @@ from __future__ import unicode_literals
from .models import emr_backends
from ..core.models import base_decorator, deprecated_base_decorator
emr_backend = emr_backends['us-east-1']
emr_backend = emr_backends["us-east-1"]
mock_emr = base_decorator(emr_backends)
mock_emr_deprecated = deprecated_base_decorator(emr_backends)

268
moto/emr/models.py
View file

@ -11,7 +11,6 @@ from .utils import random_instance_group_id, random_cluster_id, random_step_id
class FakeApplication(BaseModel):
def __init__(self, name, version, args=None, additional_info=None):
self.additional_info = additional_info or {}
self.args = args or []
@ -20,7 +19,6 @@ class FakeApplication(BaseModel):
class FakeBootstrapAction(BaseModel):
def __init__(self, args, name, script_path):
self.args = args or []
self.name = name
@ -28,20 +26,27 @@ class FakeBootstrapAction(BaseModel):
class FakeInstanceGroup(BaseModel):
def __init__(self, instance_count, instance_role, instance_type,
market='ON_DEMAND', name=None, id=None, bid_price=None):
def __init__(
self,
instance_count,
instance_role,
instance_type,
market="ON_DEMAND",
name=None,
id=None,
bid_price=None,
):
self.id = id or random_instance_group_id()
self.bid_price = bid_price
self.market = market
if name is None:
if instance_role == 'MASTER':
name = 'master'
elif instance_role == 'CORE':
name = 'slave'
if instance_role == "MASTER":
name = "master"
elif instance_role == "CORE":
name = "slave"
else:
name = 'Task instance group'
name = "Task instance group"
self.name = name
self.num_instances = instance_count
self.role = instance_role
@ -51,21 +56,22 @@ class FakeInstanceGroup(BaseModel):
self.start_datetime = datetime.now(pytz.utc)
self.ready_datetime = datetime.now(pytz.utc)
self.end_datetime = None
self.state = 'RUNNING'
self.state = "RUNNING"
def set_instance_count(self, instance_count):
self.num_instances = instance_count
class FakeStep(BaseModel):
def __init__(self,
state,
name='',
jar='',
args=None,
properties=None,
action_on_failure='TERMINATE_CLUSTER'):
def __init__(
self,
state,
name="",
jar="",
args=None,
properties=None,
action_on_failure="TERMINATE_CLUSTER",
):
self.id = random_step_id()
self.action_on_failure = action_on_failure
@ -82,23 +88,24 @@ class FakeStep(BaseModel):
class FakeCluster(BaseModel):
def __init__(self,
emr_backend,
name,
log_uri,
job_flow_role,
service_role,
steps,
instance_attrs,
bootstrap_actions=None,
configurations=None,
cluster_id=None,
visible_to_all_users='false',
release_label=None,
requested_ami_version=None,
running_ami_version=None,
custom_ami_id=None):
def __init__(
self,
emr_backend,
name,
log_uri,
job_flow_role,
service_role,
steps,
instance_attrs,
bootstrap_actions=None,
configurations=None,
cluster_id=None,
visible_to_all_users="false",
release_label=None,
requested_ami_version=None,
running_ami_version=None,
custom_ami_id=None,
):
self.id = cluster_id or random_cluster_id()
emr_backend.clusters[self.id] = self
self.emr_backend = emr_backend
@ -106,7 +113,7 @@ class FakeCluster(BaseModel):
self.applications = []
self.bootstrap_actions = []
for bootstrap_action in (bootstrap_actions or []):
for bootstrap_action in bootstrap_actions or []:
self.add_bootstrap_action(bootstrap_action)
self.configurations = configurations or []
@ -125,47 +132,68 @@ class FakeCluster(BaseModel):
self.instance_group_ids = []
self.master_instance_group_id = None
self.core_instance_group_id = None
if 'master_instance_type' in instance_attrs and instance_attrs['master_instance_type']:
if (
"master_instance_type" in instance_attrs
and instance_attrs["master_instance_type"]
):
self.emr_backend.add_instance_groups(
self.id,
[{'instance_count': 1,
'instance_role': 'MASTER',
'instance_type': instance_attrs['master_instance_type'],
'market': 'ON_DEMAND',
'name': 'master'}])
if 'slave_instance_type' in instance_attrs and instance_attrs['slave_instance_type']:
[
{
"instance_count": 1,
"instance_role": "MASTER",
"instance_type": instance_attrs["master_instance_type"],
"market": "ON_DEMAND",
"name": "master",
}
],
)
if (
"slave_instance_type" in instance_attrs
and instance_attrs["slave_instance_type"]
):
self.emr_backend.add_instance_groups(
self.id,
[{'instance_count': instance_attrs['instance_count'] - 1,
'instance_role': 'CORE',
'instance_type': instance_attrs['slave_instance_type'],
'market': 'ON_DEMAND',
'name': 'slave'}])
[
{
"instance_count": instance_attrs["instance_count"] - 1,
"instance_role": "CORE",
"instance_type": instance_attrs["slave_instance_type"],
"market": "ON_DEMAND",
"name": "slave",
}
],
)
self.additional_master_security_groups = instance_attrs.get(
'additional_master_security_groups')
"additional_master_security_groups"
)
self.additional_slave_security_groups = instance_attrs.get(
'additional_slave_security_groups')
self.availability_zone = instance_attrs.get('availability_zone')
self.ec2_key_name = instance_attrs.get('ec2_key_name')
self.ec2_subnet_id = instance_attrs.get('ec2_subnet_id')
self.hadoop_version = instance_attrs.get('hadoop_version')
"additional_slave_security_groups"
)
self.availability_zone = instance_attrs.get("availability_zone")
self.ec2_key_name = instance_attrs.get("ec2_key_name")
self.ec2_subnet_id = instance_attrs.get("ec2_subnet_id")
self.hadoop_version = instance_attrs.get("hadoop_version")
self.keep_job_flow_alive_when_no_steps = instance_attrs.get(
'keep_job_flow_alive_when_no_steps')
"keep_job_flow_alive_when_no_steps"
)
self.master_security_group = instance_attrs.get(
'emr_managed_master_security_group')
"emr_managed_master_security_group"
)
self.service_access_security_group = instance_attrs.get(
'service_access_security_group')
"service_access_security_group"
)
self.slave_security_group = instance_attrs.get(
'emr_managed_slave_security_group')
self.termination_protected = instance_attrs.get(
'termination_protected')
"emr_managed_slave_security_group"
)
self.termination_protected = instance_attrs.get("termination_protected")
self.release_label = release_label
self.requested_ami_version = requested_ami_version
self.running_ami_version = running_ami_version
self.custom_ami_id = custom_ami_id
self.role = job_flow_role or 'EMRJobflowDefault'
self.role = job_flow_role or "EMRJobflowDefault"
self.service_role = service_role
self.creation_datetime = datetime.now(pytz.utc)
@ -194,42 +222,46 @@ class FakeCluster(BaseModel):
return sum(group.num_instances for group in self.instance_groups)
def start_cluster(self):
self.state = 'STARTING'
self.state = "STARTING"
self.start_datetime = datetime.now(pytz.utc)
def run_bootstrap_actions(self):
self.state = 'BOOTSTRAPPING'
self.state = "BOOTSTRAPPING"
self.ready_datetime = datetime.now(pytz.utc)
self.state = 'WAITING'
self.state = "WAITING"
if not self.steps:
if not self.keep_job_flow_alive_when_no_steps:
self.terminate()
def terminate(self):
self.state = 'TERMINATING'
self.state = "TERMINATING"
self.end_datetime = datetime.now(pytz.utc)
self.state = 'TERMINATED'
self.state = "TERMINATED"
def add_applications(self, applications):
self.applications.extend([
FakeApplication(
name=app.get('name', ''),
version=app.get('version', ''),
args=app.get('args', []),
additional_info=app.get('additiona_info', {}))
for app in applications])
self.applications.extend(
[
FakeApplication(
name=app.get("name", ""),
version=app.get("version", ""),
args=app.get("args", []),
additional_info=app.get("additiona_info", {}),
)
for app in applications
]
)
def add_bootstrap_action(self, bootstrap_action):
self.bootstrap_actions.append(FakeBootstrapAction(**bootstrap_action))
def add_instance_group(self, instance_group):
if instance_group.role == 'MASTER':
if instance_group.role == "MASTER":
if self.master_instance_group_id:
raise Exception('Cannot add another master instance group')
raise Exception("Cannot add another master instance group")
self.master_instance_group_id = instance_group.id
if instance_group.role == 'CORE':
if instance_group.role == "CORE":
if self.core_instance_group_id:
raise Exception('Cannot add another core instance group')
raise Exception("Cannot add another core instance group")
self.core_instance_group_id = instance_group.id
self.instance_group_ids.append(instance_group.id)
@ -238,12 +270,12 @@ class FakeCluster(BaseModel):
for step in steps:
if self.steps:
# If we already have other steps, this one is pending
fake = FakeStep(state='PENDING', **step)
fake = FakeStep(state="PENDING", **step)
else:
fake = FakeStep(state='STARTING', **step)
fake = FakeStep(state="STARTING", **step)
self.steps.append(fake)
added_steps.append(fake)
self.state = 'RUNNING'
self.state = "RUNNING"
return added_steps
def add_tags(self, tags):
@ -261,7 +293,6 @@ class FakeCluster(BaseModel):
class ElasticMapReduceBackend(BaseBackend):
def __init__(self, region_name):
super(ElasticMapReduceBackend, self).__init__()
self.region_name = region_name
@ -296,12 +327,17 @@ class ElasticMapReduceBackend(BaseBackend):
cluster = self.get_cluster(cluster_id)
cluster.add_tags(tags)
def describe_job_flows(self, job_flow_ids=None, job_flow_states=None, created_after=None, created_before=None):
def describe_job_flows(
self,
job_flow_ids=None,
job_flow_states=None,
created_after=None,
created_before=None,
):
clusters = self.clusters.values()
within_two_month = datetime.now(pytz.utc) - timedelta(days=60)
clusters = [
c for c in clusters if c.creation_datetime >= within_two_month]
clusters = [c for c in clusters if c.creation_datetime >= within_two_month]
if job_flow_ids:
clusters = [c for c in clusters if c.id in job_flow_ids]
@ -309,12 +345,10 @@ class ElasticMapReduceBackend(BaseBackend):
clusters = [c for c in clusters if c.state in job_flow_states]
if created_after:
created_after = dtparse(created_after)
clusters = [
c for c in clusters if c.creation_datetime > created_after]
clusters = [c for c in clusters if c.creation_datetime > created_after]
if created_before:
created_before = dtparse(created_before)
clusters = [
c for c in clusters if c.creation_datetime < created_before]
clusters = [c for c in clusters if c.creation_datetime < created_before]
# Amazon EMR can return a maximum of 512 job flow descriptions
return sorted(clusters, key=lambda x: x.id)[:512]
@ -328,12 +362,12 @@ class ElasticMapReduceBackend(BaseBackend):
def get_cluster(self, cluster_id):
if cluster_id in self.clusters:
return self.clusters[cluster_id]
raise EmrError('ResourceNotFoundException', '', 'error_json')
raise EmrError("ResourceNotFoundException", "", "error_json")
def get_instance_groups(self, instance_group_ids):
return [
group for group_id, group
in self.instance_groups.items()
group
for group_id, group in self.instance_groups.items()
if group_id in instance_group_ids
]
@ -341,38 +375,43 @@ class ElasticMapReduceBackend(BaseBackend):
max_items = 50
actions = self.clusters[cluster_id].bootstrap_actions
start_idx = 0 if marker is None else int(marker)
marker = None if len(actions) <= start_idx + \
max_items else str(start_idx + max_items)
return actions[start_idx:start_idx + max_items], marker
marker = (
None
if len(actions) <= start_idx + max_items
else str(start_idx + max_items)
)
return actions[start_idx : start_idx + max_items], marker
def list_clusters(self, cluster_states=None, created_after=None,
created_before=None, marker=None):
def list_clusters(
self, cluster_states=None, created_after=None, created_before=None, marker=None
):
max_items = 50
clusters = self.clusters.values()
if cluster_states:
clusters = [c for c in clusters if c.state in cluster_states]
if created_after:
created_after = dtparse(created_after)
clusters = [
c for c in clusters if c.creation_datetime > created_after]
clusters = [c for c in clusters if c.creation_datetime > created_after]
if created_before:
created_before = dtparse(created_before)
clusters = [
c for c in clusters if c.creation_datetime < created_before]
clusters = [c for c in clusters if c.creation_datetime < created_before]
clusters = sorted(clusters, key=lambda x: x.id)
start_idx = 0 if marker is None else int(marker)
marker = None if len(clusters) <= start_idx + \
max_items else str(start_idx + max_items)
return clusters[start_idx:start_idx + max_items], marker
marker = (
None
if len(clusters) <= start_idx + max_items
else str(start_idx + max_items)
)
return clusters[start_idx : start_idx + max_items], marker
def list_instance_groups(self, cluster_id, marker=None):
max_items = 50
groups = sorted(self.clusters[cluster_id].instance_groups,
key=lambda x: x.id)
groups = sorted(self.clusters[cluster_id].instance_groups, key=lambda x: x.id)
start_idx = 0 if marker is None else int(marker)
marker = None if len(groups) <= start_idx + \
max_items else str(start_idx + max_items)
return groups[start_idx:start_idx + max_items], marker
marker = (
None if len(groups) <= start_idx + max_items else str(start_idx + max_items)
)
return groups[start_idx : start_idx + max_items], marker
def list_steps(self, cluster_id, marker=None, step_ids=None, step_states=None):
max_items = 50
@ -382,15 +421,16 @@ class ElasticMapReduceBackend(BaseBackend):
if step_states:
steps = [s for s in steps if s.state in step_states]
start_idx = 0 if marker is None else int(marker)
marker = None if len(steps) <= start_idx + \
max_items else str(start_idx + max_items)
return steps[start_idx:start_idx + max_items], marker
marker = (
None if len(steps) <= start_idx + max_items else str(start_idx + max_items)
)
return steps[start_idx : start_idx + max_items], marker
def modify_instance_groups(self, instance_groups):
result_groups = []
for instance_group in instance_groups:
group = self.instance_groups[instance_group['instance_group_id']]
group.set_instance_count(int(instance_group['instance_count']))
group = self.instance_groups[instance_group["instance_group_id"]]
group.set_instance_count(int(instance_group["instance_count"]))
return result_groups
def remove_tags(self, cluster_id, tag_keys):

289
moto/emr/responses.py
View file

@ -20,20 +20,27 @@ def generate_boto3_response(operation):
determined to be from boto3. Pass the API action as a parameter.
"""
def _boto3_request(method):
@wraps(method)
def f(self, *args, **kwargs):
rendered = method(self, *args, **kwargs)
if 'json' in self.headers.get('Content-Type', []):
if "json" in self.headers.get("Content-Type", []):
self.response_headers.update(
{'x-amzn-requestid': '2690d7eb-ed86-11dd-9877-6fad448a8419',
'date': datetime.now(pytz.utc).strftime('%a, %d %b %Y %H:%M:%S %Z'),
'content-type': 'application/x-amz-json-1.1'})
resp = xml_to_json_response(
self.aws_service_spec, operation, rendered)
return '' if resp is None else json.dumps(resp)
{
"x-amzn-requestid": "2690d7eb-ed86-11dd-9877-6fad448a8419",
"date": datetime.now(pytz.utc).strftime(
"%a, %d %b %Y %H:%M:%S %Z"
),
"content-type": "application/x-amz-json-1.1",
}
)
resp = xml_to_json_response(self.aws_service_spec, operation, rendered)
return "" if resp is None else json.dumps(resp)
return rendered
return f
return _boto3_request
@ -41,10 +48,12 @@ class ElasticMapReduceResponse(BaseResponse):
# EMR end points are inconsistent in the placement of region name
# in the URL, so parsing it out needs to be handled differently
region_regex = [re.compile(r'elasticmapreduce\.(.+?)\.amazonaws\.com'),
re.compile(r'(.+?)\.elasticmapreduce\.amazonaws\.com')]
region_regex = [
re.compile(r"elasticmapreduce\.(.+?)\.amazonaws\.com"),
re.compile(r"(.+?)\.elasticmapreduce\.amazonaws\.com"),
]
aws_service_spec = AWSServiceSpec('data/emr/2009-03-31/service-2.json')
aws_service_spec = AWSServiceSpec("data/emr/2009-03-31/service-2.json")
def get_region_from_url(self, request, full_url):
parsed = urlparse(full_url)
@ -58,28 +67,28 @@ class ElasticMapReduceResponse(BaseResponse):
def backend(self):
return emr_backends[self.region]
@generate_boto3_response('AddInstanceGroups')
@generate_boto3_response("AddInstanceGroups")
def add_instance_groups(self):
jobflow_id = self._get_param('JobFlowId')
instance_groups = self._get_list_prefix('InstanceGroups.member')
jobflow_id = self._get_param("JobFlowId")
instance_groups = self._get_list_prefix("InstanceGroups.member")
for item in instance_groups:
item['instance_count'] = int(item['instance_count'])
instance_groups = self.backend.add_instance_groups(
jobflow_id, instance_groups)
item["instance_count"] = int(item["instance_count"])
instance_groups = self.backend.add_instance_groups(jobflow_id, instance_groups)
template = self.response_template(ADD_INSTANCE_GROUPS_TEMPLATE)
return template.render(instance_groups=instance_groups)
@generate_boto3_response('AddJobFlowSteps')
@generate_boto3_response("AddJobFlowSteps")
def add_job_flow_steps(self):
job_flow_id = self._get_param('JobFlowId')
job_flow_id = self._get_param("JobFlowId")
steps = self.backend.add_job_flow_steps(
job_flow_id, steps_from_query_string(self._get_list_prefix('Steps.member')))
job_flow_id, steps_from_query_string(self._get_list_prefix("Steps.member"))
)
template = self.response_template(ADD_JOB_FLOW_STEPS_TEMPLATE)
return template.render(steps=steps)
@generate_boto3_response('AddTags')
@generate_boto3_response("AddTags")
def add_tags(self):
cluster_id = self._get_param('ResourceId')
cluster_id = self._get_param("ResourceId")
tags = tags_from_query_string(self.querystring)
self.backend.add_tags(cluster_id, tags)
template = self.response_template(ADD_TAGS_TEMPLATE)
@ -94,235 +103,257 @@ class ElasticMapReduceResponse(BaseResponse):
def delete_security_configuration(self):
raise NotImplementedError
@generate_boto3_response('DescribeCluster')
@generate_boto3_response("DescribeCluster")
def describe_cluster(self):
cluster_id = self._get_param('ClusterId')
cluster_id = self._get_param("ClusterId")
cluster = self.backend.get_cluster(cluster_id)
template = self.response_template(DESCRIBE_CLUSTER_TEMPLATE)
return template.render(cluster=cluster)
@generate_boto3_response('DescribeJobFlows')
@generate_boto3_response("DescribeJobFlows")
def describe_job_flows(self):
created_after = self._get_param('CreatedAfter')
created_before = self._get_param('CreatedBefore')
created_after = self._get_param("CreatedAfter")
created_before = self._get_param("CreatedBefore")
job_flow_ids = self._get_multi_param("JobFlowIds.member")
job_flow_states = self._get_multi_param('JobFlowStates.member')
job_flow_states = self._get_multi_param("JobFlowStates.member")
clusters = self.backend.describe_job_flows(
job_flow_ids, job_flow_states, created_after, created_before)
job_flow_ids, job_flow_states, created_after, created_before
)
template = self.response_template(DESCRIBE_JOB_FLOWS_TEMPLATE)
return template.render(clusters=clusters)
def describe_security_configuration(self):
raise NotImplementedError
@generate_boto3_response('DescribeStep')
@generate_boto3_response("DescribeStep")
def describe_step(self):
cluster_id = self._get_param('ClusterId')
step_id = self._get_param('StepId')
cluster_id = self._get_param("ClusterId")
step_id = self._get_param("StepId")
step = self.backend.describe_step(cluster_id, step_id)
template = self.response_template(DESCRIBE_STEP_TEMPLATE)
return template.render(step=step)
@generate_boto3_response('ListBootstrapActions')
@generate_boto3_response("ListBootstrapActions")
def list_bootstrap_actions(self):
cluster_id = self._get_param('ClusterId')
marker = self._get_param('Marker')
cluster_id = self._get_param("ClusterId")
marker = self._get_param("Marker")
bootstrap_actions, marker = self.backend.list_bootstrap_actions(
cluster_id, marker)
cluster_id, marker
)
template = self.response_template(LIST_BOOTSTRAP_ACTIONS_TEMPLATE)
return template.render(bootstrap_actions=bootstrap_actions, marker=marker)
@generate_boto3_response('ListClusters')
@generate_boto3_response("ListClusters")
def list_clusters(self):
cluster_states = self._get_multi_param('ClusterStates.member')
created_after = self._get_param('CreatedAfter')
created_before = self._get_param('CreatedBefore')
marker = self._get_param('Marker')
cluster_states = self._get_multi_param("ClusterStates.member")
created_after = self._get_param("CreatedAfter")
created_before = self._get_param("CreatedBefore")
marker = self._get_param("Marker")
clusters, marker = self.backend.list_clusters(
cluster_states, created_after, created_before, marker)
cluster_states, created_after, created_before, marker
)
template = self.response_template(LIST_CLUSTERS_TEMPLATE)
return template.render(clusters=clusters, marker=marker)
@generate_boto3_response('ListInstanceGroups')
@generate_boto3_response("ListInstanceGroups")
def list_instance_groups(self):
cluster_id = self._get_param('ClusterId')
marker = self._get_param('Marker')
cluster_id = self._get_param("ClusterId")
marker = self._get_param("Marker")
instance_groups, marker = self.backend.list_instance_groups(
cluster_id, marker=marker)
cluster_id, marker=marker
)
template = self.response_template(LIST_INSTANCE_GROUPS_TEMPLATE)
return template.render(instance_groups=instance_groups, marker=marker)
def list_instances(self):
raise NotImplementedError
@generate_boto3_response('ListSteps')
@generate_boto3_response("ListSteps")
def list_steps(self):
cluster_id = self._get_param('ClusterId')
marker = self._get_param('Marker')
step_ids = self._get_multi_param('StepIds.member')
step_states = self._get_multi_param('StepStates.member')
cluster_id = self._get_param("ClusterId")
marker = self._get_param("Marker")
step_ids = self._get_multi_param("StepIds.member")
step_states = self._get_multi_param("StepStates.member")
steps, marker = self.backend.list_steps(
cluster_id, marker=marker, step_ids=step_ids, step_states=step_states)
cluster_id, marker=marker, step_ids=step_ids, step_states=step_states
)
template = self.response_template(LIST_STEPS_TEMPLATE)
return template.render(steps=steps, marker=marker)
@generate_boto3_response('ModifyInstanceGroups')
@generate_boto3_response("ModifyInstanceGroups")
def modify_instance_groups(self):
instance_groups = self._get_list_prefix('InstanceGroups.member')
instance_groups = self._get_list_prefix("InstanceGroups.member")
for item in instance_groups:
item['instance_count'] = int(item['instance_count'])
item["instance_count"] = int(item["instance_count"])
instance_groups = self.backend.modify_instance_groups(instance_groups)
template = self.response_template(MODIFY_INSTANCE_GROUPS_TEMPLATE)
return template.render(instance_groups=instance_groups)
@generate_boto3_response('RemoveTags')
@generate_boto3_response("RemoveTags")
def remove_tags(self):
cluster_id = self._get_param('ResourceId')
tag_keys = self._get_multi_param('TagKeys.member')
cluster_id = self._get_param("ResourceId")
tag_keys = self._get_multi_param("TagKeys.member")
self.backend.remove_tags(cluster_id, tag_keys)
template = self.response_template(REMOVE_TAGS_TEMPLATE)
return template.render()
@generate_boto3_response('RunJobFlow')
@generate_boto3_response("RunJobFlow")
def run_job_flow(self):
instance_attrs = dict(
master_instance_type=self._get_param(
'Instances.MasterInstanceType'),
slave_instance_type=self._get_param('Instances.SlaveInstanceType'),
instance_count=self._get_int_param('Instances.InstanceCount', 1),
ec2_key_name=self._get_param('Instances.Ec2KeyName'),
ec2_subnet_id=self._get_param('Instances.Ec2SubnetId'),
hadoop_version=self._get_param('Instances.HadoopVersion'),
master_instance_type=self._get_param("Instances.MasterInstanceType"),
slave_instance_type=self._get_param("Instances.SlaveInstanceType"),
instance_count=self._get_int_param("Instances.InstanceCount", 1),
ec2_key_name=self._get_param("Instances.Ec2KeyName"),
ec2_subnet_id=self._get_param("Instances.Ec2SubnetId"),
hadoop_version=self._get_param("Instances.HadoopVersion"),
availability_zone=self._get_param(
'Instances.Placement.AvailabilityZone', self.backend.region_name + 'a'),
"Instances.Placement.AvailabilityZone", self.backend.region_name + "a"
),
keep_job_flow_alive_when_no_steps=self._get_bool_param(
'Instances.KeepJobFlowAliveWhenNoSteps', False),
"Instances.KeepJobFlowAliveWhenNoSteps", False
),
termination_protected=self._get_bool_param(
'Instances.TerminationProtected', False),
"Instances.TerminationProtected", False
),
emr_managed_master_security_group=self._get_param(
'Instances.EmrManagedMasterSecurityGroup'),
"Instances.EmrManagedMasterSecurityGroup"
),
emr_managed_slave_security_group=self._get_param(
'Instances.EmrManagedSlaveSecurityGroup'),
"Instances.EmrManagedSlaveSecurityGroup"
),
service_access_security_group=self._get_param(
'Instances.ServiceAccessSecurityGroup'),
"Instances.ServiceAccessSecurityGroup"
),
additional_master_security_groups=self._get_multi_param(
'Instances.AdditionalMasterSecurityGroups.member.'),
additional_slave_security_groups=self._get_multi_param('Instances.AdditionalSlaveSecurityGroups.member.'))
"Instances.AdditionalMasterSecurityGroups.member."
),
additional_slave_security_groups=self._get_multi_param(
"Instances.AdditionalSlaveSecurityGroups.member."
),
)
kwargs = dict(
name=self._get_param('Name'),
log_uri=self._get_param('LogUri'),
job_flow_role=self._get_param('JobFlowRole'),
service_role=self._get_param('ServiceRole'),
steps=steps_from_query_string(
self._get_list_prefix('Steps.member')),
visible_to_all_users=self._get_bool_param(
'VisibleToAllUsers', False),
name=self._get_param("Name"),
log_uri=self._get_param("LogUri"),
job_flow_role=self._get_param("JobFlowRole"),
service_role=self._get_param("ServiceRole"),
steps=steps_from_query_string(self._get_list_prefix("Steps.member")),
visible_to_all_users=self._get_bool_param("VisibleToAllUsers", False),
instance_attrs=instance_attrs,
)
bootstrap_actions = self._get_list_prefix('BootstrapActions.member')
bootstrap_actions = self._get_list_prefix("BootstrapActions.member")
if bootstrap_actions:
for ba in bootstrap_actions:
args = []
idx = 1
keyfmt = 'script_bootstrap_action._args.member.{0}'
keyfmt = "script_bootstrap_action._args.member.{0}"
key = keyfmt.format(idx)
while key in ba:
args.append(ba.pop(key))
idx += 1
key = keyfmt.format(idx)
ba['args'] = args
ba['script_path'] = ba.pop('script_bootstrap_action._path')
kwargs['bootstrap_actions'] = bootstrap_actions
ba["args"] = args
ba["script_path"] = ba.pop("script_bootstrap_action._path")
kwargs["bootstrap_actions"] = bootstrap_actions
configurations = self._get_list_prefix('Configurations.member')
configurations = self._get_list_prefix("Configurations.member")
if configurations:
for idx, config in enumerate(configurations, 1):
for key in list(config.keys()):
if key.startswith('properties.'):
if key.startswith("properties."):
config.pop(key)
config['properties'] = {}
config["properties"] = {}
map_items = self._get_map_prefix(
'Configurations.member.{0}.Properties.entry'.format(idx))
config['properties'] = map_items
"Configurations.member.{0}.Properties.entry".format(idx)
)
config["properties"] = map_items
kwargs['configurations'] = configurations
kwargs["configurations"] = configurations
release_label = self._get_param('ReleaseLabel')
ami_version = self._get_param('AmiVersion')
release_label = self._get_param("ReleaseLabel")
ami_version = self._get_param("AmiVersion")
if release_label:
kwargs['release_label'] = release_label
kwargs["release_label"] = release_label
if ami_version:
message = (
'Only one AMI version and release label may be specified. '
'Provided AMI: {0}, release label: {1}.').format(
ami_version, release_label)
raise EmrError(error_type="ValidationException",
message=message, template='error_json')
"Only one AMI version and release label may be specified. "
"Provided AMI: {0}, release label: {1}."
).format(ami_version, release_label)
raise EmrError(
error_type="ValidationException",
message=message,
template="error_json",
)
else:
if ami_version:
kwargs['requested_ami_version'] = ami_version
kwargs['running_ami_version'] = ami_version
kwargs["requested_ami_version"] = ami_version
kwargs["running_ami_version"] = ami_version
else:
kwargs['running_ami_version'] = '1.0.0'
kwargs["running_ami_version"] = "1.0.0"
custom_ami_id = self._get_param('CustomAmiId')
custom_ami_id = self._get_param("CustomAmiId")
if custom_ami_id:
kwargs['custom_ami_id'] = custom_ami_id
if release_label and release_label < 'emr-5.7.0':
message = 'Custom AMI is not allowed'
raise EmrError(error_type='ValidationException',
message=message, template='error_json')
kwargs["custom_ami_id"] = custom_ami_id
if release_label and release_label < "emr-5.7.0":
message = "Custom AMI is not allowed"
raise EmrError(
error_type="ValidationException",
message=message,
template="error_json",
)
elif ami_version:
message = 'Custom AMI is not supported in this version of EMR'
raise EmrError(error_type='ValidationException',
message=message, template='error_json')
message = "Custom AMI is not supported in this version of EMR"
raise EmrError(
error_type="ValidationException",
message=message,
template="error_json",
)
cluster = self.backend.run_job_flow(**kwargs)
applications = self._get_list_prefix('Applications.member')
applications = self._get_list_prefix("Applications.member")
if applications:
self.backend.add_applications(cluster.id, applications)
else:
self.backend.add_applications(
cluster.id, [{'Name': 'Hadoop', 'Version': '0.18'}])
cluster.id, [{"Name": "Hadoop", "Version": "0.18"}]
)
instance_groups = self._get_list_prefix(
'Instances.InstanceGroups.member')
instance_groups = self._get_list_prefix("Instances.InstanceGroups.member")
if instance_groups:
for ig in instance_groups:
ig['instance_count'] = int(ig['instance_count'])
ig["instance_count"] = int(ig["instance_count"])
self.backend.add_instance_groups(cluster.id, instance_groups)
tags = self._get_list_prefix('Tags.member')
tags = self._get_list_prefix("Tags.member")
if tags:
self.backend.add_tags(
cluster.id, dict((d['key'], d['value']) for d in tags))
cluster.id, dict((d["key"], d["value"]) for d in tags)
)
template = self.response_template(RUN_JOB_FLOW_TEMPLATE)
return template.render(cluster=cluster)
@generate_boto3_response('SetTerminationProtection')
@generate_boto3_response("SetTerminationProtection")
def set_termination_protection(self):
termination_protection = self._get_param('TerminationProtected')
job_ids = self._get_multi_param('JobFlowIds.member')
self.backend.set_termination_protection(
job_ids, termination_protection)
termination_protection = self._get_param("TerminationProtected")
job_ids = self._get_multi_param("JobFlowIds.member")
self.backend.set_termination_protection(job_ids, termination_protection)
template = self.response_template(SET_TERMINATION_PROTECTION_TEMPLATE)
return template.render()
@generate_boto3_response('SetVisibleToAllUsers')
@generate_boto3_response("SetVisibleToAllUsers")
def set_visible_to_all_users(self):
visible_to_all_users = self._get_param('VisibleToAllUsers')
job_ids = self._get_multi_param('JobFlowIds.member')
visible_to_all_users = self._get_param("VisibleToAllUsers")
job_ids = self._get_multi_param("JobFlowIds.member")
self.backend.set_visible_to_all_users(job_ids, visible_to_all_users)
template = self.response_template(SET_VISIBLE_TO_ALL_USERS_TEMPLATE)
return template.render()
@generate_boto3_response('TerminateJobFlows')
@generate_boto3_response("TerminateJobFlows")
def terminate_job_flows(self):
job_ids = self._get_multi_param('JobFlowIds.member.')
job_ids = self._get_multi_param("JobFlowIds.member.")
self.backend.terminate_job_flows(job_ids)
template = self.response_template(TERMINATE_JOB_FLOWS_TEMPLATE)
return template.render()

4
moto/emr/urls.py
View file

@ -6,6 +6,4 @@ url_bases = [
"https?://elasticmapreduce.(.+).amazonaws.com",
]
url_paths = {
'{0}/$': ElasticMapReduceResponse.dispatch,
}
url_paths = {"{0}/$": ElasticMapReduceResponse.dispatch}

28
moto/emr/utils.py
View file

@ -7,24 +7,24 @@ import six
def random_id(size=13):
chars = list(range(10)) + list(string.ascii_uppercase)
return ''.join(six.text_type(random.choice(chars)) for x in range(size))
return "".join(six.text_type(random.choice(chars)) for x in range(size))
def random_cluster_id(size=13):
return 'j-{0}'.format(random_id())
return "j-{0}".format(random_id())
def random_step_id(size=13):
return 's-{0}'.format(random_id())
return "s-{0}".format(random_id())
def random_instance_group_id(size=13):
return 'i-{0}'.format(random_id())
return "i-{0}".format(random_id())
def tags_from_query_string(querystring_dict):
prefix = 'Tags'
suffix = 'Key'
prefix = "Tags"
suffix = "Key"
response_values = {}
for key, value in querystring_dict.items():
if key.startswith(prefix) and key.endswith(suffix):
@ -32,8 +32,7 @@ def tags_from_query_string(querystring_dict):
tag_key = querystring_dict.get("Tags.{0}.Key".format(tag_index))[0]
tag_value_key = "Tags.{0}.Value".format(tag_index)
if tag_value_key in querystring_dict:
response_values[tag_key] = querystring_dict.get(tag_value_key)[
0]
response_values[tag_key] = querystring_dict.get(tag_value_key)[0]
else:
response_values[tag_key] = None
return response_values
@ -42,14 +41,15 @@ def tags_from_query_string(querystring_dict):
def steps_from_query_string(querystring_dict):
steps = []
for step in querystring_dict:
step['jar'] = step.pop('hadoop_jar_step._jar')
step['properties'] = dict((o['Key'], o['Value'])
for o in step.get('properties', []))
step['args'] = []
step["jar"] = step.pop("hadoop_jar_step._jar")
step["properties"] = dict(
(o["Key"], o["Value"]) for o in step.get("properties", [])
)
step["args"] = []
idx = 1
keyfmt = 'hadoop_jar_step._args.member.{0}'
keyfmt = "hadoop_jar_step._args.member.{0}"
while keyfmt.format(idx) in step:
step['args'].append(step.pop(keyfmt.format(idx)))
step["args"].append(step.pop(keyfmt.format(idx)))
idx += 1
steps.append(step)
return steps