catalin/moto
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Lints.

Browse source
This commit is contained in:
Steve Pulec 2017-02-23 21:37:43 -05:00
commit f37bad0e00
260 changed files with 6363 additions and 3766 deletions
Download patch file Download diff file Expand all files Collapse all files

2
moto/emr/__init__.py
View file

@ -1,6 +1,6 @@
from __future__ import unicode_literals
from .models import emr_backends
from ..core.models import MockAWS, base_decorator, HttprettyMockAWS, deprecated_base_decorator
from ..core.models import base_decorator, deprecated_base_decorator
emr_backend = emr_backends['us-east-1']
mock_emr = base_decorator(emr_backends)

53
moto/emr/models.py
View file

@ -11,6 +11,7 @@ from .utils import random_instance_group_id, random_cluster_id, random_step_id
class FakeApplication(object):
def __init__(self, name, version, args=None, additional_info=None):
self.additional_info = additional_info or {}
self.args = args or []
@ -19,6 +20,7 @@ class FakeApplication(object):
class FakeBootstrapAction(object):
def __init__(self, args, name, script_path):
self.args = args or []
self.name = name
@ -26,6 +28,7 @@ class FakeBootstrapAction(object):
class FakeInstanceGroup(object):
def __init__(self, instance_count, instance_role, instance_type,
market='ON_DEMAND', name=None, id=None, bid_price=None):
self.id = id or random_instance_group_id()
@ -55,6 +58,7 @@ class FakeInstanceGroup(object):
class FakeStep(object):
def __init__(self,
state,
name='',
@ -78,6 +82,7 @@ class FakeStep(object):
class FakeCluster(object):
def __init__(self,
emr_backend,
name,
@ -135,17 +140,24 @@ class FakeCluster(object):
'instance_type': instance_attrs['slave_instance_type'],
'market': 'ON_DEMAND',
'name': 'slave'}])
self.additional_master_security_groups = instance_attrs.get('additional_master_security_groups')
self.additional_slave_security_groups = instance_attrs.get('additional_slave_security_groups')
self.additional_master_security_groups = instance_attrs.get(
'additional_master_security_groups')
self.additional_slave_security_groups = instance_attrs.get(
'additional_slave_security_groups')
self.availability_zone = instance_attrs.get('availability_zone')
self.ec2_key_name = instance_attrs.get('ec2_key_name')
self.ec2_subnet_id = instance_attrs.get('ec2_subnet_id')
self.hadoop_version = instance_attrs.get('hadoop_version')
self.keep_job_flow_alive_when_no_steps = instance_attrs.get('keep_job_flow_alive_when_no_steps')
self.master_security_group = instance_attrs.get('emr_managed_master_security_group')
self.service_access_security_group = instance_attrs.get('service_access_security_group')
self.slave_security_group = instance_attrs.get('emr_managed_slave_security_group')
self.termination_protected = instance_attrs.get('termination_protected')
self.keep_job_flow_alive_when_no_steps = instance_attrs.get(
'keep_job_flow_alive_when_no_steps')
self.master_security_group = instance_attrs.get(
'emr_managed_master_security_group')
self.service_access_security_group = instance_attrs.get(
'service_access_security_group')
self.slave_security_group = instance_attrs.get(
'emr_managed_slave_security_group')
self.termination_protected = instance_attrs.get(
'termination_protected')
self.release_label = release_label
self.requested_ami_version = requested_ami_version
@ -286,7 +298,8 @@ class ElasticMapReduceBackend(BaseBackend):
clusters = self.clusters.values()
within_two_month = datetime.now(pytz.utc) - timedelta(days=60)
clusters = [c for c in clusters if c.creation_datetime >= within_two_month]
clusters = [
c for c in clusters if c.creation_datetime >= within_two_month]
if job_flow_ids:
clusters = [c for c in clusters if c.id in job_flow_ids]
@ -294,10 +307,12 @@ class ElasticMapReduceBackend(BaseBackend):
clusters = [c for c in clusters if c.state in job_flow_states]
if created_after:
created_after = dtparse(created_after)
clusters = [c for c in clusters if c.creation_datetime > created_after]
clusters = [
c for c in clusters if c.creation_datetime > created_after]
if created_before:
created_before = dtparse(created_before)
clusters = [c for c in clusters if c.creation_datetime < created_before]
clusters = [
c for c in clusters if c.creation_datetime < created_before]
# Amazon EMR can return a maximum of 512 job flow descriptions
return sorted(clusters, key=lambda x: x.id)[:512]
@ -322,7 +337,8 @@ class ElasticMapReduceBackend(BaseBackend):
max_items = 50
actions = self.clusters[cluster_id].bootstrap_actions
start_idx = 0 if marker is None else int(marker)
marker = None if len(actions) <= start_idx + max_items else str(start_idx + max_items)
marker = None if len(actions) <= start_idx + \
max_items else str(start_idx + max_items)
return actions[start_idx:start_idx + max_items], marker
def list_clusters(self, cluster_states=None, created_after=None,
@ -333,13 +349,16 @@ class ElasticMapReduceBackend(BaseBackend):
clusters = [c for c in clusters if c.state in cluster_states]
if created_after:
created_after = dtparse(created_after)
clusters = [c for c in clusters if c.creation_datetime > created_after]
clusters = [
c for c in clusters if c.creation_datetime > created_after]
if created_before:
created_before = dtparse(created_before)
clusters = [c for c in clusters if c.creation_datetime < created_before]
clusters = [
c for c in clusters if c.creation_datetime < created_before]
clusters = sorted(clusters, key=lambda x: x.id)
start_idx = 0 if marker is None else int(marker)
marker = None if len(clusters) <= start_idx + max_items else str(start_idx + max_items)
marker = None if len(clusters) <= start_idx + \
max_items else str(start_idx + max_items)
return clusters[start_idx:start_idx + max_items], marker
def list_instance_groups(self, cluster_id, marker=None):
@ -347,7 +366,8 @@ class ElasticMapReduceBackend(BaseBackend):
groups = sorted(self.clusters[cluster_id].instance_groups,
key=lambda x: x.id)
start_idx = 0 if marker is None else int(marker)
marker = None if len(groups) <= start_idx + max_items else str(start_idx + max_items)
marker = None if len(groups) <= start_idx + \
max_items else str(start_idx + max_items)
return groups[start_idx:start_idx + max_items], marker
def list_steps(self, cluster_id, marker=None, step_ids=None, step_states=None):
@ -358,7 +378,8 @@ class ElasticMapReduceBackend(BaseBackend):
if step_states:
steps = [s for s in steps if s.state in step_states]
start_idx = 0 if marker is None else int(marker)
marker = None if len(steps) <= start_idx + max_items else str(start_idx + max_items)
marker = None if len(steps) <= start_idx + \
max_items else str(start_idx + max_items)
return steps[start_idx:start_idx + max_items], marker
def modify_instance_groups(self, instance_groups):

66
moto/emr/responses.py
View file

@ -29,7 +29,8 @@ def generate_boto3_response(operation):
{'x-amzn-requestid': '2690d7eb-ed86-11dd-9877-6fad448a8419',
'date': datetime.now(pytz.utc).strftime('%a, %d %b %Y %H:%M:%S %Z'),
'content-type': 'application/x-amz-json-1.1'})
resp = xml_to_json_response(self.aws_service_spec, operation, rendered)
resp = xml_to_json_response(
self.aws_service_spec, operation, rendered)
return '' if resp is None else json.dumps(resp)
return rendered
return f
@ -63,14 +64,16 @@ class ElasticMapReduceResponse(BaseResponse):
instance_groups = self._get_list_prefix('InstanceGroups.member')
for item in instance_groups:
item['instance_count'] = int(item['instance_count'])
instance_groups = self.backend.add_instance_groups(jobflow_id, instance_groups)
instance_groups = self.backend.add_instance_groups(
jobflow_id, instance_groups)
template = self.response_template(ADD_INSTANCE_GROUPS_TEMPLATE)
return template.render(instance_groups=instance_groups)
@generate_boto3_response('AddJobFlowSteps')
def add_job_flow_steps(self):
job_flow_id = self._get_param('JobFlowId')
steps = self.backend.add_job_flow_steps(job_flow_id, steps_from_query_string(self._get_list_prefix('Steps.member')))
steps = self.backend.add_job_flow_steps(
job_flow_id, steps_from_query_string(self._get_list_prefix('Steps.member')))
template = self.response_template(ADD_JOB_FLOW_STEPS_TEMPLATE)
return template.render(steps=steps)
@ -104,7 +107,8 @@ class ElasticMapReduceResponse(BaseResponse):
created_before = self._get_param('CreatedBefore')
job_flow_ids = self._get_multi_param("JobFlowIds.member")
job_flow_states = self._get_multi_param('JobFlowStates.member')
clusters = self.backend.describe_job_flows(job_flow_ids, job_flow_states, created_after, created_before)
clusters = self.backend.describe_job_flows(
job_flow_ids, job_flow_states, created_after, created_before)
template = self.response_template(DESCRIBE_JOB_FLOWS_TEMPLATE)
return template.render(clusters=clusters)
@ -123,7 +127,8 @@ class ElasticMapReduceResponse(BaseResponse):
def list_bootstrap_actions(self):
cluster_id = self._get_param('ClusterId')
marker = self._get_param('Marker')
bootstrap_actions, marker = self.backend.list_bootstrap_actions(cluster_id, marker)
bootstrap_actions, marker = self.backend.list_bootstrap_actions(
cluster_id, marker)
template = self.response_template(LIST_BOOTSTRAP_ACTIONS_TEMPLATE)
return template.render(bootstrap_actions=bootstrap_actions, marker=marker)
@ -133,7 +138,8 @@ class ElasticMapReduceResponse(BaseResponse):
created_after = self._get_param('CreatedAfter')
created_before = self._get_param('CreatedBefore')
marker = self._get_param('Marker')
clusters, marker = self.backend.list_clusters(cluster_states, created_after, created_before, marker)
clusters, marker = self.backend.list_clusters(
cluster_states, created_after, created_before, marker)
template = self.response_template(LIST_CLUSTERS_TEMPLATE)
return template.render(clusters=clusters, marker=marker)
@ -141,7 +147,8 @@ class ElasticMapReduceResponse(BaseResponse):
def list_instance_groups(self):
cluster_id = self._get_param('ClusterId')
marker = self._get_param('Marker')
instance_groups, marker = self.backend.list_instance_groups(cluster_id, marker=marker)
instance_groups, marker = self.backend.list_instance_groups(
cluster_id, marker=marker)
template = self.response_template(LIST_INSTANCE_GROUPS_TEMPLATE)
return template.render(instance_groups=instance_groups, marker=marker)
@ -154,7 +161,8 @@ class ElasticMapReduceResponse(BaseResponse):
marker = self._get_param('Marker')
step_ids = self._get_multi_param('StepIds.member')
step_states = self._get_multi_param('StepStates.member')
steps, marker = self.backend.list_steps(cluster_id, marker=marker, step_ids=step_ids, step_states=step_states)
steps, marker = self.backend.list_steps(
cluster_id, marker=marker, step_ids=step_ids, step_states=step_states)
template = self.response_template(LIST_STEPS_TEMPLATE)
return template.render(steps=steps, marker=marker)
@ -178,19 +186,27 @@ class ElasticMapReduceResponse(BaseResponse):
@generate_boto3_response('RunJobFlow')
def run_job_flow(self):
instance_attrs = dict(
master_instance_type=self._get_param('Instances.MasterInstanceType'),
master_instance_type=self._get_param(
'Instances.MasterInstanceType'),
slave_instance_type=self._get_param('Instances.SlaveInstanceType'),
instance_count=self._get_int_param('Instances.InstanceCount', 1),
ec2_key_name=self._get_param('Instances.Ec2KeyName'),
ec2_subnet_id=self._get_param('Instances.Ec2SubnetId'),
hadoop_version=self._get_param('Instances.HadoopVersion'),
availability_zone=self._get_param('Instances.Placement.AvailabilityZone', self.backend.region_name + 'a'),
keep_job_flow_alive_when_no_steps=self._get_bool_param('Instances.KeepJobFlowAliveWhenNoSteps', False),
termination_protected=self._get_bool_param('Instances.TerminationProtected', False),
emr_managed_master_security_group=self._get_param('Instances.EmrManagedMasterSecurityGroup'),
emr_managed_slave_security_group=self._get_param('Instances.EmrManagedSlaveSecurityGroup'),
service_access_security_group=self._get_param('Instances.ServiceAccessSecurityGroup'),
additional_master_security_groups=self._get_multi_param('Instances.AdditionalMasterSecurityGroups.member.'),
availability_zone=self._get_param(
'Instances.Placement.AvailabilityZone', self.backend.region_name + 'a'),
keep_job_flow_alive_when_no_steps=self._get_bool_param(
'Instances.KeepJobFlowAliveWhenNoSteps', False),
termination_protected=self._get_bool_param(
'Instances.TerminationProtected', False),
emr_managed_master_security_group=self._get_param(
'Instances.EmrManagedMasterSecurityGroup'),
emr_managed_slave_security_group=self._get_param(
'Instances.EmrManagedSlaveSecurityGroup'),
service_access_security_group=self._get_param(
'Instances.ServiceAccessSecurityGroup'),
additional_master_security_groups=self._get_multi_param(
'Instances.AdditionalMasterSecurityGroups.member.'),
additional_slave_security_groups=self._get_multi_param('Instances.AdditionalSlaveSecurityGroups.member.'))
kwargs = dict(
@ -198,8 +214,10 @@ class ElasticMapReduceResponse(BaseResponse):
log_uri=self._get_param('LogUri'),
job_flow_role=self._get_param('JobFlowRole'),
service_role=self._get_param('ServiceRole'),
steps=steps_from_query_string(self._get_list_prefix('Steps.member')),
visible_to_all_users=self._get_bool_param('VisibleToAllUsers', False),
steps=steps_from_query_string(
self._get_list_prefix('Steps.member')),
visible_to_all_users=self._get_bool_param(
'VisibleToAllUsers', False),
instance_attrs=instance_attrs,
)
@ -225,7 +243,8 @@ class ElasticMapReduceResponse(BaseResponse):
if key.startswith('properties.'):
config.pop(key)
config['properties'] = {}
map_items = self._get_map_prefix('Configurations.member.{0}.Properties.entry'.format(idx))
map_items = self._get_map_prefix(
'Configurations.member.{0}.Properties.entry'.format(idx))
config['properties'] = map_items
kwargs['configurations'] = configurations
@ -239,7 +258,8 @@ class ElasticMapReduceResponse(BaseResponse):
'Only one AMI version and release label may be specified. '
'Provided AMI: {0}, release label: {1}.').format(
ami_version, release_label)
raise EmrError(error_type="ValidationException", message=message, template='single_error')
raise EmrError(error_type="ValidationException",
message=message, template='single_error')
else:
if ami_version:
kwargs['requested_ami_version'] = ami_version
@ -256,7 +276,8 @@ class ElasticMapReduceResponse(BaseResponse):
self.backend.add_applications(
cluster.id, [{'Name': 'Hadoop', 'Version': '0.18'}])
instance_groups = self._get_list_prefix('Instances.InstanceGroups.member')
instance_groups = self._get_list_prefix(
'Instances.InstanceGroups.member')
if instance_groups:
for ig in instance_groups:
ig['instance_count'] = int(ig['instance_count'])
@ -274,7 +295,8 @@ class ElasticMapReduceResponse(BaseResponse):
def set_termination_protection(self):
termination_protection = self._get_param('TerminationProtected')
job_ids = self._get_multi_param('JobFlowIds.member')
self.backend.set_termination_protection(job_ids, termination_protection)
self.backend.set_termination_protection(
job_ids, termination_protection)
template = self.response_template(SET_TERMINATION_PROTECTION_TEMPLATE)
return template.render()

6
moto/emr/utils.py
View file

@ -32,7 +32,8 @@ def tags_from_query_string(querystring_dict):
tag_key = querystring_dict.get("Tags.{0}.Key".format(tag_index))[0]
tag_value_key = "Tags.{0}.Value".format(tag_index)
if tag_value_key in querystring_dict:
response_values[tag_key] = querystring_dict.get(tag_value_key)[0]
response_values[tag_key] = querystring_dict.get(tag_value_key)[
0]
else:
response_values[tag_key] = None
return response_values
@ -42,7 +43,8 @@ def steps_from_query_string(querystring_dict):
steps = []
for step in querystring_dict:
step['jar'] = step.pop('hadoop_jar_step._jar')
step['properties'] = dict((o['Key'], o['Value']) for o in step.get('properties', []))
step['properties'] = dict((o['Key'], o['Value'])
for o in step.get('properties', []))
step['args'] = []
idx = 1
keyfmt = 'hadoop_jar_step._args.member.{0}'