catalin/moto
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/master' into adding_rds2_support

Browse source
This commit is contained in:
Mike Fuller 2015-02-17 07:04:33 +11:00
commit 95e48336d9
28 changed files with 893 additions and 156 deletions
Download patch file Download diff file Expand all files Collapse all files

2
moto/cloudformation/responses.py
View file

@ -1,7 +1,7 @@
from __future__ import unicode_literals
import json
from six.moves.urllib_parse import urlparse
from six.moves.urllib.parse import urlparse
from moto.core.responses import BaseResponse
from moto.s3 import s3_backend

28
moto/core/exceptions.py Normal file
View file

@ -0,0 +1,28 @@
from werkzeug.exceptions import HTTPException
from jinja2 import DictLoader, Environment
ERROR_RESPONSE = u"""<?xml version="1.0" encoding="UTF-8"?>
<Response>
<Errors>
<Error>
<Code>{{code}}</Code>
<Message>{{message}}</Message>
{% block extra %}{% endblock %}
</Error>
</Errors>
<RequestID>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestID>
</Response>
"""
class RESTError(HTTPException):
templates = {
'error': ERROR_RESPONSE
}
def __init__(self, code, message, template='error', **kwargs):
super(RESTError, self).__init__()
env = Environment(loader=DictLoader(self.templates))
self.description = env.get_template(template).render(
code=code, message=message, **kwargs)

24
moto/ec2/exceptions.py
View file

@ -1,13 +1,9 @@
from __future__ import unicode_literals
from werkzeug.exceptions import BadRequest
from jinja2 import Template
from moto.core.exceptions import RESTError
class EC2ClientError(BadRequest):
def __init__(self, code, message):
super(EC2ClientError, self).__init__()
self.description = ERROR_RESPONSE_TEMPLATE.render(
code=code, message=message)
class EC2ClientError(RESTError):
code = 400
class DependencyViolationError(EC2ClientError):
@ -306,17 +302,3 @@ class InvalidCIDRSubnetError(EC2ClientError):
"InvalidParameterValue",
"invalid CIDR subnet specification: {0}"
.format(cidr))
ERROR_RESPONSE = u"""<?xml version="1.0" encoding="UTF-8"?>
<Response>
<Errors>
<Error>
<Code>{{code}}</Code>
<Message>{{message}}</Message>
</Error>
</Errors>
<RequestID>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestID>
</Response>
"""
ERROR_RESPONSE_TEMPLATE = Template(ERROR_RESPONSE)

68
moto/ec2/models.py
View file

@ -69,6 +69,7 @@ from .utils import (
random_internet_gateway_id,
random_ip,
random_key_pair,
random_private_ip,
random_public_ip,
random_reservation_id,
random_route_table_id,
@ -163,7 +164,7 @@ class NetworkInterface(object):
group = self.ec2_backend.get_security_group_from_id(group_id)
if not group:
# Create with specific group ID.
group = SecurityGroup(group_id, group_id, group_id, vpc_id=subnet.vpc_id)
group = SecurityGroup(group.ec2_backend, group_id, group_id, group_id, vpc_id=subnet.vpc_id)
self.ec2_backend.groups[subnet.vpc_id][group_id] = group
if group:
self._group_set.append(group)
@ -174,9 +175,12 @@ class NetworkInterface(object):
security_group_ids = properties.get('SecurityGroups', [])
subnet_id = properties['SubnetId']
ec2_backend = ec2_backends[region_name]
subnet = ec2_backend.get_subnet(subnet_id)
subnet_id = properties.get('SubnetId')
if subnet_id:
subnet = ec2_backend.get_subnet(subnet_id)
else:
subnet = None
private_ip_address = properties.get('PrivateIpAddress', None)
@ -224,7 +228,7 @@ class NetworkInterfaceBackend(object):
super(NetworkInterfaceBackend, self).__init__()
def create_network_interface(self, subnet, private_ip_address, group_ids=None, **kwargs):
eni = NetworkInterface(self, subnet, private_ip_address, group_ids=group_ids)
eni = NetworkInterface(self, subnet, private_ip_address, group_ids=group_ids, **kwargs)
self.enis[eni.id] = eni
return eni
@ -297,10 +301,14 @@ class Instance(BotoInstance, TaggedEC2Resource):
self.instance_type = kwargs.get("instance_type", "m1.small")
self.vpc_id = None
self.subnet_id = kwargs.get("subnet_id")
in_ec2_classic = not bool(self.subnet_id)
self.key_name = kwargs.get("key_name")
self.source_dest_check = "true"
self.launch_time = datetime.utcnow().isoformat()
self.private_ip_address = kwargs.get('private_ip_address')
associate_public_ip = kwargs.get("associate_public_ip", False)
if in_ec2_classic:
# If we are in EC2-Classic, autoassign a public IP
associate_public_ip = True
self.block_device_mapping = BlockDeviceMapping()
self.block_device_mapping['/dev/sda1'] = BlockDeviceType(volume_id=random_volume_id())
@ -326,9 +334,26 @@ class Instance(BotoInstance, TaggedEC2Resource):
self.vpc_id = subnet.vpc_id
self.prep_nics(kwargs.get("nics", {}),
subnet_id=kwargs.get("subnet_id"),
subnet_id=self.subnet_id,
private_ip=kwargs.get("private_ip"),
associate_public_ip=kwargs.get("associate_public_ip"))
associate_public_ip=associate_public_ip)
@property
def private_ip(self):
return self.nics[0].private_ip_address
@property
def private_dns(self):
return "ip-{0}.ec2.internal".format(self.private_ip)
@property
def public_ip(self):
return self.nics[0].public_ip
@property
def public_dns(self):
if self.public_ip:
return "ec2-{0}.compute-1.amazonaws.com".format(self.public_ip)
@classmethod
def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name):
@ -346,7 +371,7 @@ class Instance(BotoInstance, TaggedEC2Resource):
instance_type=properties.get("InstanceType", "m1.small"),
subnet_id=properties.get("SubnetId"),
key_name=properties.get("KeyName"),
private_ip_address=properties.get('PrivateIpAddress'),
private_ip=properties.get('PrivateIpAddress'),
)
return reservation.instances[0]
@ -407,6 +432,9 @@ class Instance(BotoInstance, TaggedEC2Resource):
def prep_nics(self, nic_spec, subnet_id=None, private_ip=None, associate_public_ip=None):
self.nics = {}
if not private_ip:
private_ip = random_private_ip()
# Primary NIC defaults
primary_nic = {'SubnetId': subnet_id,
'PrivateIpAddress': private_ip,
@ -434,7 +462,10 @@ class Instance(BotoInstance, TaggedEC2Resource):
if device_index == 0 and primary_nic:
nic.update(primary_nic)
subnet = self.ec2_backend.get_subnet(nic['SubnetId'])
if 'SubnetId' in nic:
subnet = self.ec2_backend.get_subnet(nic['SubnetId'])
else:
subnet = None
group_id = nic.get('SecurityGroupId')
group_ids = [group_id] if group_id else []
@ -468,13 +499,13 @@ class Instance(BotoInstance, TaggedEC2Resource):
if attribute_name == 'AvailabilityZone':
return self.placement
elif attribute_name == 'PrivateDnsName':
return self.private_dns_name
return self.private_dns
elif attribute_name == 'PublicDnsName':
return self.public_dns_name
return self.public_dns
elif attribute_name == 'PrivateIp':
return self.private_ip_address
return self.private_ip
elif attribute_name == 'PublicIp':
return self.ip_address
return self.public_ip
raise UnformattedGetAttTemplateException()
@ -1016,8 +1047,9 @@ class SecurityRule(object):
return self.unique_representation == other.unique_representation
class SecurityGroup(object):
def __init__(self, group_id, name, description, vpc_id=None):
class SecurityGroup(TaggedEC2Resource):
def __init__(self, ec2_backend, group_id, name, description, vpc_id=None):
self.ec2_backend = ec2_backend
self.id = group_id
self.name = name
self.description = description
@ -1116,7 +1148,7 @@ class SecurityGroupBackend(object):
existing_group = self.get_security_group_from_name(name, vpc_id)
if existing_group:
raise InvalidSecurityGroupDuplicateError(name)
group = SecurityGroup(group_id, name, description, vpc_id=vpc_id)
group = SecurityGroup(self, group_id, name, description, vpc_id=vpc_id)
self.groups[vpc_id][group_id] = group
return group
@ -2031,10 +2063,12 @@ class VPCGatewayAttachment(object):
properties = cloudformation_json['Properties']
ec2_backend = ec2_backends[region_name]
return ec2_backend.create_vpc_gateway_attachment(
attachment = ec2_backend.create_vpc_gateway_attachment(
gateway_id=properties['InternetGatewayId'],
vpc_id=properties['VpcId'],
)
ec2_backend.attach_internet_gateway(properties['InternetGatewayId'], properties['VpcId'])
return attachment
@property
def physical_resource_id(self):

5
moto/ec2/responses/elastic_network_interfaces.py
View file

@ -25,6 +25,11 @@ class ElasticNetworkInterfaces(BaseResponse):
def describe_network_interfaces(self):
# Partially implemented. Supports only network-interface-id and group-id filters
filters = filters_from_querystring(self.querystring)
eni_ids = self._get_multi_param('NetworkInterfaceId.')
if 'network-interface-id' not in filters and eni_ids:
# Network interfaces can be filtered by passing the 'network-interface-id'
# filter or by passing the NetworkInterfaceId parameter
filters['network-interface-id'] = eni_ids
enis = self.ec2_backend.describe_network_interfaces(filters)
template = self.response_template(DESCRIBE_NETWORK_INTERFACES_RESPONSE)
return template.render(enis=enis)

16
moto/ec2/responses/instances.py
View file

@ -198,8 +198,8 @@ EC2_RUN_INSTANCES = """<RunInstancesResponse xmlns="http://ec2.amazonaws.com/doc
<code>0</code>
<name>pending</name>
</instanceState>
<privateDnsName/>
<dnsName/>
<privateDnsName>{{ instance.private_dns }}</privateDnsName>
<publicDnsName>{{ instance.public_dns }}</publicDnsName>
<reason/>
<keyName>{{ instance.key_name }}</keyName>
<amiLaunchIndex>0</amiLaunchIndex>
@ -216,9 +216,9 @@ EC2_RUN_INSTANCES = """<RunInstancesResponse xmlns="http://ec2.amazonaws.com/doc
{% if instance.nics %}
<subnetId>{{ instance.nics[0].subnet.id }}</subnetId>
<vpcId>{{ instance.nics[0].subnet.vpc_id }}</vpcId>
<privateIpAddress>{{ instance.nics[0].private_ip_address }}</privateIpAddress>
{% if instance.nics[0].public_ip %}
<ipAddress>{{ instance.nics[0].public_ip }}</ipAddress>
<privateIpAddress>{{ instance.private_ip }}</privateIpAddress>
{% if instance.public_ip %}
<ipAddress>{{ instance.public_ip }}</ipAddress>
{% endif %}
{% else %}
<subnetId>{{ instance.subnet_id }}</subnetId>
@ -318,8 +318,8 @@ EC2_DESCRIBE_INSTANCES = """<DescribeInstancesResponse xmlns='http://ec2.amazona
<code>{{ instance._state.code }}</code>
<name>{{ instance._state.name }}</name>
</instanceState>
<privateDnsName>ip-10.0.0.12.ec2.internal</privateDnsName>
<dnsName>ec2-46.51.219.63.compute-1.amazonaws.com</dnsName>
<privateDnsName>{{ instance.private_dns }}</privateDnsName>
<publicDnsName>{{ instance.public_dns }}</publicDnsName>
<reason>{{ instance._reason }}</reason>
<keyName>{{ instance.key_name }}</keyName>
<amiLaunchIndex>0</amiLaunchIndex>
@ -340,7 +340,7 @@ EC2_DESCRIBE_INSTANCES = """<DescribeInstancesResponse xmlns='http://ec2.amazona
{% if instance.nics %}
<subnetId>{{ instance.nics[0].subnet.id }}</subnetId>
<vpcId>{{ instance.nics[0].subnet.vpc_id }}</vpcId>
<privateIpAddress>{{ instance.nics[0].private_ip_address }}</privateIpAddress>
<privateIpAddress>{{ instance.private_ip }}</privateIpAddress>
{% if instance.nics[0].public_ip %}
<ipAddress>{{ instance.nics[0].public_ip }}</ipAddress>
{% endif %}

10
moto/ec2/responses/security_groups.py
View file

@ -133,6 +133,16 @@ DESCRIBE_SECURITY_GROUPS_RESPONSE = """<DescribeSecurityGroupsResponse xmlns="ht
{% endfor %}
</ipPermissions>
<ipPermissionsEgress/>
<tagSet>
{% for tag in group.get_tags() %}
<item>
<resourceId>{{ tag.resource_id }}</resourceId>
<resourceType>{{ tag.resource_type }}</resourceType>
<key>{{ tag.key }}</key>
<value>{{ tag.value }}</value>
</item>
{% endfor %}
</tagSet>
</item>
{% endfor %}
</securityGroupInfo>

12
moto/ec2/utils.py
View file

@ -130,6 +130,12 @@ def random_public_ip():
random.choice(range(255)))
def random_private_ip():
return '10.{0}.{1}.{2}'.format(random.choice(range(255)),
random.choice(range(255)),
random.choice(range(255)))
def random_ip():
return "127.{0}.{1}.{2}".format(
random.randint(0, 255),
@ -331,7 +337,7 @@ filter_dict_attribute_mapping = {
def passes_filter_dict(instance, filter_dict):
for filter_name, filter_values in filter_dict.items():
if filter_name in filter_dict_attribute_mapping:
instance_attr = filter_dict_attribute_mapping[filter_name]
instance_value = get_object_value(instance, instance_attr)
@ -347,14 +353,16 @@ def passes_filter_dict(instance, filter_dict):
filter_name)
return True
def instance_value_in_filter_values(instance_value, filter_values):
if isinstance(instance_value, list):
if not set(filter_values).intersection(set(instance_value)):
return False
elif instance_value not in filter_values:
elif instance_value not in filter_values:
return False
return True
def filter_reservations(reservations, filter_dict):
result = []
for reservation in reservations:

67
moto/emr/models.py
View file

@ -20,6 +20,35 @@ class FakeInstanceGroup(object):
self.num_instances = instance_count
class Cluster(object):
def __init__(self, id, name, availability_zone, ec2_key_name, subnet_id,
ec2_iam_profile, log_uri):
self.id = id
self.name = name
self.applications = []
self.auto_terminate = "false"
self.availability_zone = availability_zone
self.subnet_id = subnet_id
self.ec2_iam_profile = ec2_iam_profile
self.log_uri = log_uri
self.master_public_dns_name = ""
self.normalized_instance_hours = 0
self.requested_ami_version = "2.4.2"
self.running_ami_version = "2.4.2"
self.service_role = "my-service-role"
self.state = "RUNNING"
self.tags = {}
self.termination_protected = "false"
self.visible_to_all_users = "false"
def add_tags(self, tags):
self.tags.update(tags)
def remove_tags(self, tag_keys):
for key in tag_keys:
self.tags.pop(key, None)
class FakeStep(object):
def __init__(self, state, **kwargs):
# 'Steps.member.1.HadoopJarStep.Jar': ['/home/hadoop/contrib/streaming/hadoop-streaming.jar'],
@ -68,11 +97,24 @@ class FakeJobFlow(object):
self.normalized_instance_hours = 0
self.ec2_key_name = instance_attrs.get('ec2_key_name')
self.availability_zone = instance_attrs.get('placement.availability_zone')
self.subnet_id = instance_attrs.get('ec2_subnet_id')
self.keep_job_flow_alive_when_no_steps = instance_attrs.get('keep_job_flow_alive_when_no_steps')
self.termination_protected = instance_attrs.get('termination_protected')
self.instance_group_ids = []
def create_cluster(self):
cluster = Cluster(
id=self.id,
name=self.name,
availability_zone=self.availability_zone,
ec2_key_name=self.ec2_key_name,
subnet_id=self.subnet_id,
ec2_iam_profile=self.role,
log_uri=self.log_uri,
)
return cluster
def terminate(self):
self.state = 'TERMINATED'
@ -129,12 +171,15 @@ class ElasticMapReduceBackend(BaseBackend):
def __init__(self):
self.job_flows = {}
self.clusters = {}
self.instance_groups = {}
def run_job_flow(self, name, log_uri, job_flow_role, visible_to_all_users, steps, instance_attrs):
job_id = random_job_id()
job_flow = FakeJobFlow(job_id, name, log_uri, job_flow_role, visible_to_all_users, steps, instance_attrs)
self.job_flows[job_id] = job_flow
cluster = job_flow.create_cluster()
self.clusters[cluster.id] = cluster
return job_flow
def add_job_flow_steps(self, job_flow_id, steps):
@ -142,8 +187,12 @@ class ElasticMapReduceBackend(BaseBackend):
job_flow.add_steps(steps)
return job_flow
def describe_job_flows(self):
return self.job_flows.values()
def describe_job_flows(self, job_flow_ids=None):
jobs = self.job_flows.values()
if job_flow_ids:
return [job for job in jobs if job.id in job_flow_ids]
else:
return jobs
def terminate_job_flows(self, job_ids):
flows = [flow for flow in self.describe_job_flows() if flow.id in job_ids]
@ -151,6 +200,12 @@ class ElasticMapReduceBackend(BaseBackend):
flow.terminate()
return flows
def list_clusters(self):
return self.clusters.values()
def get_cluster(self, cluster_id):
return self.clusters[cluster_id]
def get_instance_groups(self, instance_group_ids):
return [
group for group_id, group
@ -181,5 +236,13 @@ class ElasticMapReduceBackend(BaseBackend):
job = self.job_flows[job_id]
job.set_visibility(visible_to_all_users)
def add_tags(self, cluster_id, tags):
cluster = self.get_cluster(cluster_id)
cluster.add_tags(tags)
def remove_tags(self, cluster_id, tag_keys):
cluster = self.get_cluster(cluster_id)
cluster.remove_tags(tag_keys)
emr_backend = ElasticMapReduceBackend()

125
moto/emr/responses.py
View file

@ -2,6 +2,7 @@ from __future__ import unicode_literals
from moto.core.responses import BaseResponse
from .models import emr_backend
from .utils import tags_from_query_string
class ElasticMapReduceResponse(BaseResponse):
@ -30,7 +31,8 @@ class ElasticMapReduceResponse(BaseResponse):
return template.render(job_flow=job_flow)
def describe_job_flows(self):
job_flows = emr_backend.describe_job_flows()
job_flow_ids = self._get_multi_param("JobFlowIds.member")
job_flows = emr_backend.describe_job_flows(job_flow_ids)
template = self.response_template(DESCRIBE_JOB_FLOWS_TEMPLATE)
return template.render(job_flows=job_flows)
@ -60,6 +62,31 @@ class ElasticMapReduceResponse(BaseResponse):
template = self.response_template(SET_VISIBLE_TO_ALL_USERS_TEMPLATE)
return template.render()
def list_clusters(self):
clusters = emr_backend.list_clusters()
template = self.response_template(LIST_CLUSTERS_TEMPLATE)
return template.render(clusters=clusters)
def describe_cluster(self):
cluster_id = self._get_param('ClusterId')
cluster = emr_backend.get_cluster(cluster_id)
template = self.response_template(DESCRIBE_CLUSTER_TEMPLATE)
return template.render(cluster=cluster)
def add_tags(self):
cluster_id = self._get_param('ResourceId')
tags = tags_from_query_string(self.querystring)
emr_backend.add_tags(cluster_id, tags)
template = self.response_template(ADD_TAGS_TEMPLATE)
return template.render()
def remove_tags(self):
cluster_id = self._get_param('ResourceId')
tag_keys = self._get_multi_param('TagKeys.member')
emr_backend.remove_tags(cluster_id, tag_keys)
template = self.response_template(REMOVE_TAGS_TEMPLATE)
return template.render()
RUN_JOB_FLOW_TEMPLATE = """<RunJobFlowResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
<RunJobFlowResult>
@ -163,6 +190,85 @@ ADD_JOB_FLOW_STEPS_TEMPLATE = """<AddJobFlowStepsResponse xmlns="http://elasticm
</ResponseMetadata>
</AddJobFlowStepsResponse>"""
LIST_CLUSTERS_TEMPLATE = """<ListClustersResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
<Clusters>
{% for cluster in clusters %}
<member>
<Id>{{ cluster.id }}</Id>
<Name>{{ cluster.name }}</Name>
<NormalizedInstanceHours>{{ cluster.normalized_instance_hours }}</NormalizedInstanceHours>
<Status>
<State>{{ cluster.state }}</State>
<StateChangeReason>
<Code></Code>
<Message></Message>
</StateChangeReason>
<Timeline></Timeline>
</Status>
</member>
{% endfor %}
</Clusters>
<Marker></Marker>
<ResponseMetadata>
<RequestId>
2690d7eb-ed86-11dd-9877-6fad448a8418
</RequestId>
</ResponseMetadata>
</ListClustersResponse>"""
DESCRIBE_CLUSTER_TEMPLATE = """<DescribeClusterResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
<DescribeClusterResult>
<Cluster>
<Id>{{ cluster.id }}</Id>
<Tags>
{% for tag_key, tag_value in cluster.tags.items() %}
<member>
<Key>{{ tag_key }}</Key>
<Value>{{ tag_value }}</Value>
</member>
{% endfor %}
</Tags>
<Ec2InstanceAttributes>
<Ec2AvailabilityZone>{{ cluster.availability_zone }}</Ec2AvailabilityZone>
<Ec2SubnetId>{{ cluster.subnet_id }}</Ec2SubnetId>
<Ec2KeyName>{{ cluster.ec2_key_name }}</Ec2KeyName>
</Ec2InstanceAttributes>
<RunningAmiVersion>{{ cluster.running_ami_version }}</RunningAmiVersion>
<VisibleToAllUsers>{{ cluster.visible_to_all_users }}</VisibleToAllUsers>
<Status>
<StateChangeReason>
<Message>Terminated by user request</Message>
<Code>USER_REQUEST</Code>
</StateChangeReason>
<State>{{ cluster.state }}</State>
<Timeline>
<CreationDateTime>2014-01-24T01:21:21Z</CreationDateTime>
<ReadyDateTime>2014-01-24T01:25:26Z</ReadyDateTime>
<EndDateTime>2014-01-24T02:19:46Z</EndDateTime>
</Timeline>
</Status>
<AutoTerminate>{{ cluster.auto_terminate }}</AutoTerminate>
<Name>{{ cluster.name }}</Name>
<RequestedAmiVersion>{{ cluster.requested_ami_version }}</RequestedAmiVersion>
<Applications>
{% for application in cluster.applications %}
<member>
<Name>{{ application.name }}</Name>
<Version>{{ application.version }}</Version>
</member>
{% endfor %}
</Applications>
<TerminationProtected>{{ cluster.termination_protection }}</TerminationProtected>
<MasterPublicDnsName>ec2-184-0-0-1.us-west-1.compute.amazonaws.com</MasterPublicDnsName>
<NormalizedInstanceHours>{{ cluster.normalized_instance_hours }}</NormalizedInstanceHours>
<ServiceRole>{{ cluster.service_role }}</ServiceRole>
</Cluster>
</DescribeClusterResult>
<ResponseMetadata>
<RequestId>aaaaaaaa-bbbb-cccc-dddd-eeeeeeeeeeee</RequestId>
</ResponseMetadata>
</DescribeClusterResponse>"""
ADD_INSTANCE_GROUPS_TEMPLATE = """<AddInstanceGroupsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
<InstanceGroupIds>{% for instance_group in instance_groups %}{{ instance_group.id }}{% if loop.index != loop.length %},{% endif %}{% endfor %}</InstanceGroupIds>
</AddInstanceGroupsResponse>"""
@ -182,3 +288,20 @@ SET_VISIBLE_TO_ALL_USERS_TEMPLATE = """<SetVisibleToAllUsersResponse xmlns="http
</RequestId>
</ResponseMetadata>
</SetVisibleToAllUsersResponse>"""
ADD_TAGS_TEMPLATE = """<AddTagsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
<ResponseMetadata>
<RequestId>
2690d7eb-ed86-11dd-9877-6fad448a8419
</RequestId>
</ResponseMetadata>
</AddTagsResponse>"""
REMOVE_TAGS_TEMPLATE = """<RemoveTagsResponse xmlns="http://elasticmapreduce.amazonaws.com/doc/2009-03-31">
<ResponseMetadata>
<RequestId>
2690d7eb-ed86-11dd-9877-6fad448a8419
</RequestId>
</ResponseMetadata>
</RemoveTagsResponse>"""

16
moto/emr/utils.py
View file

@ -14,3 +14,19 @@ def random_instance_group_id(size=13):
chars = list(range(10)) + list(string.ascii_uppercase)
job_tag = ''.join(six.text_type(random.choice(chars)) for x in range(size))
return 'i-{0}'.format(job_tag)
def tags_from_query_string(querystring_dict):
prefix = 'Tags'
suffix = 'Key'
response_values = {}
for key, value in querystring_dict.items():
if key.startswith(prefix) and key.endswith(suffix):
tag_index = key.replace(prefix + ".", "").replace("." + suffix, "")
tag_key = querystring_dict.get("Tags.{0}.Key".format(tag_index))[0]
tag_value_key = "Tags.{0}.Value".format(tag_index)
if tag_value_key in querystring_dict:
response_values[tag_key] = querystring_dict.get(tag_value_key)[0]
else:
response_values[tag_key] = None
return response_values

57
moto/iam/models.py
View file

@ -4,7 +4,7 @@ from boto.exception import BotoServerError
from moto.core import BaseBackend
from .utils import random_access_key, random_alphanumeric, random_resource_id
from datetime import datetime
import base64
class Role(object):
@ -135,7 +135,7 @@ class User(object):
datetime.utcnow(),
"%Y-%m-%d-%H-%M-%S"
)
self.arn = 'arn:aws:iam::123456789012:user/{0}'.format(name)
self.policies = {}
self.access_keys = []
self.password = None
@ -184,6 +184,45 @@ class User(object):
raise NotImplementedError('"Fn::GetAtt" : [ "{0}" , "Arn" ]"')
raise UnformattedGetAttTemplateException()
def to_csv(self):
date_format = '%Y-%m-%dT%H:%M:%S+00:00'
date_created = datetime.strptime(self.created, '%Y-%m-%d-%H-%M-%S')
# aagrawal,arn:aws:iam::509284790694:user/aagrawal,2014-09-01T22:28:48+00:00,true,2014-11-12T23:36:49+00:00,2014-09-03T18:59:00+00:00,N/A,false,true,2014-09-01T22:28:48+00:00,false,N/A,false,N/A,false,N/A
if not self.password:
password_enabled = 'false'
password_last_used = 'not_supported'
else:
password_enabled = 'true'
password_last_used = 'no_information'
if len(self.access_keys) == 0:
access_key_1_active = 'false'
access_key_1_last_rotated = 'N/A'
access_key_2_active = 'false'
access_key_2_last_rotated = 'N/A'
elif len(self.access_keys) == 1:
access_key_1_active = 'true'
access_key_1_last_rotated = date_created.strftime(date_format)
access_key_2_active = 'false'
access_key_2_last_rotated = 'N/A'
else:
access_key_1_active = 'true'
access_key_1_last_rotated = date_created.strftime(date_format)
access_key_2_active = 'true'
access_key_2_last_rotated = date_created.strftime(date_format)
return '{0},{1},{2},{3},{4},{5},not_supported,false,{6},{7},{8},{9},false,N/A,false,N/A'.format(self.name,
self.arn,
date_created.strftime(date_format),
password_enabled,
password_last_used,
date_created.strftime(date_format),
access_key_1_active,
access_key_1_last_rotated,
access_key_2_active,
access_key_2_last_rotated
)
class IAMBackend(BaseBackend):
@ -193,6 +232,7 @@ class IAMBackend(BaseBackend):
self.certificates = {}
self.groups = {}
self.users = {}
self.credential_report = None
super(IAMBackend, self).__init__()
def create_role(self, role_name, assume_role_policy_document, path):
@ -394,5 +434,18 @@ class IAMBackend(BaseBackend):
except KeyError:
raise BotoServerError(404, 'Not Found')
def report_generated(self):
return self.credential_report
def generate_report(self):
self.credential_report = True
def get_credential_report(self):
if not self.credential_report:
raise BotoServerError(410, 'ReportNotPresent')
report = 'user,arn,user_creation_time,password_enabled,password_last_used,password_last_changed,password_next_rotation,mfa_active,access_key_1_active,access_key_1_last_rotated,access_key_2_active,access_key_2_last_rotated,cert_1_active,cert_1_last_rotated,cert_2_active,cert_2_last_rotated\n'
for user in self.users:
report += self.users[user].to_csv()
return base64.b64encode(report.encode('ascii')).decode('ascii')
iam_backend = IAMBackend()

43
moto/iam/responses.py
View file

@ -219,6 +219,18 @@ class IamResponse(BaseResponse):
template = self.response_template(GENERIC_EMPTY_TEMPLATE)
return template.render(name='DeleteUser')
def generate_credential_report(self):
if iam_backend.report_generated():
template = self.response_template(CREDENTIAL_REPORT_GENERATED)
else:
template = self.response_template(CREDENTIAL_REPORT_GENERATING)
iam_backend.generate_report()
return template.render()
def get_credential_report(self):
report = iam_backend.get_credential_report()
template = self.response_template(CREDENTIAL_REPORT)
return template.render(report=report)
GENERIC_EMPTY_TEMPLATE = """<{{ name }}Response>
<ResponseMetadata>
@ -559,3 +571,34 @@ LIST_ACCESS_KEYS_TEMPLATE = """<ListAccessKeysResponse>
<RequestId>7a62c49f-347e-4fc4-9331-6e8eEXAMPLE</RequestId>
</ResponseMetadata>
</ListAccessKeysResponse>"""
CREDENTIAL_REPORT_GENERATING = """
<GenerateCredentialReportResponse>
<GenerateCredentialReportResult>
<state>STARTED</state>
<description>No report exists. Starting a new report generation task</description>
</GenerateCredentialReportResult>
<ResponseMetadata>
<RequestId>fa788a82-aa8a-11e4-a278-1786c418872b"</RequestId>
</ResponseMetadata>
</GenerateCredentialReportResponse>"""
CREDENTIAL_REPORT_GENERATED = """<GenerateCredentialReportResponse>
<GenerateCredentialReportResult>
<state>COMPLETE</state>
</GenerateCredentialReportResult>
<ResponseMetadata>
<RequestId>fa788a82-aa8a-11e4-a278-1786c418872b"</RequestId>
</ResponseMetadata>
</GenerateCredentialReportResponse>"""
CREDENTIAL_REPORT = """<GetCredentialReportResponse>
<GetCredentialReportResult>
<content>{{ report }}</content>
<GeneratedTime>2015-02-02T20:02:02Z</GeneratedTime>
<ReportFormat>text/csv</ReportFormat>
</GetCredentialReportResult>
<ResponseMetadata>
<RequestId>fa788a82-aa8a-11e4-a278-1786c418872b"</RequestId>
</ResponseMetadata>
</GetCredentialReportResponse>"""

6
moto/rds/models.py
View file

@ -199,6 +199,12 @@ class Database(object):
<PubliclyAccessible>{{ database.publicly_accessible }}</PubliclyAccessible>
<AutoMinorVersionUpgrade>{{ database.auto_minor_version_upgrade }}</AutoMinorVersionUpgrade>
<AllocatedStorage>{{ database.allocated_storage }}</AllocatedStorage>
{% if database.iops %}
<Iops>{{ database.iops }}</Iops>
<StorageType>io1</StorageType>
{% else %}
<StorageType>{{ database.storage_type }}</StorageType>
{% endif %}
<DBInstanceClass>{{ database.db_instance_class }}</DBInstanceClass>
<MasterUsername>{{ database.master_username }}</MasterUsername>
<Endpoint>

2
moto/rds/urls.py
View file

@ -2,7 +2,7 @@ from __future__ import unicode_literals
from .responses import RDSResponse
url_bases = [
"https?://rds.(.+).amazonaws.com",
"https?://rds(\..+)?.amazonaws.com",
]
url_paths = {

70
moto/s3/exceptions.py
View file

@ -1,9 +1,73 @@
from __future__ import unicode_literals
from moto.core.exceptions import RESTError
class BucketAlreadyExists(Exception):
ERROR_WITH_BUCKET_NAME = """{% extends 'error' %}
{% block extra %}<BucketName>{{ bucket }}</BucketName>{% endblock %}
"""
class S3ClientError(RESTError):
pass
class MissingBucket(Exception):
pass
class BucketError(S3ClientError):
def __init__(self, *args, **kwargs):
kwargs.setdefault('template', 'bucket_error')
self.templates['bucket_error'] = ERROR_WITH_BUCKET_NAME
super(BucketError, self).__init__(*args, **kwargs)
class BucketAlreadyExists(BucketError):
code = 409
def __init__(self, *args, **kwargs):
super(BucketAlreadyExists, self).__init__(
"BucketAlreadyExists",
("The requested bucket name is not available. The bucket "
"namespace is shared by all users of the system. Please "
"select a different name and try again"),
*args, **kwargs)
class MissingBucket(BucketError):
code = 404
def __init__(self, *args, **kwargs):
super(MissingBucket, self).__init__(
"NoSuchBucket",
"The specified bucket does not exist",
*args, **kwargs)
class InvalidPartOrder(S3ClientError):
code = 400
def __init__(self, *args, **kwargs):
super(InvalidPartOrder, self).__init__(
"InvalidPartOrder",
("The list of parts was not in ascending order. The parts "
"list must be specified in order by part number."),
*args, **kwargs)
class InvalidPart(S3ClientError):
code = 400
def __init__(self, *args, **kwargs):
super(InvalidPart, self).__init__(
"InvalidPart",
("One or more of the specified parts could not be found. "
"The part might not have been uploaded, or the specified "
"entity tag might not have matched the part's entity tag."),
*args, **kwargs)
class EntityTooSmall(S3ClientError):
code = 400
def __init__(self, *args, **kwargs):
super(EntityTooSmall, self).__init__(
"EntityTooSmall",
"Your proposed upload is smaller than the minimum allowed object size.",
*args, **kwargs)

44
moto/s3/models.py
View file

@ -8,9 +8,10 @@ import itertools
import codecs
import six
from bisect import insort
from moto.core import BaseBackend
from moto.core.utils import iso_8601_datetime_with_milliseconds, rfc_1123_datetime
from .exceptions import BucketAlreadyExists, MissingBucket
from .exceptions import BucketAlreadyExists, MissingBucket, InvalidPart, EntityTooSmall
from .utils import clean_key_name, _VersionedKeyStore
UPLOAD_ID_BYTES = 43
@ -118,25 +119,32 @@ class FakeMultipart(object):
self.key_name = key_name
self.metadata = metadata
self.parts = {}
self.partlist = [] # ordered list of part ID's
rand_b64 = base64.b64encode(os.urandom(UPLOAD_ID_BYTES))
self.id = rand_b64.decode('utf-8').replace('=', '').replace('+', '')
def complete(self):
def complete(self, body):
decode_hex = codecs.getdecoder("hex_codec")
total = bytearray()
md5s = bytearray()
last_part_name = len(self.list_parts())
for part in self.list_parts():
if part.name != last_part_name and len(part.value) < UPLOAD_PART_MIN_SIZE:
return None, None
last = None
count = 0
for pn, etag in body:
part = self.parts.get(pn)
if part is None or part.etag != etag:
raise InvalidPart()
if last is not None and len(last.value) < UPLOAD_PART_MIN_SIZE:
raise EntityTooSmall()
part_etag = part.etag.replace('"', '')
md5s.extend(decode_hex(part_etag)[0])
total.extend(part.value)
last = part
count += 1
etag = hashlib.md5()
etag.update(bytes(md5s))
return total, "{0}-{1}".format(etag.hexdigest(), last_part_name)
return total, "{0}-{1}".format(etag.hexdigest(), count)
def set_part(self, part_id, value):
if part_id < 1:
@ -144,18 +152,12 @@ class FakeMultipart(object):
key = FakeKey(part_id, value)
self.parts[part_id] = key
insort(self.partlist, part_id)
return key
def list_parts(self):
parts = []
for part_id, index in enumerate(sorted(self.parts.keys()), start=1):
# Make sure part ids are continuous
if part_id != index:
return
parts.append(self.parts[part_id])
return parts
for part_id in self.partlist:
yield self.parts[part_id]
class FakeBucket(object):
@ -191,7 +193,7 @@ class S3Backend(BaseBackend):
def create_bucket(self, bucket_name, region_name):
if bucket_name in self.buckets:
raise BucketAlreadyExists()
raise BucketAlreadyExists(bucket=bucket_name)
new_bucket = FakeBucket(name=bucket_name, region_name=region_name)
self.buckets[bucket_name] = new_bucket
return new_bucket
@ -203,7 +205,7 @@ class S3Backend(BaseBackend):
try:
return self.buckets[bucket_name]
except KeyError:
raise MissingBucket()
raise MissingBucket(bucket=bucket_name)
def delete_bucket(self, bucket_name):
bucket = self.get_bucket(bucket_name)
@ -279,10 +281,10 @@ class S3Backend(BaseBackend):
return new_multipart
def complete_multipart(self, bucket_name, multipart_id):
def complete_multipart(self, bucket_name, multipart_id, body):
bucket = self.get_bucket(bucket_name)
multipart = bucket.multiparts[multipart_id]
value, etag = multipart.complete()
value, etag = multipart.complete(body)
if value is None:
return
del bucket.multiparts[multipart_id]
@ -297,7 +299,7 @@ class S3Backend(BaseBackend):
def list_multipart(self, bucket_name, multipart_id):
bucket = self.get_bucket(bucket_name)
return bucket.multiparts[multipart_id].list_parts()
return list(bucket.multiparts[multipart_id].list_parts())
def get_all_multiparts(self, bucket_name):
bucket = self.get_bucket(bucket_name)

107
moto/s3/responses.py
View file

@ -7,7 +7,7 @@ from six.moves.urllib.parse import parse_qs, urlparse
from moto.core.responses import _TemplateEnvironmentMixin
from .exceptions import BucketAlreadyExists, MissingBucket
from .exceptions import BucketAlreadyExists, S3ClientError, InvalidPartOrder
from .models import s3_backend
from .utils import bucket_name_from_url, metadata_from_headers
from xml.dom import minidom
@ -35,8 +35,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
def bucket_response(self, request, full_url, headers):
try:
response = self._bucket_response(request, full_url, headers)
except MissingBucket:
return 404, headers, ""
except S3ClientError as s3error:
response = s3error.code, headers, s3error.description
if isinstance(response, six.string_types):
return 200, headers, response.encode("utf-8")
@ -72,12 +72,8 @@ class ResponseObject(_TemplateEnvironmentMixin):
raise NotImplementedError("Method {0} has not been impelemented in the S3 backend yet".format(method))
def _bucket_response_head(self, bucket_name, headers):
try:
self.backend.get_bucket(bucket_name)
except MissingBucket:
return 404, headers, ""
else:
return 200, headers, ""
self.backend.get_bucket(bucket_name)
return 200, headers, ""
def _bucket_response_get(self, bucket_name, querystring, headers):
if 'uploads' in querystring:
@ -127,11 +123,7 @@ class ResponseObject(_TemplateEnvironmentMixin):
is_truncated='false',
)
try:
bucket = self.backend.get_bucket(bucket_name)
except MissingBucket:
return 404, headers, ""
bucket = self.backend.get_bucket(bucket_name)
prefix = querystring.get('prefix', [None])[0]
delimiter = querystring.get('delimiter', [None])[0]
result_keys, result_folders = self.backend.prefix_query(bucket, prefix, delimiter)
@ -161,17 +153,12 @@ class ResponseObject(_TemplateEnvironmentMixin):
# us-east-1 has different behavior
new_bucket = self.backend.get_bucket(bucket_name)
else:
return 409, headers, ""
raise
template = self.response_template(S3_BUCKET_CREATE_RESPONSE)
return 200, headers, template.render(bucket=new_bucket)
def _bucket_response_delete(self, bucket_name, headers):
try:
removed_bucket = self.backend.delete_bucket(bucket_name)
except MissingBucket:
# Non-existant bucket
template = self.response_template(S3_DELETE_NON_EXISTING_BUCKET)
return 404, headers, template.render(bucket_name=bucket_name)
removed_bucket = self.backend.delete_bucket(bucket_name)
if removed_bucket:
# Bucket exists
@ -228,17 +215,43 @@ class ResponseObject(_TemplateEnvironmentMixin):
return 200, headers, template.render(deleted=deleted_names, delete_errors=error_names)
def _handle_range_header(self, request, headers, response_content):
length = len(response_content)
last = length - 1
_, rspec = request.headers.get('range').split('=')
if ',' in rspec:
raise NotImplementedError(
"Multiple range specifiers not supported")
toint = lambda i: int(i) if i else None
begin, end = map(toint, rspec.split('-'))
if begin is not None: # byte range
end = last if end is None else end
elif end is not None: # suffix byte range
begin = length - end
end = last
else:
return 400, headers, ""
if begin < 0 or end > length or begin > min(end, last):
return 416, headers, ""
headers['content-range'] = "bytes {0}-{1}/{2}".format(
begin, end, length)
return 206, headers, response_content[begin:end + 1]
def key_response(self, request, full_url, headers):
try:
response = self._key_response(request, full_url, headers)
except MissingBucket:
return 404, headers, ""
except S3ClientError as s3error:
response = s3error.code, headers, s3error.description
if isinstance(response, six.string_types):
return 200, headers, response
status_code = 200
response_content = response
else:
status_code, headers, response_content = response
return status_code, headers, response_content
if status_code == 200 and 'range' in request.headers:
return self._handle_range_header(request, headers, response_content)
return status_code, headers, response_content
def _key_response(self, request, full_url, headers):
parsed_url = urlparse(full_url)
@ -364,6 +377,15 @@ class ResponseObject(_TemplateEnvironmentMixin):
template = self.response_template(S3_DELETE_OBJECT_SUCCESS)
return 204, headers, template.render(bucket=removed_key)
def _complete_multipart_body(self, body):
ps = minidom.parseString(body).getElementsByTagName('Part')
prev = 0
for p in ps:
pn = int(p.getElementsByTagName('PartNumber')[0].firstChild.wholeText)
if pn <= prev:
raise InvalidPartOrder()
yield (pn, p.getElementsByTagName('ETag')[0].firstChild.wholeText)
def _key_response_post(self, request, body, parsed_url, bucket_name, query, key_name, headers):
if body == b'' and parsed_url.query == 'uploads':
metadata = metadata_from_headers(request.headers)
@ -378,18 +400,15 @@ class ResponseObject(_TemplateEnvironmentMixin):
return 200, headers, response
if 'uploadId' in query:
body = self._complete_multipart_body(body)
upload_id = query['uploadId'][0]
key = self.backend.complete_multipart(bucket_name, upload_id)
if key is not None:
template = self.response_template(S3_MULTIPART_COMPLETE_RESPONSE)
return template.render(
bucket_name=bucket_name,
key_name=key.name,
etag=key.etag,
)
template = self.response_template(S3_MULTIPART_COMPLETE_TOO_SMALL_ERROR)
return 400, headers, template.render()
key = self.backend.complete_multipart(bucket_name, upload_id, body)
template = self.response_template(S3_MULTIPART_COMPLETE_RESPONSE)
return template.render(
bucket_name=bucket_name,
key_name=key.name,
etag=key.etag,
)
elif parsed_url.query == 'restore':
es = minidom.parseString(body).getElementsByTagName('Days')
days = es[0].childNodes[0].wholeText
@ -461,14 +480,6 @@ S3_DELETE_BUCKET_SUCCESS = """<DeleteBucketResponse xmlns="http://s3.amazonaws.c
</DeleteBucketResponse>
</DeleteBucketResponse>"""
S3_DELETE_NON_EXISTING_BUCKET = """<?xml version="1.0" encoding="UTF-8"?>
<Error><Code>NoSuchBucket</Code>
<Message>The specified bucket does not exist</Message>
<BucketName>{{ bucket_name }}</BucketName>
<RequestId>asdfasdfsadf</RequestId>
<HostId>asfasdfsfsafasdf</HostId>
</Error>"""
S3_DELETE_BUCKET_WITH_ITEMS_ERROR = """<?xml version="1.0" encoding="UTF-8"?>
<Error><Code>BucketNotEmpty</Code>
<Message>The bucket you tried to delete is not empty</Message>
@ -609,14 +620,6 @@ S3_MULTIPART_COMPLETE_RESPONSE = """<?xml version="1.0" encoding="UTF-8"?>
</CompleteMultipartUploadResult>
"""
S3_MULTIPART_COMPLETE_TOO_SMALL_ERROR = """<?xml version="1.0" encoding="UTF-8"?>
<Error>
<Code>EntityTooSmall</Code>
<Message>Your proposed upload is smaller than the minimum allowed object size.</Message>
<RequestId>asdfasdfsdafds</RequestId>
<HostId>sdfgdsfgdsfgdfsdsfgdfs</HostId>
</Error>"""
S3_ALL_MULTIPARTS = """<?xml version="1.0" encoding="UTF-8"?>
<ListMultipartUploadsResult xmlns="http://s3.amazonaws.com/doc/2006-03-01/">
<Bucket>{{ bucket_name }}</Bucket>