diff --git a/moto/core/responses.py b/moto/core/responses.py index 5a66423c..af421724 100644 --- a/moto/core/responses.py +++ b/moto/core/responses.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals import datetime import json +import logging import re import pytz @@ -18,6 +19,9 @@ from moto.compat import OrderedDict from moto.core.utils import camelcase_to_underscores, method_names_from_class +log = logging.getLogger(__name__) + + def _decode_dict(d): decoded = {} for key, value in d.items(): @@ -550,11 +554,20 @@ def xml_to_json_response(service_spec, operation, xml, result_node=None): od = OrderedDict() for k, v in value.items(): - if k.startswith('@') or v is None: + if k.startswith('@'): + continue + + if k not in spec: + # this can happen when with an older version of + # botocore for which the node in XML template is not + # defined in service spec. + log.warning('Field %s is not defined by the botocore version in use', k) continue if spec[k]['type'] == 'list': - if len(spec[k]['member']) == 1: + if v is None: + od[k] = [] + elif len(spec[k]['member']) == 1: if isinstance(v['member'], list): od[k] = transform(v['member'], spec[k]['member']) else: @@ -566,11 +579,22 @@ def xml_to_json_response(service_spec, operation, xml, result_node=None): else: raise ValueError('Malformatted input') elif spec[k]['type'] == 'map': - key = from_str(v['entry']['key'], spec[k]['key']) - val = from_str(v['entry']['value'], spec[k]['value']) - od[k] = {key: val} + if v is None: + od[k] = {} + else: + items = ([v['entry']] if not isinstance(v['entry'], list) else + v['entry']) + for item in items: + key = from_str(item['key'], spec[k]['key']) + val = from_str(item['value'], spec[k]['value']) + if k not in od: + od[k] = {} + od[k][key] = val else: - od[k] = transform(v, spec[k]) + if v is None: + od[k] = None + else: + od[k] = transform(v, spec[k]) return od dic = xmltodict.parse(xml) diff --git a/moto/emr/models.py b/moto/emr/models.py index 48ccd7c2..f9242833 100644 --- a/moto/emr/models.py +++ b/moto/emr/models.py @@ -26,12 +26,19 @@ class FakeBootstrapAction(object): class FakeInstanceGroup(object): - def __init__(self, instance_count, instance_role, instance_type, market, name, - id=None, bid_price=None): + def __init__(self, instance_count, instance_role, instance_type, + market='ON_DEMAND', name=None, id=None, bid_price=None): self.id = id or random_instance_group_id() self.bid_price = bid_price self.market = market + if name is None: + if instance_role == 'MASTER': + name = 'master' + elif instance_role == 'CORE': + name = 'slave' + else: + name = 'Task instance group' self.name = name self.num_instances = instance_count self.role = instance_role diff --git a/requirements-dev.txt b/requirements-dev.txt index 781b1dcb..9bdccc6e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -5,21 +5,6 @@ sure==1.2.24 coverage freezegun flask -# botocore 1.4.29 breaks the following tests: -# test_nat_gateway.test_delete_nat_gateway -# test `list -> create -> list -> get -> delete -> list` integration -# test_lambda.test_get_function -# test_lambda.test_delete_function -# test_lambda.test_create_function_from_zipfile -# test_lambda.test_create_function_from_aws_bucket -# test_apigateway.test_integrations -# test_apigateway.test_integration_response -# test_apigateway.test_deployment -# test_apigateway.test_create_resource -# test_apigateway.test_create_method_response -# test_apigateway.test_create_method -# test_apigateway.test_child_resource -# so we need to pin a boto3 and botocore revision pair that we know works -boto3==1.3.1 -botocore==1.4.28 +boto3>=1.3.1 +botocore>=1.4.28 six diff --git a/tests/test_apigateway/test_apigateway.py b/tests/test_apigateway/test_apigateway.py index b60f41e4..fc41d3bc 100644 --- a/tests/test_apigateway/test_apigateway.py +++ b/tests/test_apigateway/test_apigateway.py @@ -79,6 +79,7 @@ def test_create_resource(): resourceId=root_id, ) root_resource['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + root_resource['ResponseMetadata'].pop('RetryAttempts', None) root_resource.should.equal({ 'path': '/', 'id': root_id, @@ -137,6 +138,7 @@ def test_child_resource(): resourceId=tags_id, ) child_resource['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + child_resource['ResponseMetadata'].pop('RetryAttempts', None) child_resource.should.equal({ 'path': '/users/tags', 'pathPart': 'tags', @@ -173,6 +175,7 @@ def test_create_method(): ) response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response.should.equal({ 'httpMethod': 'GET', 'authorizationType': 'none', @@ -212,6 +215,7 @@ def test_create_method_response(): statusCode='200', ) response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response.should.equal({ 'ResponseMetadata': {'HTTPStatusCode': 200}, 'statusCode': '200' @@ -224,6 +228,7 @@ def test_create_method_response(): statusCode='200', ) response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response.should.equal({ 'ResponseMetadata': {'HTTPStatusCode': 200}, 'statusCode': '200' @@ -236,6 +241,7 @@ def test_create_method_response(): statusCode='200', ) response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response.should.equal({'ResponseMetadata': {'HTTPStatusCode': 200}}) @@ -273,6 +279,7 @@ def test_integrations(): uri='http://httpbin.org/robots.txt', ) response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response.should.equal({ 'ResponseMetadata': {'HTTPStatusCode': 200}, 'httpMethod': 'GET', @@ -294,6 +301,7 @@ def test_integrations(): httpMethod='GET' ) response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response.should.equal({ 'ResponseMetadata': {'HTTPStatusCode': 200}, 'httpMethod': 'GET', @@ -314,6 +322,7 @@ def test_integrations(): resourceId=root_id, ) response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response['resourceMethods']['GET']['methodIntegration'].should.equal({ 'httpMethod': 'GET', 'integrationResponses': { @@ -363,6 +372,7 @@ def test_integrations(): requestTemplates=templates ) response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response['ResponseMetadata'].should.equal({'HTTPStatusCode': 200}) response = client.get_integration( @@ -416,6 +426,7 @@ def test_integration_response(): selectionPattern='foobar', ) response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response.should.equal({ 'statusCode': '200', 'selectionPattern': 'foobar', @@ -432,6 +443,7 @@ def test_integration_response(): statusCode='200', ) response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response.should.equal({ 'statusCode': '200', 'selectionPattern': 'foobar', @@ -447,6 +459,7 @@ def test_integration_response(): httpMethod='GET', ) response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response['methodIntegration']['integrationResponses'].should.equal({ '200': { 'responseTemplates': { @@ -495,6 +508,7 @@ def test_update_stage_configuration(): ) response.pop('createdDate',None) # createdDate is hard to match against, remove it response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response.should.equal({ 'id': deployment_id, 'ResponseMetadata': {'HTTPStatusCode': 200}, @@ -658,6 +672,7 @@ def test_create_stage(): ) response.pop('createdDate',None) # createdDate is hard to match against, remove it response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response.should.equal({ 'id': deployment_id, 'ResponseMetadata': {'HTTPStatusCode': 200}, @@ -677,6 +692,7 @@ def test_create_stage(): ) response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response['items'][0].pop('createdDate') response['items'][1].pop('createdDate') @@ -688,6 +704,7 @@ def test_create_stage(): response = client.create_stage(restApiId=api_id,stageName=new_stage_name,deploymentId=deployment_id2) response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response.should.equal({ 'stageName':new_stage_name, @@ -712,6 +729,7 @@ def test_create_stage(): }) response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response.should.equal({ 'stageName':new_stage_name_with_vars, @@ -737,6 +755,7 @@ def test_create_stage(): }, cacheClusterEnabled=True,description="hello moto") response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response.should.equal({ 'stageName':new_stage_name, @@ -762,6 +781,7 @@ def test_create_stage(): }, cacheClusterEnabled=True,cacheClusterSize="1.6",description="hello moto") response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response.should.equal({ 'stageName':new_stage_name, @@ -807,6 +827,7 @@ def test_deployment(): ) response.pop('createdDate',None) # createdDate is hard to match against, remove it response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response.should.equal({ 'id': deployment_id, 'ResponseMetadata': {'HTTPStatusCode': 200}, diff --git a/tests/test_ec2/test_nat_gateway.py b/tests/test_ec2/test_nat_gateway.py index f162a9d2..b9c95f7c 100644 --- a/tests/test_ec2/test_nat_gateway.py +++ b/tests/test_ec2/test_nat_gateway.py @@ -57,6 +57,7 @@ def test_delete_nat_gateway(): response = conn.delete_nat_gateway(NatGatewayId=nat_gateway_id) response['ResponseMetadata'].pop('HTTPHeaders', None) # this is hard to match against, so remove it + response['ResponseMetadata'].pop('RetryAttempts', None) response.should.equal({ 'NatGatewayId': nat_gateway_id, 'ResponseMetadata': { diff --git a/tests/test_emr/test_emr.py b/tests/test_emr/test_emr.py index 08cb75dc..71b3b8ec 100644 --- a/tests/test_emr/test_emr.py +++ b/tests/test_emr/test_emr.py @@ -46,6 +46,8 @@ def test_describe_cluster(): 'Configurations.member.1.Classification': 'yarn-site', 'Configurations.member.1.Properties.entry.1.key': 'someproperty', 'Configurations.member.1.Properties.entry.1.value': 'somevalue', + 'Configurations.member.1.Properties.entry.2.key': 'someotherproperty', + 'Configurations.member.1.Properties.entry.2.value': 'someothervalue', 'Instances.EmrManagedMasterSecurityGroup': 'master-security-group', 'Instances.Ec2SubnetId': 'subnet-8be41cec', }, diff --git a/tests/test_emr/test_emr_boto3.py b/tests/test_emr/test_emr_boto3.py index 4a9df162..1a735967 100644 --- a/tests/test_emr/test_emr_boto3.py +++ b/tests/test_emr/test_emr_boto3.py @@ -63,7 +63,8 @@ def test_describe_cluster(): args['Applications'] = [{'Name': 'Spark', 'Version': '2.4.2'}] args['Configurations'] = [ {'Classification': 'yarn-site', - 'Properties': {'someproperty': 'somevalue'}}] + 'Properties': {'someproperty': 'somevalue', + 'someotherproperty': 'someothervalue'}}] args['Instances']['AdditionalMasterSecurityGroups'] = ['additional-master'] args['Instances']['AdditionalSlaveSecurityGroups'] = ['additional-slave'] args['Instances']['Ec2KeyName'] = 'mykey' @@ -232,8 +233,8 @@ def test_describe_job_flow(): jf['LogUri'].should.equal(args['LogUri']) jf['Name'].should.equal(args['Name']) jf['ServiceRole'].should.equal(args['ServiceRole']) - jf.shouldnt.have.key('Steps') - jf.shouldnt.have.key('SupportedProducts') + jf['Steps'].should.equal([]) + jf['SupportedProducts'].should.equal([]) jf['VisibleToAllUsers'].should.equal(True) @@ -320,7 +321,7 @@ def test_run_job_flow(): resp['LogUri'].should.equal(args['LogUri']) resp['VisibleToAllUsers'].should.equal(args['VisibleToAllUsers']) resp['Instances']['NormalizedInstanceHours'].should.equal(0) - resp.shouldnt.have.key('Steps') + resp['Steps'].should.equal([]) @mock_emr @@ -445,10 +446,11 @@ def test_bootstrap_actions(): {'Name': 'bs1', 'ScriptBootstrapAction': { 'Args': ['arg1', 'arg2'], - 'Path': 'path/to/script'}}, + 'Path': 's3://path/to/script'}}, {'Name': 'bs2', 'ScriptBootstrapAction': { - 'Path': 'path/to/anotherscript'}} + 'Args': [], + 'Path': 's3://path/to/anotherscript'}} ] client = boto3.client('emr', region_name='us-east-1') @@ -658,4 +660,4 @@ def test_tags(): client.remove_tags(ResourceId=cluster_id, TagKeys=[t['Key'] for t in input_tags]) resp = client.describe_cluster(ClusterId=cluster_id)['Cluster'] - resp.shouldnt.have.key('Tags') + resp['Tags'].should.equal([])