Lints.
This commit is contained in:
Steve Pulec
parent
1433f28846
commit
f37bad0e00
260 changed files with 6363 additions and 3766 deletions
|
|
@ -100,7 +100,8 @@ def test_describe_cluster():
|
|||
# cluster.status.timeline.enddatetime.should.be.a(six.string_types)
|
||||
# cluster.status.timeline.readydatetime.should.be.a(six.string_types)
|
||||
|
||||
dict((item.key, item.value) for item in cluster.tags).should.equal(input_tags)
|
||||
dict((item.key, item.value)
|
||||
for item in cluster.tags).should.equal(input_tags)
|
||||
|
||||
cluster.terminationprotected.should.equal('false')
|
||||
cluster.visibletoallusers.should.equal('true')
|
||||
|
|
@ -285,7 +286,8 @@ def test_list_clusters():
|
|||
y = expected[x.id]
|
||||
x.id.should.equal(y['id'])
|
||||
x.name.should.equal(y['name'])
|
||||
x.normalizedinstancehours.should.equal(y['normalizedinstancehours'])
|
||||
x.normalizedinstancehours.should.equal(
|
||||
y['normalizedinstancehours'])
|
||||
x.status.state.should.equal(y['state'])
|
||||
x.status.timeline.creationdatetime.should.be.a(six.string_types)
|
||||
if y['state'] == 'TERMINATED':
|
||||
|
|
@ -371,11 +373,13 @@ def test_run_jobflow_with_instance_groups():
|
|||
job_id = conn.run_jobflow(instance_groups=input_instance_groups,
|
||||
**run_jobflow_args)
|
||||
job_flow = conn.describe_jobflow(job_id)
|
||||
int(job_flow.instancecount).should.equal(sum(g.num_instances for g in input_instance_groups))
|
||||
int(job_flow.instancecount).should.equal(
|
||||
sum(g.num_instances for g in input_instance_groups))
|
||||
for instance_group in job_flow.instancegroups:
|
||||
expected = input_groups[instance_group.name]
|
||||
instance_group.should.have.property('instancegroupid')
|
||||
int(instance_group.instancerunningcount).should.equal(expected.num_instances)
|
||||
int(instance_group.instancerunningcount).should.equal(
|
||||
expected.num_instances)
|
||||
instance_group.instancerole.should.equal(expected.role)
|
||||
instance_group.instancetype.should.equal(expected.type)
|
||||
instance_group.market.should.equal(expected.market)
|
||||
|
|
@ -483,7 +487,8 @@ def test_instance_groups():
|
|||
conn.add_instance_groups(job_id, input_instance_groups[2:])
|
||||
|
||||
jf = conn.describe_jobflow(job_id)
|
||||
int(jf.instancecount).should.equal(sum(g.num_instances for g in input_instance_groups))
|
||||
int(jf.instancecount).should.equal(
|
||||
sum(g.num_instances for g in input_instance_groups))
|
||||
for x in jf.instancegroups:
|
||||
y = input_groups[x.name]
|
||||
if hasattr(y, 'bidprice'):
|
||||
|
|
@ -572,7 +577,8 @@ def test_steps():
|
|||
list(arg.value for arg in step.args).should.have.length_of(8)
|
||||
step.creationdatetime.should.be.a(six.string_types)
|
||||
# step.enddatetime.should.be.a(six.string_types)
|
||||
step.jar.should.equal('/home/hadoop/contrib/streaming/hadoop-streaming.jar')
|
||||
step.jar.should.equal(
|
||||
'/home/hadoop/contrib/streaming/hadoop-streaming.jar')
|
||||
step.laststatechangereason.should.be.a(six.string_types)
|
||||
step.mainclass.should.equal('')
|
||||
step.name.should.be.a(six.string_types)
|
||||
|
|
@ -592,7 +598,8 @@ def test_steps():
|
|||
'-input', y.input,
|
||||
'-output', y.output,
|
||||
])
|
||||
x.config.jar.should.equal('/home/hadoop/contrib/streaming/hadoop-streaming.jar')
|
||||
x.config.jar.should.equal(
|
||||
'/home/hadoop/contrib/streaming/hadoop-streaming.jar')
|
||||
x.config.mainclass.should.equal('')
|
||||
# properties
|
||||
x.should.have.property('id').should.be.a(six.string_types)
|
||||
|
|
@ -610,7 +617,8 @@ def test_steps():
|
|||
'-input', y.input,
|
||||
'-output', y.output,
|
||||
])
|
||||
x.config.jar.should.equal('/home/hadoop/contrib/streaming/hadoop-streaming.jar')
|
||||
x.config.jar.should.equal(
|
||||
'/home/hadoop/contrib/streaming/hadoop-streaming.jar')
|
||||
x.config.mainclass.should.equal('')
|
||||
# properties
|
||||
x.should.have.property('id').should.be.a(six.string_types)
|
||||
|
|
|
|||
|
|
@ -88,15 +88,20 @@ def test_describe_cluster():
|
|||
config['Properties'].should.equal(args['Configurations'][0]['Properties'])
|
||||
|
||||
attrs = cl['Ec2InstanceAttributes']
|
||||
attrs['AdditionalMasterSecurityGroups'].should.equal(args['Instances']['AdditionalMasterSecurityGroups'])
|
||||
attrs['AdditionalSlaveSecurityGroups'].should.equal(args['Instances']['AdditionalSlaveSecurityGroups'])
|
||||
attrs['AdditionalMasterSecurityGroups'].should.equal(
|
||||
args['Instances']['AdditionalMasterSecurityGroups'])
|
||||
attrs['AdditionalSlaveSecurityGroups'].should.equal(
|
||||
args['Instances']['AdditionalSlaveSecurityGroups'])
|
||||
attrs['Ec2AvailabilityZone'].should.equal('us-east-1a')
|
||||
attrs['Ec2KeyName'].should.equal(args['Instances']['Ec2KeyName'])
|
||||
attrs['Ec2SubnetId'].should.equal(args['Instances']['Ec2SubnetId'])
|
||||
attrs['EmrManagedMasterSecurityGroup'].should.equal(args['Instances']['EmrManagedMasterSecurityGroup'])
|
||||
attrs['EmrManagedSlaveSecurityGroup'].should.equal(args['Instances']['EmrManagedSlaveSecurityGroup'])
|
||||
attrs['EmrManagedMasterSecurityGroup'].should.equal(
|
||||
args['Instances']['EmrManagedMasterSecurityGroup'])
|
||||
attrs['EmrManagedSlaveSecurityGroup'].should.equal(
|
||||
args['Instances']['EmrManagedSlaveSecurityGroup'])
|
||||
attrs['IamInstanceProfile'].should.equal(args['JobFlowRole'])
|
||||
attrs['ServiceAccessSecurityGroup'].should.equal(args['Instances']['ServiceAccessSecurityGroup'])
|
||||
attrs['ServiceAccessSecurityGroup'].should.equal(
|
||||
args['Instances']['ServiceAccessSecurityGroup'])
|
||||
cl['Id'].should.equal(cluster_id)
|
||||
cl['LogUri'].should.equal(args['LogUri'])
|
||||
cl['MasterPublicDnsName'].should.be.a(six.string_types)
|
||||
|
|
@ -222,11 +227,14 @@ def test_describe_job_flow():
|
|||
ig['State'].should.equal('RUNNING')
|
||||
attrs['KeepJobFlowAliveWhenNoSteps'].should.equal(True)
|
||||
# attrs['MasterInstanceId'].should.be.a(six.string_types)
|
||||
attrs['MasterInstanceType'].should.equal(args['Instances']['MasterInstanceType'])
|
||||
attrs['MasterInstanceType'].should.equal(
|
||||
args['Instances']['MasterInstanceType'])
|
||||
attrs['MasterPublicDnsName'].should.be.a(six.string_types)
|
||||
attrs['NormalizedInstanceHours'].should.equal(0)
|
||||
attrs['Placement']['AvailabilityZone'].should.equal(args['Instances']['Placement']['AvailabilityZone'])
|
||||
attrs['SlaveInstanceType'].should.equal(args['Instances']['SlaveInstanceType'])
|
||||
attrs['Placement']['AvailabilityZone'].should.equal(
|
||||
args['Instances']['Placement']['AvailabilityZone'])
|
||||
attrs['SlaveInstanceType'].should.equal(
|
||||
args['Instances']['SlaveInstanceType'])
|
||||
attrs['TerminationProtected'].should.equal(False)
|
||||
jf['JobFlowId'].should.equal(cluster_id)
|
||||
jf['JobFlowRole'].should.equal(args['JobFlowRole'])
|
||||
|
|
@ -282,14 +290,18 @@ def test_list_clusters():
|
|||
y = expected[x['Id']]
|
||||
x['Id'].should.equal(y['Id'])
|
||||
x['Name'].should.equal(y['Name'])
|
||||
x['NormalizedInstanceHours'].should.equal(y['NormalizedInstanceHours'])
|
||||
x['NormalizedInstanceHours'].should.equal(
|
||||
y['NormalizedInstanceHours'])
|
||||
x['Status']['State'].should.equal(y['State'])
|
||||
x['Status']['Timeline']['CreationDateTime'].should.be.a('datetime.datetime')
|
||||
x['Status']['Timeline'][
|
||||
'CreationDateTime'].should.be.a('datetime.datetime')
|
||||
if y['State'] == 'TERMINATED':
|
||||
x['Status']['Timeline']['EndDateTime'].should.be.a('datetime.datetime')
|
||||
x['Status']['Timeline'][
|
||||
'EndDateTime'].should.be.a('datetime.datetime')
|
||||
else:
|
||||
x['Status']['Timeline'].shouldnt.have.key('EndDateTime')
|
||||
x['Status']['Timeline']['ReadyDateTime'].should.be.a('datetime.datetime')
|
||||
x['Status']['Timeline'][
|
||||
'ReadyDateTime'].should.be.a('datetime.datetime')
|
||||
marker = resp.get('Marker')
|
||||
if marker is None:
|
||||
break
|
||||
|
|
@ -316,8 +328,10 @@ def test_run_job_flow():
|
|||
resp['ExecutionStatusDetail']['State'].should.equal('WAITING')
|
||||
resp['JobFlowId'].should.equal(cluster_id)
|
||||
resp['Name'].should.equal(args['Name'])
|
||||
resp['Instances']['MasterInstanceType'].should.equal(args['Instances']['MasterInstanceType'])
|
||||
resp['Instances']['SlaveInstanceType'].should.equal(args['Instances']['SlaveInstanceType'])
|
||||
resp['Instances']['MasterInstanceType'].should.equal(
|
||||
args['Instances']['MasterInstanceType'])
|
||||
resp['Instances']['SlaveInstanceType'].should.equal(
|
||||
args['Instances']['SlaveInstanceType'])
|
||||
resp['LogUri'].should.equal(args['LogUri'])
|
||||
resp['VisibleToAllUsers'].should.equal(args['VisibleToAllUsers'])
|
||||
resp['Instances']['NormalizedInstanceHours'].should.equal(0)
|
||||
|
|
@ -333,7 +347,8 @@ def test_run_job_flow_with_invalid_params():
|
|||
args['AmiVersion'] = '2.4'
|
||||
args['ReleaseLabel'] = 'emr-5.0.0'
|
||||
client.run_job_flow(**args)
|
||||
ex.exception.response['Error']['Message'].should.contain('ValidationException')
|
||||
ex.exception.response['Error'][
|
||||
'Message'].should.contain('ValidationException')
|
||||
|
||||
|
||||
@mock_emr
|
||||
|
|
@ -378,7 +393,8 @@ def test_run_job_flow_with_instance_groups():
|
|||
args = deepcopy(run_job_flow_args)
|
||||
args['Instances'] = {'InstanceGroups': input_instance_groups}
|
||||
cluster_id = client.run_job_flow(**args)['JobFlowId']
|
||||
groups = client.list_instance_groups(ClusterId=cluster_id)['InstanceGroups']
|
||||
groups = client.list_instance_groups(ClusterId=cluster_id)[
|
||||
'InstanceGroups']
|
||||
for x in groups:
|
||||
y = input_groups[x['Name']]
|
||||
x.should.have.key('Id')
|
||||
|
|
@ -484,10 +500,12 @@ def test_instance_groups():
|
|||
jf = client.describe_job_flows(JobFlowIds=[cluster_id])['JobFlows'][0]
|
||||
base_instance_count = jf['Instances']['InstanceCount']
|
||||
|
||||
client.add_instance_groups(JobFlowId=cluster_id, InstanceGroups=input_instance_groups[2:])
|
||||
client.add_instance_groups(
|
||||
JobFlowId=cluster_id, InstanceGroups=input_instance_groups[2:])
|
||||
|
||||
jf = client.describe_job_flows(JobFlowIds=[cluster_id])['JobFlows'][0]
|
||||
jf['Instances']['InstanceCount'].should.equal(sum(g['InstanceCount'] for g in input_instance_groups))
|
||||
jf['Instances']['InstanceCount'].should.equal(
|
||||
sum(g['InstanceCount'] for g in input_instance_groups))
|
||||
for x in jf['Instances']['InstanceGroups']:
|
||||
y = input_groups[x['Name']]
|
||||
if hasattr(y, 'BidPrice'):
|
||||
|
|
@ -506,7 +524,8 @@ def test_instance_groups():
|
|||
x['StartDateTime'].should.be.a('datetime.datetime')
|
||||
x['State'].should.equal('RUNNING')
|
||||
|
||||
groups = client.list_instance_groups(ClusterId=cluster_id)['InstanceGroups']
|
||||
groups = client.list_instance_groups(ClusterId=cluster_id)[
|
||||
'InstanceGroups']
|
||||
for x in groups:
|
||||
y = input_groups[x['Name']]
|
||||
if hasattr(y, 'BidPrice'):
|
||||
|
|
@ -525,9 +544,11 @@ def test_instance_groups():
|
|||
x['Status']['State'].should.equal('RUNNING')
|
||||
x['Status']['StateChangeReason']['Code'].should.be.a(six.string_types)
|
||||
# x['Status']['StateChangeReason']['Message'].should.be.a(six.string_types)
|
||||
x['Status']['Timeline']['CreationDateTime'].should.be.a('datetime.datetime')
|
||||
x['Status']['Timeline'][
|
||||
'CreationDateTime'].should.be.a('datetime.datetime')
|
||||
# x['Status']['Timeline']['EndDateTime'].should.be.a('datetime.datetime')
|
||||
x['Status']['Timeline']['ReadyDateTime'].should.be.a('datetime.datetime')
|
||||
x['Status']['Timeline'][
|
||||
'ReadyDateTime'].should.be.a('datetime.datetime')
|
||||
|
||||
igs = dict((g['Name'], g) for g in groups)
|
||||
client.modify_instance_groups(
|
||||
|
|
@ -592,14 +613,19 @@ def test_steps():
|
|||
# x['ExecutionStatusDetail'].should.have.key('EndDateTime')
|
||||
# x['ExecutionStatusDetail'].should.have.key('LastStateChangeReason')
|
||||
# x['ExecutionStatusDetail'].should.have.key('StartDateTime')
|
||||
x['ExecutionStatusDetail']['State'].should.equal('STARTING' if idx == 0 else 'PENDING')
|
||||
x['ExecutionStatusDetail']['State'].should.equal(
|
||||
'STARTING' if idx == 0 else 'PENDING')
|
||||
x['StepConfig']['ActionOnFailure'].should.equal('TERMINATE_CLUSTER')
|
||||
x['StepConfig']['HadoopJarStep']['Args'].should.equal(y['HadoopJarStep']['Args'])
|
||||
x['StepConfig']['HadoopJarStep']['Jar'].should.equal(y['HadoopJarStep']['Jar'])
|
||||
x['StepConfig']['HadoopJarStep'][
|
||||
'Args'].should.equal(y['HadoopJarStep']['Args'])
|
||||
x['StepConfig']['HadoopJarStep'][
|
||||
'Jar'].should.equal(y['HadoopJarStep']['Jar'])
|
||||
if 'MainClass' in y['HadoopJarStep']:
|
||||
x['StepConfig']['HadoopJarStep']['MainClass'].should.equal(y['HadoopJarStep']['MainClass'])
|
||||
x['StepConfig']['HadoopJarStep']['MainClass'].should.equal(
|
||||
y['HadoopJarStep']['MainClass'])
|
||||
if 'Properties' in y['HadoopJarStep']:
|
||||
x['StepConfig']['HadoopJarStep']['Properties'].should.equal(y['HadoopJarStep']['Properties'])
|
||||
x['StepConfig']['HadoopJarStep']['Properties'].should.equal(
|
||||
y['HadoopJarStep']['Properties'])
|
||||
x['StepConfig']['Name'].should.equal(y['Name'])
|
||||
|
||||
expected = dict((s['Name'], s) for s in input_steps)
|
||||
|
|
@ -617,7 +643,8 @@ def test_steps():
|
|||
x['Name'].should.equal(y['Name'])
|
||||
x['Status']['State'].should.be.within(['STARTING', 'PENDING'])
|
||||
# StateChangeReason
|
||||
x['Status']['Timeline']['CreationDateTime'].should.be.a('datetime.datetime')
|
||||
x['Status']['Timeline'][
|
||||
'CreationDateTime'].should.be.a('datetime.datetime')
|
||||
# x['Status']['Timeline']['EndDateTime'].should.be.a('datetime.datetime')
|
||||
# x['Status']['Timeline']['StartDateTime'].should.be.a('datetime.datetime')
|
||||
|
||||
|
|
@ -631,7 +658,8 @@ def test_steps():
|
|||
x['Name'].should.equal(y['Name'])
|
||||
x['Status']['State'].should.be.within(['STARTING', 'PENDING'])
|
||||
# StateChangeReason
|
||||
x['Status']['Timeline']['CreationDateTime'].should.be.a('datetime.datetime')
|
||||
x['Status']['Timeline'][
|
||||
'CreationDateTime'].should.be.a('datetime.datetime')
|
||||
# x['Status']['Timeline']['EndDateTime'].should.be.a('datetime.datetime')
|
||||
# x['Status']['Timeline']['StartDateTime'].should.be.a('datetime.datetime')
|
||||
|
||||
|
|
@ -640,7 +668,8 @@ def test_steps():
|
|||
steps.should.have.length_of(1)
|
||||
steps[0]['Id'].should.equal(step_id)
|
||||
|
||||
steps = client.list_steps(ClusterId=cluster_id, StepStates=['STARTING'])['Steps']
|
||||
steps = client.list_steps(ClusterId=cluster_id,
|
||||
StepStates=['STARTING'])['Steps']
|
||||
steps.should.have.length_of(1)
|
||||
steps[0]['Id'].should.equal(step_id)
|
||||
|
||||
|
|
@ -656,8 +685,10 @@ def test_tags():
|
|||
client.add_tags(ResourceId=cluster_id, Tags=input_tags)
|
||||
resp = client.describe_cluster(ClusterId=cluster_id)['Cluster']
|
||||
resp['Tags'].should.have.length_of(2)
|
||||
dict((t['Key'], t['Value']) for t in resp['Tags']).should.equal(dict((t['Key'], t['Value']) for t in input_tags))
|
||||
dict((t['Key'], t['Value']) for t in resp['Tags']).should.equal(
|
||||
dict((t['Key'], t['Value']) for t in input_tags))
|
||||
|
||||
client.remove_tags(ResourceId=cluster_id, TagKeys=[t['Key'] for t in input_tags])
|
||||
client.remove_tags(ResourceId=cluster_id, TagKeys=[
|
||||
t['Key'] for t in input_tags])
|
||||
resp = client.describe_cluster(ClusterId=cluster_id)['Cluster']
|
||||
resp['Tags'].should.equal([])
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue