catalin/moto
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

lambda + SNS enhancements (#1048)

Browse source 
* updates

- support lambda messages from SNS
- run lambda in docker container

* decode output

* populate timeout

* simplify

* whoops

* skeletons of cloudwatchlogs

* impl filter log streams

* fix logging

* PEP fixes

* PEP fixes

* fix reset

* fix reset

* add new endpoint

* fix region name

* add docker

* try to fix tests

* try to fix travis issue with boto

* fix escaping in urls

* fix environment variables

* fix PEP

* more pep

* switch back to precise

* another fix attempt

* fix typo

* fix lambda invoke

* fix more unittests

* work on getting this to work in new scheme

* fix py2

* fix error

* fix tests when running in server mode

* more lambda fixes

* try running with latest docker

adapted from aiodocker

* switch to docker python client

* pep fixes

* switch to docker volume

* fix unittest

* fix invoke from sns

* fix zip2tar

* add hack impl for get_function with zip

* try fix

* fix for py < 3.6

* add volume refcount

* try to fix travis

* docker test

* fix yaml

* try fix

* update endpoints

* fix

* another attempt

* try again

* fix recursive import

* refactor fix

* revert changes with better fix

* more reverts

* wait for service to come up

* add back detached mode

* sleep and add another exception type

* put this back for logging

* put back with note

* whoops :)

* docker in docker!

* fix invalid url

* hopefully last fix!

* fix lambda regions

* fix protocol

* travis!!!!

* just run lambda test for now

* use one print

* fix escaping

* another attempt

* yet another

* re-enable all tests

* fixes

* fix for py2

* revert change

* fix for py2.7

* fix output ordering

* remove this given there's a new unittest that covers it

* changes based on review

- add skeleton logs test file
- switch to docker image that matches test env
- fix mock_logs import

* add readme entry
This commit is contained in:
Alexander Mohr 2017-09-27 16:04:58 -07:00 • committed by Jack Danger
commit 9008b85299
21 changed files with 836 additions and 167 deletions
Download patch file Download diff file Expand all files Collapse all files

150
tests/test_awslambda/test_lambda.py
View file

@ -12,11 +12,13 @@ import sure # noqa
from freezegun import freeze_time
from moto import mock_lambda, mock_s3, mock_ec2, settings
_lambda_region = 'us-east-1' if settings.TEST_SERVER_MODE else 'us-west-2'
def _process_lamda(pfunc):
def _process_lambda(func_str):
zip_output = io.BytesIO()
zip_file = zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED)
zip_file.writestr('lambda_function.zip', pfunc)
zip_file.writestr('lambda_function.py', func_str)
zip_file.close()
zip_output.seek(0)
return zip_output.read()
@ -27,21 +29,23 @@ def get_test_zip_file1():
def lambda_handler(event, context):
return event
"""
return _process_lamda(pfunc)
return _process_lambda(pfunc)
def get_test_zip_file2():
pfunc = """
func_str = """
import boto3
def lambda_handler(event, context):
ec2 = boto3.resource('ec2', region_name='us-west-2', endpoint_url='http://{base_url}')
volume_id = event.get('volume_id')
print('get volume details for %s' % volume_id)
import boto3
ec2 = boto3.resource('ec2', region_name='us-west-2', endpoint_url="http://{base_url}")
vol = ec2.Volume(volume_id)
print('Volume - %s state=%s, size=%s' % (volume_id, vol.state, vol.size))
print('get volume details for %s\\nVolume - %s state=%s, size=%s' % (volume_id, volume_id, vol.state, vol.size))
return event
""".format(base_url="localhost:5000" if settings.TEST_SERVER_MODE else "ec2.us-west-2.amazonaws.com")
return _process_lamda(pfunc)
""".format(base_url="motoserver:5000" if settings.TEST_SERVER_MODE else "ec2.us-west-2.amazonaws.com")
return _process_lambda(func_str)
@mock_lambda
@ -58,7 +62,7 @@ def test_invoke_requestresponse_function():
FunctionName='testFunction',
Runtime='python2.7',
Role='test-iam-role',
Handler='lambda_function.handler',
Handler='lambda_function.lambda_handler',
Code={
'ZipFile': get_test_zip_file1(),
},
@ -73,10 +77,13 @@ def test_invoke_requestresponse_function():
Payload=json.dumps(in_data))
success_result["StatusCode"].should.equal(202)
base64.b64decode(success_result["LogResult"]).decode(
'utf-8').should.equal(json.dumps(in_data))
json.loads(success_result["Payload"].read().decode(
'utf-8')).should.equal(in_data)
result_obj = json.loads(
base64.b64decode(success_result["LogResult"]).decode('utf-8'))
result_obj.should.equal(in_data)
payload = success_result["Payload"].read().decode('utf-8')
json.loads(payload).should.equal(in_data)
@mock_lambda
@ -86,7 +93,7 @@ def test_invoke_event_function():
FunctionName='testFunction',
Runtime='python2.7',
Role='test-iam-role',
Handler='lambda_function.handler',
Handler='lambda_function.lambda_handler',
Code={
'ZipFile': get_test_zip_file1(),
},
@ -110,36 +117,47 @@ def test_invoke_event_function():
'utf-8')).should.equal({})
@mock_ec2
@mock_lambda
def test_invoke_function_get_ec2_volume():
conn = boto3.resource("ec2", "us-west-2")
vol = conn.create_volume(Size=99, AvailabilityZone='us-west-2')
vol = conn.Volume(vol.id)
if settings.TEST_SERVER_MODE:
@mock_ec2
@mock_lambda
def test_invoke_function_get_ec2_volume():
conn = boto3.resource("ec2", "us-west-2")
vol = conn.create_volume(Size=99, AvailabilityZone='us-west-2')
vol = conn.Volume(vol.id)
conn = boto3.client('lambda', 'us-west-2')
conn.create_function(
FunctionName='testFunction',
Runtime='python2.7',
Role='test-iam-role',
Handler='lambda_function.handler',
Code={
'ZipFile': get_test_zip_file2(),
},
Description='test lambda function',
Timeout=3,
MemorySize=128,
Publish=True,
)
conn = boto3.client('lambda', 'us-west-2')
conn.create_function(
FunctionName='testFunction',
Runtime='python2.7',
Role='test-iam-role',
Handler='lambda_function.lambda_handler',
Code={
'ZipFile': get_test_zip_file2(),
},
Description='test lambda function',
Timeout=3,
MemorySize=128,
Publish=True,
)
in_data = {'volume_id': vol.id}
result = conn.invoke(FunctionName='testFunction',
InvocationType='RequestResponse', Payload=json.dumps(in_data))
result["StatusCode"].should.equal(202)
msg = 'get volume details for %s\nVolume - %s state=%s, size=%s\n%s' % (
vol.id, vol.id, vol.state, vol.size, json.dumps(in_data))
base64.b64decode(result["LogResult"]).decode('utf-8').should.equal(msg)
result['Payload'].read().decode('utf-8').should.equal(msg)
in_data = {'volume_id': vol.id}
result = conn.invoke(FunctionName='testFunction',
InvocationType='RequestResponse', Payload=json.dumps(in_data))
result["StatusCode"].should.equal(202)
msg = 'get volume details for %s\nVolume - %s state=%s, size=%s\n%s' % (
vol.id, vol.id, vol.state, vol.size, json.dumps(in_data))
log_result = base64.b64decode(result["LogResult"]).decode('utf-8')
# fix for running under travis (TODO: investigate why it has an extra newline)
log_result = log_result.replace('\n\n', '\n')
log_result.should.equal(msg)
payload = result['Payload'].read().decode('utf-8')
# fix for running under travis (TODO: investigate why it has an extra newline)
payload = payload.replace('\n\n', '\n')
payload.should.equal(msg)
@mock_lambda
@ -150,7 +168,7 @@ def test_create_based_on_s3_with_missing_bucket():
FunctionName='testFunction',
Runtime='python2.7',
Role='test-iam-role',
Handler='lambda_function.handler',
Handler='lambda_function.lambda_handler',
Code={
'S3Bucket': 'this-bucket-does-not-exist',
'S3Key': 'test.zip',
@ -181,7 +199,7 @@ def test_create_function_from_aws_bucket():
FunctionName='testFunction',
Runtime='python2.7',
Role='test-iam-role',
Handler='lambda_function.handler',
Handler='lambda_function.lambda_handler',
Code={
'S3Bucket': 'test-bucket',
'S3Key': 'test.zip',
@ -202,10 +220,10 @@ def test_create_function_from_aws_bucket():
result.pop('LastModified')
result.should.equal({
'FunctionName': 'testFunction',
'FunctionArn': 'arn:aws:lambda:123456789012:function:testFunction',
'FunctionArn': 'arn:aws:lambda:{}:123456789012:function:testFunction'.format(_lambda_region),
'Runtime': 'python2.7',
'Role': 'test-iam-role',
'Handler': 'lambda_function.handler',
'Handler': 'lambda_function.lambda_handler',
"CodeSha256": hashlib.sha256(zip_content).hexdigest(),
"CodeSize": len(zip_content),
'Description': 'test lambda function',
@ -230,7 +248,7 @@ def test_create_function_from_zipfile():
FunctionName='testFunction',
Runtime='python2.7',
Role='test-iam-role',
Handler='lambda_function.handler',
Handler='lambda_function.lambda_handler',
Code={
'ZipFile': zip_content,
},
@ -247,10 +265,10 @@ def test_create_function_from_zipfile():
result.should.equal({
'FunctionName': 'testFunction',
'FunctionArn': 'arn:aws:lambda:123456789012:function:testFunction',
'FunctionArn': 'arn:aws:lambda:{}:123456789012:function:testFunction'.format(_lambda_region),
'Runtime': 'python2.7',
'Role': 'test-iam-role',
'Handler': 'lambda_function.handler',
'Handler': 'lambda_function.lambda_handler',
'CodeSize': len(zip_content),
'Description': 'test lambda function',
'Timeout': 3,
@ -281,7 +299,7 @@ def test_get_function():
FunctionName='testFunction',
Runtime='python2.7',
Role='test-iam-role',
Handler='lambda_function.handler',
Handler='lambda_function.lambda_handler',
Code={
'S3Bucket': 'test-bucket',
'S3Key': 'test.zip',
@ -301,16 +319,16 @@ def test_get_function():
result.should.equal({
"Code": {
"Location": "s3://lambda-functions.aws.amazon.com/test.zip",
"Location": "s3://awslambda-{0}-tasks.s3-{0}.amazonaws.com/test.zip".format(_lambda_region),
"RepositoryType": "S3"
},
"Configuration": {
"CodeSha256": hashlib.sha256(zip_content).hexdigest(),
"CodeSize": len(zip_content),
"Description": "test lambda function",
"FunctionArn": "arn:aws:lambda:123456789012:function:testFunction",
"FunctionArn": 'arn:aws:lambda:{}:123456789012:function:testFunction'.format(_lambda_region),
"FunctionName": "testFunction",
"Handler": "lambda_function.handler",
"Handler": "lambda_function.lambda_handler",
"MemorySize": 128,
"Role": "test-iam-role",
"Runtime": "python2.7",
@ -339,7 +357,7 @@ def test_delete_function():
FunctionName='testFunction',
Runtime='python2.7',
Role='test-iam-role',
Handler='lambda_function.handler',
Handler='lambda_function.lambda_handler',
Code={
'S3Bucket': 'test-bucket',
'S3Key': 'test.zip',
@ -383,7 +401,7 @@ def test_list_create_list_get_delete_list():
FunctionName='testFunction',
Runtime='python2.7',
Role='test-iam-role',
Handler='lambda_function.handler',
Handler='lambda_function.lambda_handler',
Code={
'S3Bucket': 'test-bucket',
'S3Key': 'test.zip',
@ -395,16 +413,16 @@ def test_list_create_list_get_delete_list():
)
expected_function_result = {
"Code": {
"Location": "s3://lambda-functions.aws.amazon.com/test.zip",
"Location": "s3://awslambda-{0}-tasks.s3-{0}.amazonaws.com/test.zip".format(_lambda_region),
"RepositoryType": "S3"
},
"Configuration": {
"CodeSha256": hashlib.sha256(zip_content).hexdigest(),
"CodeSize": len(zip_content),
"Description": "test lambda function",
"FunctionArn": "arn:aws:lambda:123456789012:function:testFunction",
"FunctionArn": 'arn:aws:lambda:{}:123456789012:function:testFunction'.format(_lambda_region),
"FunctionName": "testFunction",
"Handler": "lambda_function.handler",
"Handler": "lambda_function.lambda_handler",
"MemorySize": 128,
"Role": "test-iam-role",
"Runtime": "python2.7",
@ -437,12 +455,12 @@ def test_list_create_list_get_delete_list():
@mock_lambda
def test_invoke_lambda_error():
lambda_fx = """
def lambda_handler(event, context):
raise Exception('failsauce')
def lambda_handler(event, context):
raise Exception('failsauce')
"""
zip_output = io.BytesIO()
zip_file = zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED)
zip_file.writestr('lambda_function.zip', lambda_fx)
zip_file.writestr('lambda_function.py', lambda_fx)
zip_file.close()
zip_output.seek(0)
@ -605,13 +623,15 @@ def test_get_function_created_with_zipfile():
response['Configuration'].pop('LastModified')
response['ResponseMetadata']['HTTPStatusCode'].should.equal(200)
assert 'Code' not in response
assert len(response['Code']) == 2
assert response['Code']['RepositoryType'] == 'S3'
assert response['Code']['Location'].startswith('s3://awslambda-{0}-tasks.s3-{0}.amazonaws.com'.format(_lambda_region))
response['Configuration'].should.equal(
{
"CodeSha256": hashlib.sha256(zip_content).hexdigest(),
"CodeSize": len(zip_content),
"Description": "test lambda function",
"FunctionArn": "arn:aws:lambda:123456789012:function:testFunction",
"FunctionArn":'arn:aws:lambda:{}:123456789012:function:testFunction'.format(_lambda_region),
"FunctionName": "testFunction",
"Handler": "lambda_function.handler",
"MemorySize": 128,