catalin/moto
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Run black on moto & test directories.

Browse source
This commit is contained in:
Asher Foa 2019-10-31 08:44:26 -07:00
commit 96e5b1993d
507 changed files with 52541 additions and 47814 deletions
Download patch file Download diff file Expand all files Collapse all files

489
moto/awslambda/models.py
View file

@ -38,7 +38,7 @@ from moto.dynamodbstreams import dynamodbstreams_backends
logger = logging.getLogger(__name__)
ACCOUNT_ID = '123456789012'
ACCOUNT_ID = "123456789012"
try:
@ -47,20 +47,22 @@ except ImportError:
from backports.tempfile import TemporaryDirectory
_stderr_regex = re.compile(r'START|END|REPORT RequestId: .*')
_stderr_regex = re.compile(r"START|END|REPORT RequestId: .*")
_orig_adapter_send = requests.adapters.HTTPAdapter.send
docker_3 = docker.__version__[0] >= '3'
docker_3 = docker.__version__[0] >= "3"
def zip2tar(zip_bytes):
with TemporaryDirectory() as td:
tarname = os.path.join(td, 'data.tar')
timeshift = int((datetime.datetime.now() -
datetime.datetime.utcnow()).total_seconds())
with zipfile.ZipFile(io.BytesIO(zip_bytes), 'r') as zipf, \
tarfile.TarFile(tarname, 'w') as tarf:
tarname = os.path.join(td, "data.tar")
timeshift = int(
(datetime.datetime.now() - datetime.datetime.utcnow()).total_seconds()
)
with zipfile.ZipFile(io.BytesIO(zip_bytes), "r") as zipf, tarfile.TarFile(
tarname, "w"
) as tarf:
for zipinfo in zipf.infolist():
if zipinfo.filename[-1] == '/': # is_dir() is py3.6+
if zipinfo.filename[-1] == "/": # is_dir() is py3.6+
continue
tarinfo = tarfile.TarInfo(name=zipinfo.filename)
@ -69,7 +71,7 @@ def zip2tar(zip_bytes):
infile = zipf.open(zipinfo.filename)
tarf.addfile(tarinfo, infile)
with open(tarname, 'rb') as f:
with open(tarname, "rb") as f:
tar_data = f.read()
return tar_data
@ -83,7 +85,9 @@ class _VolumeRefCount:
class _DockerDataVolumeContext:
_data_vol_map = defaultdict(lambda: _VolumeRefCount(0, None)) # {sha256: _VolumeRefCount}
_data_vol_map = defaultdict(
lambda: _VolumeRefCount(0, None)
) # {sha256: _VolumeRefCount}
_lock = threading.Lock()
def __init__(self, lambda_func):
@ -109,15 +113,19 @@ class _DockerDataVolumeContext:
return self
# It doesn't exist so we need to create it
self._vol_ref.volume = self._lambda_func.docker_client.volumes.create(self._lambda_func.code_sha_256)
self._vol_ref.volume = self._lambda_func.docker_client.volumes.create(
self._lambda_func.code_sha_256
)
if docker_3:
volumes = {self.name: {'bind': '/tmp/data', 'mode': 'rw'}}
volumes = {self.name: {"bind": "/tmp/data", "mode": "rw"}}
else:
volumes = {self.name: '/tmp/data'}
container = self._lambda_func.docker_client.containers.run('alpine', 'sleep 100', volumes=volumes, detach=True)
volumes = {self.name: "/tmp/data"}
container = self._lambda_func.docker_client.containers.run(
"alpine", "sleep 100", volumes=volumes, detach=True
)
try:
tar_bytes = zip2tar(self._lambda_func.code_bytes)
container.put_archive('/tmp/data', tar_bytes)
container.put_archive("/tmp/data", tar_bytes)
finally:
container.remove(force=True)
@ -140,13 +148,13 @@ class LambdaFunction(BaseModel):
def __init__(self, spec, region, validate_s3=True, version=1):
# required
self.region = region
self.code = spec['Code']
self.function_name = spec['FunctionName']
self.handler = spec['Handler']
self.role = spec['Role']
self.run_time = spec['Runtime']
self.code = spec["Code"]
self.function_name = spec["FunctionName"]
self.handler = spec["Handler"]
self.role = spec["Role"]
self.run_time = spec["Runtime"]
self.logs_backend = logs_backends[self.region]
self.environment_vars = spec.get('Environment', {}).get('Variables', {})
self.environment_vars = spec.get("Environment", {}).get("Variables", {})
self.docker_client = docker.from_env()
self.policy = ""
@ -161,77 +169,82 @@ class LambdaFunction(BaseModel):
if isinstance(adapter, requests.adapters.HTTPAdapter):
adapter.send = functools.partial(_orig_adapter_send, adapter)
return adapter
self.docker_client.api.get_adapter = replace_adapter_send
# optional
self.description = spec.get('Description', '')
self.memory_size = spec.get('MemorySize', 128)
self.publish = spec.get('Publish', False) # this is ignored currently
self.timeout = spec.get('Timeout', 3)
self.description = spec.get("Description", "")
self.memory_size = spec.get("MemorySize", 128)
self.publish = spec.get("Publish", False) # this is ignored currently
self.timeout = spec.get("Timeout", 3)
self.logs_group_name = '/aws/lambda/{}'.format(self.function_name)
self.logs_group_name = "/aws/lambda/{}".format(self.function_name)
self.logs_backend.ensure_log_group(self.logs_group_name, [])
# this isn't finished yet. it needs to find out the VpcId value
self._vpc_config = spec.get(
'VpcConfig', {'SubnetIds': [], 'SecurityGroupIds': []})
"VpcConfig", {"SubnetIds": [], "SecurityGroupIds": []}
)
# auto-generated
self.version = version
self.last_modified = datetime.datetime.utcnow().strftime(
'%Y-%m-%d %H:%M:%S')
self.last_modified = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
if 'ZipFile' in self.code:
if "ZipFile" in self.code:
# more hackery to handle unicode/bytes/str in python3 and python2 -
# argh!
try:
to_unzip_code = base64.b64decode(
bytes(self.code['ZipFile'], 'utf-8'))
to_unzip_code = base64.b64decode(bytes(self.code["ZipFile"], "utf-8"))
except Exception:
to_unzip_code = base64.b64decode(self.code['ZipFile'])
to_unzip_code = base64.b64decode(self.code["ZipFile"])
self.code_bytes = to_unzip_code
self.code_size = len(to_unzip_code)
self.code_sha_256 = hashlib.sha256(to_unzip_code).hexdigest()
# TODO: we should be putting this in a lambda bucket
self.code['UUID'] = str(uuid.uuid4())
self.code['S3Key'] = '{}-{}'.format(self.function_name, self.code['UUID'])
self.code["UUID"] = str(uuid.uuid4())
self.code["S3Key"] = "{}-{}".format(self.function_name, self.code["UUID"])
else:
# validate s3 bucket and key
key = None
try:
# FIXME: does not validate bucket region
key = s3_backend.get_key(
self.code['S3Bucket'], self.code['S3Key'])
key = s3_backend.get_key(self.code["S3Bucket"], self.code["S3Key"])
except MissingBucket:
if do_validate_s3():
raise ValueError(
"InvalidParameterValueException",
"Error occurred while GetObject. S3 Error Code: NoSuchBucket. S3 Error Message: The specified bucket does not exist")
"Error occurred while GetObject. S3 Error Code: NoSuchBucket. S3 Error Message: The specified bucket does not exist",
)
except MissingKey:
if do_validate_s3():
raise ValueError(
"InvalidParameterValueException",
"Error occurred while GetObject. S3 Error Code: NoSuchKey. S3 Error Message: The specified key does not exist.")
"Error occurred while GetObject. S3 Error Code: NoSuchKey. S3 Error Message: The specified key does not exist.",
)
if key:
self.code_bytes = key.value
self.code_size = key.size
self.code_sha_256 = hashlib.sha256(key.value).hexdigest()
self.function_arn = make_function_arn(self.region, ACCOUNT_ID, self.function_name)
self.function_arn = make_function_arn(
self.region, ACCOUNT_ID, self.function_name
)
self.tags = dict()
def set_version(self, version):
self.function_arn = make_function_ver_arn(self.region, ACCOUNT_ID, self.function_name, version)
self.function_arn = make_function_ver_arn(
self.region, ACCOUNT_ID, self.function_name, version
)
self.version = version
self.last_modified = datetime.datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S')
self.last_modified = datetime.datetime.utcnow().strftime("%Y-%m-%d %H:%M:%S")
@property
def vpc_config(self):
config = self._vpc_config.copy()
if config['SecurityGroupIds']:
if config["SecurityGroupIds"]:
config.update({"VpcId": "vpc-123abc"})
return config
@ -260,17 +273,17 @@ class LambdaFunction(BaseModel):
}
if self.environment_vars:
config['Environment'] = {
'Variables': self.environment_vars
}
config["Environment"] = {"Variables": self.environment_vars}
return config
def get_code(self):
return {
"Code": {
"Location": "s3://awslambda-{0}-tasks.s3-{0}.amazonaws.com/{1}".format(self.region, self.code['S3Key']),
"RepositoryType": "S3"
"Location": "s3://awslambda-{0}-tasks.s3-{0}.amazonaws.com/{1}".format(
self.region, self.code["S3Key"]
),
"RepositoryType": "S3",
},
"Configuration": self.get_configuration(),
}
@ -295,43 +308,48 @@ class LambdaFunction(BaseModel):
return self.get_configuration()
def update_function_code(self, updated_spec):
if 'DryRun' in updated_spec and updated_spec['DryRun']:
if "DryRun" in updated_spec and updated_spec["DryRun"]:
return self.get_configuration()
if 'ZipFile' in updated_spec:
self.code['ZipFile'] = updated_spec['ZipFile']
if "ZipFile" in updated_spec:
self.code["ZipFile"] = updated_spec["ZipFile"]
# using the "hackery" from __init__ because it seems to work
# TODOs and FIXMEs included, because they'll need to be fixed
# in both places now
try:
to_unzip_code = base64.b64decode(
bytes(updated_spec['ZipFile'], 'utf-8'))
bytes(updated_spec["ZipFile"], "utf-8")
)
except Exception:
to_unzip_code = base64.b64decode(updated_spec['ZipFile'])
to_unzip_code = base64.b64decode(updated_spec["ZipFile"])
self.code_bytes = to_unzip_code
self.code_size = len(to_unzip_code)
self.code_sha_256 = hashlib.sha256(to_unzip_code).hexdigest()
# TODO: we should be putting this in a lambda bucket
self.code['UUID'] = str(uuid.uuid4())
self.code['S3Key'] = '{}-{}'.format(self.function_name, self.code['UUID'])
elif 'S3Bucket' in updated_spec and 'S3Key' in updated_spec:
self.code["UUID"] = str(uuid.uuid4())
self.code["S3Key"] = "{}-{}".format(self.function_name, self.code["UUID"])
elif "S3Bucket" in updated_spec and "S3Key" in updated_spec:
key = None
try:
# FIXME: does not validate bucket region
key = s3_backend.get_key(updated_spec['S3Bucket'], updated_spec['S3Key'])
key = s3_backend.get_key(
updated_spec["S3Bucket"], updated_spec["S3Key"]
)
except MissingBucket:
if do_validate_s3():
raise ValueError(
"InvalidParameterValueException",
"Error occurred while GetObject. S3 Error Code: NoSuchBucket. S3 Error Message: The specified bucket does not exist")
"Error occurred while GetObject. S3 Error Code: NoSuchBucket. S3 Error Message: The specified bucket does not exist",
)
except MissingKey:
if do_validate_s3():
raise ValueError(
"InvalidParameterValueException",
"Error occurred while GetObject. S3 Error Code: NoSuchKey. S3 Error Message: The specified key does not exist.")
"Error occurred while GetObject. S3 Error Code: NoSuchKey. S3 Error Message: The specified key does not exist.",
)
if key:
self.code_bytes = key.value
self.code_size = key.size
@ -342,7 +360,7 @@ class LambdaFunction(BaseModel):
@staticmethod
def convert(s):
try:
return str(s, encoding='utf-8')
return str(s, encoding="utf-8")
except Exception:
return s
@ -370,12 +388,21 @@ class LambdaFunction(BaseModel):
container = output = exit_code = None
with _DockerDataVolumeContext(self) as data_vol:
try:
run_kwargs = dict(links={'motoserver': 'motoserver'}) if settings.TEST_SERVER_MODE else {}
run_kwargs = (
dict(links={"motoserver": "motoserver"})
if settings.TEST_SERVER_MODE
else {}
)
container = self.docker_client.containers.run(
"lambci/lambda:{}".format(self.run_time),
[self.handler, json.dumps(event)], remove=False,
[self.handler, json.dumps(event)],
remove=False,
mem_limit="{}m".format(self.memory_size),
volumes=["{}:/var/task".format(data_vol.name)], environment=env_vars, detach=True, **run_kwargs)
volumes=["{}:/var/task".format(data_vol.name)],
environment=env_vars,
detach=True,
**run_kwargs
)
finally:
if container:
try:
@ -386,32 +413,43 @@ class LambdaFunction(BaseModel):
container.kill()
else:
if docker_3:
exit_code = exit_code['StatusCode']
exit_code = exit_code["StatusCode"]
output = container.logs(stdout=False, stderr=True)
output += container.logs(stdout=True, stderr=False)
container.remove()
output = output.decode('utf-8')
output = output.decode("utf-8")
# Send output to "logs" backend
invoke_id = uuid.uuid4().hex
log_stream_name = "{date.year}/{date.month:02d}/{date.day:02d}/[{version}]{invoke_id}".format(
date=datetime.datetime.utcnow(), version=self.version, invoke_id=invoke_id
date=datetime.datetime.utcnow(),
version=self.version,
invoke_id=invoke_id,
)
self.logs_backend.create_log_stream(self.logs_group_name, log_stream_name)
log_events = [{'timestamp': unix_time_millis(), "message": line}
for line in output.splitlines()]
self.logs_backend.put_log_events(self.logs_group_name, log_stream_name, log_events, None)
log_events = [
{"timestamp": unix_time_millis(), "message": line}
for line in output.splitlines()
]
self.logs_backend.put_log_events(
self.logs_group_name, log_stream_name, log_events, None
)
if exit_code != 0:
raise Exception(
'lambda invoke failed output: {}'.format(output))
raise Exception("lambda invoke failed output: {}".format(output))
# strip out RequestId lines
output = os.linesep.join([line for line in self.convert(output).splitlines() if not _stderr_regex.match(line)])
output = os.linesep.join(
[
line
for line in self.convert(output).splitlines()
if not _stderr_regex.match(line)
]
)
return output, False
except BaseException as e:
traceback.print_exc()
@ -426,31 +464,34 @@ class LambdaFunction(BaseModel):
# Get the invocation type:
res, errored = self._invoke_lambda(code=self.code, event=body)
if request_headers.get("x-amz-invocation-type") == "RequestResponse":
encoded = base64.b64encode(res.encode('utf-8'))
response_headers["x-amz-log-result"] = encoded.decode('utf-8')
payload['result'] = response_headers["x-amz-log-result"]
result = res.encode('utf-8')
encoded = base64.b64encode(res.encode("utf-8"))
response_headers["x-amz-log-result"] = encoded.decode("utf-8")
payload["result"] = response_headers["x-amz-log-result"]
result = res.encode("utf-8")
else:
result = json.dumps(payload)
if errored:
response_headers['x-amz-function-error'] = "Handled"
response_headers["x-amz-function-error"] = "Handled"
return result
@classmethod
def create_from_cloudformation_json(cls, resource_name, cloudformation_json,
region_name):
properties = cloudformation_json['Properties']
def create_from_cloudformation_json(
cls, resource_name, cloudformation_json, region_name
):
properties = cloudformation_json["Properties"]
# required
spec = {
'Code': properties['Code'],
'FunctionName': resource_name,
'Handler': properties['Handler'],
'Role': properties['Role'],
'Runtime': properties['Runtime'],
"Code": properties["Code"],
"FunctionName": resource_name,
"Handler": properties["Handler"],
"Role": properties["Role"],
"Runtime": properties["Runtime"],
}
optional_properties = 'Description MemorySize Publish Timeout VpcConfig Environment'.split()
optional_properties = (
"Description MemorySize Publish Timeout VpcConfig Environment".split()
)
# NOTE: Not doing `properties.get(k, DEFAULT)` to avoid duplicating the
# default logic
for prop in optional_properties:
@ -460,27 +501,27 @@ class LambdaFunction(BaseModel):
# when ZipFile is present in CloudFormation, per the official docs,
# the code it's a plaintext code snippet up to 4096 bytes.
# this snippet converts this plaintext code to a proper base64-encoded ZIP file.
if 'ZipFile' in properties['Code']:
spec['Code']['ZipFile'] = base64.b64encode(
cls._create_zipfile_from_plaintext_code(
spec['Code']['ZipFile']))
if "ZipFile" in properties["Code"]:
spec["Code"]["ZipFile"] = base64.b64encode(
cls._create_zipfile_from_plaintext_code(spec["Code"]["ZipFile"])
)
backend = lambda_backends[region_name]
fn = backend.create_function(spec)
return fn
def get_cfn_attribute(self, attribute_name):
from moto.cloudformation.exceptions import \
UnformattedGetAttTemplateException
if attribute_name == 'Arn':
from moto.cloudformation.exceptions import UnformattedGetAttTemplateException
if attribute_name == "Arn":
return make_function_arn(self.region, ACCOUNT_ID, self.function_name)
raise UnformattedGetAttTemplateException()
@staticmethod
def _create_zipfile_from_plaintext_code(code):
zip_output = io.BytesIO()
zip_file = zipfile.ZipFile(zip_output, 'w', zipfile.ZIP_DEFLATED)
zip_file.writestr('lambda_function.zip', code)
zip_file = zipfile.ZipFile(zip_output, "w", zipfile.ZIP_DEFLATED)
zip_file.writestr("lambda_function.zip", code)
zip_file.close()
zip_output.seek(0)
return zip_output.read()
@ -489,61 +530,66 @@ class LambdaFunction(BaseModel):
class EventSourceMapping(BaseModel):
def __init__(self, spec):
# required
self.function_arn = spec['FunctionArn']
self.event_source_arn = spec['EventSourceArn']
self.function_arn = spec["FunctionArn"]
self.event_source_arn = spec["EventSourceArn"]
self.uuid = str(uuid.uuid4())
self.last_modified = time.mktime(datetime.datetime.utcnow().timetuple())
# BatchSize service default/max mapping
batch_size_map = {
'kinesis': (100, 10000),
'dynamodb': (100, 1000),
'sqs': (10, 10),
"kinesis": (100, 10000),
"dynamodb": (100, 1000),
"sqs": (10, 10),
}
source_type = self.event_source_arn.split(":")[2].lower()
batch_size_entry = batch_size_map.get(source_type)
if batch_size_entry:
# Use service default if not provided
batch_size = int(spec.get('BatchSize', batch_size_entry[0]))
batch_size = int(spec.get("BatchSize", batch_size_entry[0]))
if batch_size > batch_size_entry[1]:
raise ValueError("InvalidParameterValueException",
"BatchSize {} exceeds the max of {}".format(batch_size, batch_size_entry[1]))
raise ValueError(
"InvalidParameterValueException",
"BatchSize {} exceeds the max of {}".format(
batch_size, batch_size_entry[1]
),
)
else:
self.batch_size = batch_size
else:
raise ValueError("InvalidParameterValueException",
"Unsupported event source type")
raise ValueError(
"InvalidParameterValueException", "Unsupported event source type"
)
# optional
self.starting_position = spec.get('StartingPosition', 'TRIM_HORIZON')
self.enabled = spec.get('Enabled', True)
self.starting_position_timestamp = spec.get('StartingPositionTimestamp',
None)
self.starting_position = spec.get("StartingPosition", "TRIM_HORIZON")
self.enabled = spec.get("Enabled", True)
self.starting_position_timestamp = spec.get("StartingPositionTimestamp", None)
def get_configuration(self):
return {
'UUID': self.uuid,
'BatchSize': self.batch_size,
'EventSourceArn': self.event_source_arn,
'FunctionArn': self.function_arn,
'LastModified': self.last_modified,
'LastProcessingResult': '',
'State': 'Enabled' if self.enabled else 'Disabled',
'StateTransitionReason': 'User initiated'
"UUID": self.uuid,
"BatchSize": self.batch_size,
"EventSourceArn": self.event_source_arn,
"FunctionArn": self.function_arn,
"LastModified": self.last_modified,
"LastProcessingResult": "",
"State": "Enabled" if self.enabled else "Disabled",
"StateTransitionReason": "User initiated",
}
@classmethod
def create_from_cloudformation_json(cls, resource_name, cloudformation_json,
region_name):
properties = cloudformation_json['Properties']
func = lambda_backends[region_name].get_function(properties['FunctionName'])
def create_from_cloudformation_json(
cls, resource_name, cloudformation_json, region_name
):
properties = cloudformation_json["Properties"]
func = lambda_backends[region_name].get_function(properties["FunctionName"])
spec = {
'FunctionArn': func.function_arn,
'EventSourceArn': properties['EventSourceArn'],
'StartingPosition': properties['StartingPosition'],
'BatchSize': properties.get('BatchSize', 100)
"FunctionArn": func.function_arn,
"EventSourceArn": properties["EventSourceArn"],
"StartingPosition": properties["StartingPosition"],
"BatchSize": properties.get("BatchSize", 100),
}
optional_properties = 'BatchSize Enabled StartingPositionTimestamp'.split()
optional_properties = "BatchSize Enabled StartingPositionTimestamp".split()
for prop in optional_properties:
if prop in properties:
spec[prop] = properties[prop]
@ -552,20 +598,19 @@ class EventSourceMapping(BaseModel):
class LambdaVersion(BaseModel):
def __init__(self, spec):
self.version = spec['Version']
self.version = spec["Version"]
def __repr__(self):
return str(self.logical_resource_id)
@classmethod
def create_from_cloudformation_json(cls, resource_name, cloudformation_json,
region_name):
properties = cloudformation_json['Properties']
function_name = properties['FunctionName']
def create_from_cloudformation_json(
cls, resource_name, cloudformation_json, region_name
):
properties = cloudformation_json["Properties"]
function_name = properties["FunctionName"]
func = lambda_backends[region_name].publish_function(function_name)
spec = {
'Version': func.version
}
spec = {"Version": func.version}
return LambdaVersion(spec)
@ -576,18 +621,18 @@ class LambdaStorage(object):
self._arns = weakref.WeakValueDictionary()
def _get_latest(self, name):
return self._functions[name]['latest']
return self._functions[name]["latest"]
def _get_version(self, name, version):
index = version - 1
try:
return self._functions[name]['versions'][index]
return self._functions[name]["versions"][index]
except IndexError:
return None
def _get_alias(self, name, alias):
return self._functions[name]['alias'].get(alias, None)
return self._functions[name]["alias"].get(alias, None)
def get_function(self, name, qualifier=None):
if name not in self._functions:
@ -599,15 +644,15 @@ class LambdaStorage(object):
try:
return self._get_version(name, int(qualifier))
except ValueError:
return self._functions[name]['latest']
return self._functions[name]["latest"]
def list_versions_by_function(self, name):
if name not in self._functions:
return None
latest = copy.copy(self._functions[name]['latest'])
latest.function_arn += ':$LATEST'
return [latest] + self._functions[name]['versions']
latest = copy.copy(self._functions[name]["latest"])
latest.function_arn += ":$LATEST"
return [latest] + self._functions[name]["versions"]
def get_arn(self, arn):
return self._arns.get(arn, None)
@ -621,12 +666,12 @@ class LambdaStorage(object):
:type fn: LambdaFunction
"""
if fn.function_name in self._functions:
self._functions[fn.function_name]['latest'] = fn
self._functions[fn.function_name]["latest"] = fn
else:
self._functions[fn.function_name] = {
'latest': fn,
'versions': [],
'alias': weakref.WeakValueDictionary()
"latest": fn,
"versions": [],
"alias": weakref.WeakValueDictionary(),
}
self._arns[fn.function_arn] = fn
@ -634,14 +679,14 @@ class LambdaStorage(object):
def publish_function(self, name):
if name not in self._functions:
return None
if not self._functions[name]['latest']:
if not self._functions[name]["latest"]:
return None
new_version = len(self._functions[name]['versions']) + 1
fn = copy.copy(self._functions[name]['latest'])
new_version = len(self._functions[name]["versions"]) + 1
fn = copy.copy(self._functions[name]["latest"])
fn.set_version(new_version)
self._functions[name]['versions'].append(fn)
self._functions[name]["versions"].append(fn)
self._arns[fn.function_arn] = fn
return fn
@ -651,21 +696,24 @@ class LambdaStorage(object):
name = function.function_name
if not qualifier:
# Something is still reffing this so delete all arns
latest = self._functions[name]['latest'].function_arn
latest = self._functions[name]["latest"].function_arn
del self._arns[latest]
for fn in self._functions[name]['versions']:
for fn in self._functions[name]["versions"]:
del self._arns[fn.function_arn]
del self._functions[name]
return True
elif qualifier == '$LATEST':
self._functions[name]['latest'] = None
elif qualifier == "$LATEST":
self._functions[name]["latest"] = None
# If theres no functions left
if not self._functions[name]['versions'] and not self._functions[name]['latest']:
if (
not self._functions[name]["versions"]
and not self._functions[name]["latest"]
):
del self._functions[name]
return True
@ -673,10 +721,13 @@ class LambdaStorage(object):
else:
fn = self.get_function(name, qualifier)
if fn:
self._functions[name]['versions'].remove(fn)
self._functions[name]["versions"].remove(fn)
# If theres no functions left
if not self._functions[name]['versions'] and not self._functions[name]['latest']:
if (
not self._functions[name]["versions"]
and not self._functions[name]["latest"]
):
del self._functions[name]
return True
@ -687,10 +738,10 @@ class LambdaStorage(object):
result = []
for function_group in self._functions.values():
if function_group['latest'] is not None:
result.append(function_group['latest'])
if function_group["latest"] is not None:
result.append(function_group["latest"])
result.extend(function_group['versions'])
result.extend(function_group["versions"])
return result
@ -707,44 +758,47 @@ class LambdaBackend(BaseBackend):
self.__init__(region_name)
def create_function(self, spec):
function_name = spec.get('FunctionName', None)
function_name = spec.get("FunctionName", None)
if function_name is None:
raise RESTError('InvalidParameterValueException', 'Missing FunctionName')
raise RESTError("InvalidParameterValueException", "Missing FunctionName")
fn = LambdaFunction(spec, self.region_name, version='$LATEST')
fn = LambdaFunction(spec, self.region_name, version="$LATEST")
self._lambdas.put_function(fn)
if spec.get('Publish'):
if spec.get("Publish"):
ver = self.publish_function(function_name)
fn.version = ver.version
return fn
def create_event_source_mapping(self, spec):
required = [
'EventSourceArn',
'FunctionName',
]
required = ["EventSourceArn", "FunctionName"]
for param in required:
if not spec.get(param):
raise RESTError('InvalidParameterValueException', 'Missing {}'.format(param))
raise RESTError(
"InvalidParameterValueException", "Missing {}".format(param)
)
# Validate function name
func = self._lambdas.get_function_by_name_or_arn(spec.pop('FunctionName', ''))
func = self._lambdas.get_function_by_name_or_arn(spec.pop("FunctionName", ""))
if not func:
raise RESTError('ResourceNotFoundException', 'Invalid FunctionName')
raise RESTError("ResourceNotFoundException", "Invalid FunctionName")
# Validate queue
for queue in sqs_backends[self.region_name].queues.values():
if queue.queue_arn == spec['EventSourceArn']:
if queue.lambda_event_source_mappings.get('func.function_arn'):
if queue.queue_arn == spec["EventSourceArn"]:
if queue.lambda_event_source_mappings.get("func.function_arn"):
# TODO: Correct exception?
raise RESTError('ResourceConflictException', 'The resource already exists.')
raise RESTError(
"ResourceConflictException", "The resource already exists."
)
if queue.fifo_queue:
raise RESTError('InvalidParameterValueException',
'{} is FIFO'.format(queue.queue_arn))
raise RESTError(
"InvalidParameterValueException",
"{} is FIFO".format(queue.queue_arn),
)
else:
spec.update({'FunctionArn': func.function_arn})
spec.update({"FunctionArn": func.function_arn})
esm = EventSourceMapping(spec)
self._event_source_mappings[esm.uuid] = esm
@ -752,16 +806,18 @@ class LambdaBackend(BaseBackend):
queue.lambda_event_source_mappings[esm.function_arn] = esm
return esm
for stream in json.loads(dynamodbstreams_backends[self.region_name].list_streams())['Streams']:
if stream['StreamArn'] == spec['EventSourceArn']:
spec.update({'FunctionArn': func.function_arn})
for stream in json.loads(
dynamodbstreams_backends[self.region_name].list_streams()
)["Streams"]:
if stream["StreamArn"] == spec["EventSourceArn"]:
spec.update({"FunctionArn": func.function_arn})
esm = EventSourceMapping(spec)
self._event_source_mappings[esm.uuid] = esm
table_name = stream['TableName']
table_name = stream["TableName"]
table = dynamodb_backends2[self.region_name].get_table(table_name)
table.lambda_event_source_mappings[esm.function_arn] = esm
return esm
raise RESTError('ResourceNotFoundException', 'Invalid EventSourceArn')
raise RESTError("ResourceNotFoundException", "Invalid EventSourceArn")
def publish_function(self, function_name):
return self._lambdas.publish_function(function_name)
@ -781,13 +837,15 @@ class LambdaBackend(BaseBackend):
def update_event_source_mapping(self, uuid, spec):
esm = self.get_event_source_mapping(uuid)
if esm:
if spec.get('FunctionName'):
func = self._lambdas.get_function_by_name_or_arn(spec.get('FunctionName'))
if spec.get("FunctionName"):
func = self._lambdas.get_function_by_name_or_arn(
spec.get("FunctionName")
)
esm.function_arn = func.function_arn
if 'BatchSize' in spec:
esm.batch_size = spec['BatchSize']
if 'Enabled' in spec:
esm.enabled = spec['Enabled']
if "BatchSize" in spec:
esm.batch_size = spec["BatchSize"]
if "Enabled" in spec:
esm.enabled = spec["Enabled"]
return esm
return False
@ -828,13 +886,13 @@ class LambdaBackend(BaseBackend):
"ApproximateReceiveCount": "1",
"SentTimestamp": "1545082649183",
"SenderId": "AIDAIENQZJOLO23YVJ4VO",
"ApproximateFirstReceiveTimestamp": "1545082649185"
"ApproximateFirstReceiveTimestamp": "1545082649185",
},
"messageAttributes": {},
"md5OfBody": "098f6bcd4621d373cade4e832627b4f6",
"eventSource": "aws:sqs",
"eventSourceARN": queue_arn,
"awsRegion": self.region_name
"awsRegion": self.region_name,
}
]
}
@ -842,7 +900,7 @@ class LambdaBackend(BaseBackend):
request_headers = {}
response_headers = {}
func.invoke(json.dumps(event), request_headers, response_headers)
return 'x-amz-function-error' not in response_headers
return "x-amz-function-error" not in response_headers
def send_sns_message(self, function_name, message, subject=None, qualifier=None):
event = {
@ -859,37 +917,35 @@ class LambdaBackend(BaseBackend):
"MessageId": "95df01b4-ee98-5cb9-9903-4c221d41eb5e",
"Message": message,
"MessageAttributes": {
"Test": {
"Type": "String",
"Value": "TestString"
},
"TestBinary": {
"Type": "Binary",
"Value": "TestBinary"
}
"Test": {"Type": "String", "Value": "TestString"},
"TestBinary": {"Type": "Binary", "Value": "TestBinary"},
},
"Type": "Notification",
"UnsubscribeUrl": "EXAMPLE",
"TopicArn": "arn:aws:sns:EXAMPLE",
"Subject": subject or "TestInvoke"
}
"Subject": subject or "TestInvoke",
},
}
]
}
func = self._lambdas.get_function(function_name, qualifier)
func.invoke(json.dumps(event), {}, {})
def send_dynamodb_items(self, function_arn, items, source):
event = {'Records': [
{
'eventID': item.to_json()['eventID'],
'eventName': 'INSERT',
'eventVersion': item.to_json()['eventVersion'],
'eventSource': item.to_json()['eventSource'],
'awsRegion': self.region_name,
'dynamodb': item.to_json()['dynamodb'],
'eventSourceARN': source} for item in items]}
event = {
"Records": [
{
"eventID": item.to_json()["eventID"],
"eventName": "INSERT",
"eventVersion": item.to_json()["eventVersion"],
"eventSource": item.to_json()["eventSource"],
"awsRegion": self.region_name,
"dynamodb": item.to_json()["dynamodb"],
"eventSourceARN": source,
}
for item in items
]
}
func = self._lambdas.get_arn(function_arn)
func.invoke(json.dumps(event), {}, {})
@ -921,12 +977,13 @@ class LambdaBackend(BaseBackend):
def do_validate_s3():
return os.environ.get('VALIDATE_LAMBDA_S3', '') in ['', '1', 'true']
return os.environ.get("VALIDATE_LAMBDA_S3", "") in ["", "1", "true"]
# Handle us forgotten regions, unless Lambda truly only runs out of US and
lambda_backends = {_region.name: LambdaBackend(_region.name)
for _region in boto.awslambda.regions()}
lambda_backends = {
_region.name: LambdaBackend(_region.name) for _region in boto.awslambda.regions()
}
lambda_backends['ap-southeast-2'] = LambdaBackend('ap-southeast-2')
lambda_backends['us-gov-west-1'] = LambdaBackend('us-gov-west-1')
lambda_backends["ap-southeast-2"] = LambdaBackend("ap-southeast-2")
lambda_backends["us-gov-west-1"] = LambdaBackend("us-gov-west-1")