Merge branch 'master' into bugfix/1874

This commit is contained in:
Bert Blommers 2019-10-09 08:33:53 +01:00
commit 1fb844972f
13 changed files with 1135 additions and 116 deletions

View file

@ -332,15 +332,16 @@ class ApiKey(BaseModel, dict):
class UsagePlan(BaseModel, dict):
def __init__(self, name=None, description=None, apiStages=[],
throttle=None, quota=None):
def __init__(self, name=None, description=None, apiStages=None,
throttle=None, quota=None, tags=None):
super(UsagePlan, self).__init__()
self['id'] = create_id()
self['name'] = name
self['description'] = description
self['apiStages'] = apiStages
self['apiStages'] = apiStages if apiStages else []
self['throttle'] = throttle
self['quota'] = quota
self['tags'] = tags
class UsagePlanKey(BaseModel, dict):

View file

@ -235,7 +235,33 @@ class Item(BaseModel):
value = re.sub(r'{0}\b'.format(k), v, value)
if action == "REMOVE":
self.attrs.pop(value, None)
key = value
if '.' not in key:
self.attrs.pop(value, None)
else:
# Handle nested dict updates
key_parts = key.split('.')
attr = key_parts.pop(0)
if attr not in self.attrs:
raise ValueError
last_val = self.attrs[attr].value
for key_part in key_parts[:-1]:
# Hack but it'll do, traverses into a dict
last_val_type = list(last_val.keys())
if last_val_type and last_val_type[0] == 'M':
last_val = last_val['M']
if key_part not in last_val:
last_val[key_part] = {'M': {}}
last_val = last_val[key_part]
last_val_type = list(last_val.keys())
if last_val_type and last_val_type[0] == 'M':
last_val['M'].pop(key_parts[-1], None)
else:
last_val.pop(key_parts[-1], None)
elif action == 'SET':
key, value = value.split("=", 1)
key = key.strip()
@ -1119,12 +1145,23 @@ class DynamoDBBackend(BaseBackend):
item.update_with_attribute_updates(attribute_updates)
return item
def delete_item(self, table_name, keys):
def delete_item(self, table_name, key, expression_attribute_names=None, expression_attribute_values=None,
condition_expression=None):
table = self.get_table(table_name)
if not table:
return None
hash_key, range_key = self.get_keys_value(table, keys)
return table.delete_item(hash_key, range_key)
hash_value, range_value = self.get_keys_value(table, key)
item = table.get_item(hash_value, range_value)
condition_op = get_filter_expression(
condition_expression,
expression_attribute_names,
expression_attribute_values)
if not condition_op.expr(item):
raise ValueError('The conditional request failed')
return table.delete_item(hash_value, range_value)
def update_ttl(self, table_name, ttl_spec):
table = self.tables.get(table_name)

View file

@ -582,7 +582,7 @@ class DynamoHandler(BaseResponse):
def delete_item(self):
name = self.body['TableName']
keys = self.body['Key']
key = self.body['Key']
return_values = self.body.get('ReturnValues', 'NONE')
if return_values not in ('ALL_OLD', 'NONE'):
er = 'com.amazonaws.dynamodb.v20111205#ValidationException'
@ -593,7 +593,21 @@ class DynamoHandler(BaseResponse):
er = 'com.amazonaws.dynamodb.v20120810#ConditionalCheckFailedException'
return self.error(er, 'A condition specified in the operation could not be evaluated.')
item = self.dynamodb_backend.delete_item(name, keys)
# Attempt to parse simple ConditionExpressions into an Expected
# expression
condition_expression = self.body.get('ConditionExpression')
expression_attribute_names = self.body.get('ExpressionAttributeNames', {})
expression_attribute_values = self.body.get('ExpressionAttributeValues', {})
try:
item = self.dynamodb_backend.delete_item(
name, key, expression_attribute_names, expression_attribute_values,
condition_expression
)
except ValueError:
er = 'com.amazonaws.dynamodb.v20111205#ConditionalCheckFailedException'
return self.error(er, 'A condition specified in the operation could not be evaluated.')
if item and return_values == 'ALL_OLD':
item_dict = item.to_json()
else:

View file

@ -329,7 +329,8 @@ class FakeGrant(BaseModel):
class FakeAcl(BaseModel):
def __init__(self, grants=[]):
def __init__(self, grants=None):
grants = grants or []
self.grants = grants
@property
@ -396,7 +397,7 @@ class FakeTag(BaseModel):
class LifecycleFilter(BaseModel):
def __init__(self, prefix=None, tag=None, and_filter=None):
self.prefix = prefix or ''
self.prefix = prefix
self.tag = tag
self.and_filter = and_filter
@ -404,7 +405,7 @@ class LifecycleFilter(BaseModel):
class LifecycleAndFilter(BaseModel):
def __init__(self, prefix=None, tags=None):
self.prefix = prefix or ''
self.prefix = prefix
self.tags = tags
@ -478,6 +479,8 @@ class FakeBucket(BaseModel):
self.logging = {}
self.notification_configuration = None
self.accelerate_configuration = None
self.payer = 'BucketOwner'
self.creation_date = datetime.datetime.utcnow()
@property
def location(self):
@ -494,6 +497,11 @@ class FakeBucket(BaseModel):
expiration = rule.get('Expiration')
transition = rule.get('Transition')
try:
top_level_prefix = rule['Prefix'] or '' # If it's `None` the set to the empty string
except KeyError:
top_level_prefix = None
nve_noncurrent_days = None
if rule.get('NoncurrentVersionExpiration') is not None:
if rule["NoncurrentVersionExpiration"].get('NoncurrentDays') is None:
@ -528,13 +536,22 @@ class FakeBucket(BaseModel):
if rule.get("Filter"):
# Can't have both `Filter` and `Prefix` (need to check for the presence of the key):
try:
# 'Prefix' cannot be outside of a Filter:
if rule["Prefix"] or not rule["Prefix"]:
raise MalformedXML()
except KeyError:
pass
filters = 0
try:
prefix_filter = rule['Filter']['Prefix'] or '' # If it's `None` the set to the empty string
filters += 1
except KeyError:
prefix_filter = None
and_filter = None
if rule["Filter"].get("And"):
filters += 1
and_tags = []
if rule["Filter"]["And"].get("Tag"):
if not isinstance(rule["Filter"]["And"]["Tag"], list):
@ -543,17 +560,34 @@ class FakeBucket(BaseModel):
for t in rule["Filter"]["And"]["Tag"]:
and_tags.append(FakeTag(t["Key"], t.get("Value", '')))
and_filter = LifecycleAndFilter(prefix=rule["Filter"]["And"]["Prefix"], tags=and_tags)
try:
and_prefix = rule["Filter"]["And"]["Prefix"] or '' # If it's `None` then set to the empty string
except KeyError:
and_prefix = None
and_filter = LifecycleAndFilter(prefix=and_prefix, tags=and_tags)
filter_tag = None
if rule["Filter"].get("Tag"):
filters += 1
filter_tag = FakeTag(rule["Filter"]["Tag"]["Key"], rule["Filter"]["Tag"].get("Value", ''))
lc_filter = LifecycleFilter(prefix=rule["Filter"]["Prefix"], tag=filter_tag, and_filter=and_filter)
# Can't have more than 1 filter:
if filters > 1:
raise MalformedXML()
lc_filter = LifecycleFilter(prefix=prefix_filter, tag=filter_tag, and_filter=and_filter)
# If no top level prefix and no filter is present, then this is invalid:
if top_level_prefix is None:
try:
rule['Filter']
except KeyError:
raise MalformedXML()
self.rules.append(LifecycleRule(
id=rule.get('ID'),
prefix=rule.get('Prefix'),
prefix=top_level_prefix,
lc_filter=lc_filter,
status=rule['Status'],
expiration_days=expiration.get('Days') if expiration else None,

View file

@ -1310,7 +1310,7 @@ S3_ALL_BUCKETS = """<ListAllMyBucketsResult xmlns="http://s3.amazonaws.com/doc/2
{% for bucket in buckets %}
<Bucket>
<Name>{{ bucket.name }}</Name>
<CreationDate>2006-02-03T16:45:09.000Z</CreationDate>
<CreationDate>{{ bucket.creation_date }}</CreationDate>
</Bucket>
{% endfor %}
</Buckets>
@ -1416,7 +1416,9 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """<?xml version="1.0" encoding="UTF-8"?>
<ID>{{ rule.id }}</ID>
{% if rule.filter %}
<Filter>
{% if rule.filter.prefix != None %}
<Prefix>{{ rule.filter.prefix }}</Prefix>
{% endif %}
{% if rule.filter.tag %}
<Tag>
<Key>{{ rule.filter.tag.key }}</Key>
@ -1425,7 +1427,9 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """<?xml version="1.0" encoding="UTF-8"?>
{% endif %}
{% if rule.filter.and_filter %}
<And>
{% if rule.filter.and_filter.prefix != None %}
<Prefix>{{ rule.filter.and_filter.prefix }}</Prefix>
{% endif %}
{% for tag in rule.filter.and_filter.tags %}
<Tag>
<Key>{{ tag.key }}</Key>
@ -1436,7 +1440,9 @@ S3_BUCKET_LIFECYCLE_CONFIGURATION = """<?xml version="1.0" encoding="UTF-8"?>
{% endif %}
</Filter>
{% else %}
<Prefix>{{ rule.prefix if rule.prefix != None }}</Prefix>
{% if rule.prefix != None %}
<Prefix>{{ rule.prefix }}</Prefix>
{% endif %}
{% endif %}
<Status>{{ rule.status }}</Status>
{% if rule.storage_class %}

34
moto/ssm/exceptions.py Normal file
View file

@ -0,0 +1,34 @@
from __future__ import unicode_literals
from moto.core.exceptions import JsonRESTError
class InvalidFilterKey(JsonRESTError):
code = 400
def __init__(self, message):
super(InvalidFilterKey, self).__init__(
"InvalidFilterKey", message)
class InvalidFilterOption(JsonRESTError):
code = 400
def __init__(self, message):
super(InvalidFilterOption, self).__init__(
"InvalidFilterOption", message)
class InvalidFilterValue(JsonRESTError):
code = 400
def __init__(self, message):
super(InvalidFilterValue, self).__init__(
"InvalidFilterValue", message)
class ValidationException(JsonRESTError):
code = 400
def __init__(self, message):
super(ValidationException, self).__init__(
"ValidationException", message)

View file

@ -1,5 +1,6 @@
from __future__ import unicode_literals
import re
from collections import defaultdict
from moto.core import BaseBackend, BaseModel
@ -12,6 +13,8 @@ import time
import uuid
import itertools
from .exceptions import ValidationException, InvalidFilterValue, InvalidFilterOption, InvalidFilterKey
class Parameter(BaseModel):
def __init__(self, name, value, type, description, allowed_pattern, keyid,
@ -25,12 +28,15 @@ class Parameter(BaseModel):
self.version = version
if self.type == 'SecureString':
if not self.keyid:
self.keyid = 'alias/aws/ssm'
self.value = self.encrypt(value)
else:
self.value = value
def encrypt(self, value):
return 'kms:{}:'.format(self.keyid or 'default') + value
return 'kms:{}:'.format(self.keyid) + value
def decrypt(self, value):
if self.type != 'SecureString':
@ -217,6 +223,7 @@ class SimpleSystemManagerBackend(BaseBackend):
self._parameters = {}
self._resource_tags = defaultdict(lambda: defaultdict(dict))
self._commands = []
self._errors = []
# figure out what region we're in
for region, backend in ssm_backends.items():
@ -239,6 +246,179 @@ class SimpleSystemManagerBackend(BaseBackend):
pass
return result
def describe_parameters(self, filters, parameter_filters):
if filters and parameter_filters:
raise ValidationException('You can use either Filters or ParameterFilters in a single request.')
self._validate_parameter_filters(parameter_filters, by_path=False)
result = []
for param in self._parameters:
ssm_parameter = self._parameters[param]
if not self._match_filters(ssm_parameter, parameter_filters):
continue
if filters:
for filter in filters:
if filter['Key'] == 'Name':
k = ssm_parameter.name
for v in filter['Values']:
if k.startswith(v):
result.append(ssm_parameter)
break
elif filter['Key'] == 'Type':
k = ssm_parameter.type
for v in filter['Values']:
if k == v:
result.append(ssm_parameter)
break
elif filter['Key'] == 'KeyId':
k = ssm_parameter.keyid
if k:
for v in filter['Values']:
if k == v:
result.append(ssm_parameter)
break
continue
result.append(ssm_parameter)
return result
def _validate_parameter_filters(self, parameter_filters, by_path):
for index, filter_obj in enumerate(parameter_filters or []):
key = filter_obj['Key']
values = filter_obj.get('Values', [])
if key == 'Path':
option = filter_obj.get('Option', 'OneLevel')
else:
option = filter_obj.get('Option', 'Equals')
if not re.match(r'^tag:.+|Name|Type|KeyId|Path|Label|Tier$', key):
self._errors.append(self._format_error(
key='parameterFilters.{index}.member.key'.format(index=(index + 1)),
value=key,
constraint='Member must satisfy regular expression pattern: tag:.+|Name|Type|KeyId|Path|Label|Tier',
))
if len(key) > 132:
self._errors.append(self._format_error(
key='parameterFilters.{index}.member.key'.format(index=(index + 1)),
value=key,
constraint='Member must have length less than or equal to 132',
))
if len(option) > 10:
self._errors.append(self._format_error(
key='parameterFilters.{index}.member.option'.format(index=(index + 1)),
value='over 10 chars',
constraint='Member must have length less than or equal to 10',
))
if len(values) > 50:
self._errors.append(self._format_error(
key='parameterFilters.{index}.member.values'.format(index=(index + 1)),
value=values,
constraint='Member must have length less than or equal to 50',
))
if any(len(value) > 1024 for value in values):
self._errors.append(self._format_error(
key='parameterFilters.{index}.member.values'.format(index=(index + 1)),
value=values,
constraint='[Member must have length less than or equal to 1024, Member must have length greater than or equal to 1]',
))
self._raise_errors()
filter_keys = []
for filter_obj in (parameter_filters or []):
key = filter_obj['Key']
values = filter_obj.get('Values')
if key == 'Path':
option = filter_obj.get('Option', 'OneLevel')
else:
option = filter_obj.get('Option', 'Equals')
if not by_path and key == 'Label':
raise InvalidFilterKey('The following filter key is not valid: Label. Valid filter keys include: [Path, Name, Type, KeyId, Tier].')
if not values:
raise InvalidFilterValue('The following filter values are missing : null for filter key Name.')
if key in filter_keys:
raise InvalidFilterKey(
'The following filter is duplicated in the request: Name. A request can contain only one occurrence of a specific filter.'
)
if key == 'Path':
if option not in ['Recursive', 'OneLevel']:
raise InvalidFilterOption(
'The following filter option is not valid: {option}. Valid options include: [Recursive, OneLevel].'.format(option=option)
)
if any(value.lower().startswith(('/aws', '/ssm')) for value in values):
raise ValidationException(
'Filters for common parameters can\'t be prefixed with "aws" or "ssm" (case-insensitive). '
'When using global parameters, please specify within a global namespace.'
)
for value in values:
if value.lower().startswith(('/aws', '/ssm')):
raise ValidationException(
'Filters for common parameters can\'t be prefixed with "aws" or "ssm" (case-insensitive). '
'When using global parameters, please specify within a global namespace.'
)
if ('//' in value or
not value.startswith('/') or
not re.match('^[a-zA-Z0-9_.-/]*$', value)):
raise ValidationException(
'The parameter doesn\'t meet the parameter name requirements. The parameter name must begin with a forward slash "/". '
'It can\'t be prefixed with \"aws\" or \"ssm\" (case-insensitive). '
'It must use only letters, numbers, or the following symbols: . (period), - (hyphen), _ (underscore). '
'Special characters are not allowed. All sub-paths, if specified, must use the forward slash symbol "/". '
'Valid example: /get/parameters2-/by1./path0_.'
)
if key == 'Tier':
for value in values:
if value not in ['Standard', 'Advanced', 'Intelligent-Tiering']:
raise InvalidFilterOption(
'The following filter value is not valid: {value}. Valid values include: [Standard, Advanced, Intelligent-Tiering].'.format(value=value)
)
if key == 'Type':
for value in values:
if value not in ['String', 'StringList', 'SecureString']:
raise InvalidFilterOption(
'The following filter value is not valid: {value}. Valid values include: [String, StringList, SecureString].'.format(value=value)
)
if key != 'Path' and option not in ['Equals', 'BeginsWith']:
raise InvalidFilterOption(
'The following filter option is not valid: {option}. Valid options include: [BeginsWith, Equals].'.format(option=option)
)
filter_keys.append(key)
def _format_error(self, key, value, constraint):
return 'Value "{value}" at "{key}" failed to satisfy constraint: {constraint}'.format(
constraint=constraint,
key=key,
value=value,
)
def _raise_errors(self):
if self._errors:
count = len(self._errors)
plural = "s" if len(self._errors) > 1 else ""
errors = "; ".join(self._errors)
self._errors = [] # reset collected errors
raise ValidationException('{count} validation error{plural} detected: {errors}'.format(
count=count, plural=plural, errors=errors,
))
def get_all_parameters(self):
result = []
for k, _ in self._parameters.items():
@ -269,26 +449,53 @@ class SimpleSystemManagerBackend(BaseBackend):
return result
@staticmethod
def _match_filters(parameter, filters=None):
def _match_filters(self, parameter, filters=None):
"""Return True if the given parameter matches all the filters"""
for filter_obj in (filters or []):
key = filter_obj['Key']
option = filter_obj.get('Option', 'Equals')
values = filter_obj.get('Values', [])
what = None
if key == 'Type':
what = parameter.type
elif key == 'KeyId':
what = parameter.keyid
if key == 'Path':
option = filter_obj.get('Option', 'OneLevel')
else:
option = filter_obj.get('Option', 'Equals')
if option == 'Equals'\
and not any(what == value for value in values):
what = None
if key == 'KeyId':
what = parameter.keyid
elif key == 'Name':
what = '/' + parameter.name.lstrip('/')
values = ['/' + value.lstrip('/') for value in values]
elif key == 'Path':
what = '/' + parameter.name.lstrip('/')
values = ['/' + value.strip('/') for value in values]
elif key == 'Type':
what = parameter.type
if what is None:
return False
elif option == 'BeginsWith'\
and not any(what.startswith(value) for value in values):
elif (option == 'BeginsWith' and
not any(what.startswith(value) for value in values)):
return False
elif (option == 'Equals' and
not any(what == value for value in values)):
return False
elif option == 'OneLevel':
if any(value == '/' and len(what.split('/')) == 2 for value in values):
continue
elif any(value != '/' and
what.startswith(value + '/') and
len(what.split('/')) - 1 == len(value.split('/')) for value in values):
continue
else:
return False
elif option == 'Recursive':
if any(value == '/' for value in values):
continue
elif any(what.startswith(value + '/') for value in values):
continue
else:
return False
# True if no false match (or no filters at all)
return True

View file

@ -104,6 +104,7 @@ class SimpleSystemManagerResponse(BaseResponse):
def describe_parameters(self):
page_size = 10
filters = self._get_param('Filters')
parameter_filters = self._get_param('ParameterFilters')
token = self._get_param('NextToken')
if hasattr(token, 'strip'):
token = token.strip()
@ -111,42 +112,17 @@ class SimpleSystemManagerResponse(BaseResponse):
token = '0'
token = int(token)
result = self.ssm_backend.get_all_parameters()
result = self.ssm_backend.describe_parameters(
filters, parameter_filters
)
response = {
'Parameters': [],
}
end = token + page_size
for parameter in result[token:]:
param_data = parameter.describe_response_object(False)
add = False
if filters:
for filter in filters:
if filter['Key'] == 'Name':
k = param_data['Name']
for v in filter['Values']:
if k.startswith(v):
add = True
break
elif filter['Key'] == 'Type':
k = param_data['Type']
for v in filter['Values']:
if k == v:
add = True
break
elif filter['Key'] == 'KeyId':
k = param_data.get('KeyId')
if k:
for v in filter['Values']:
if k == v:
add = True
break
else:
add = True
if add:
response['Parameters'].append(param_data)
response['Parameters'].append(parameter.describe_response_object(False))
token = token + 1
if len(response['Parameters']) == page_size: