Merge branch 'master' into batch
This commit is contained in:
Terry Cain
commit
ddd52a5a97
18 changed files with 1011 additions and 43 deletions
|
|
@ -1,4 +1,6 @@
|
|||
from __future__ import unicode_literals
|
||||
import re
|
||||
import six
|
||||
# TODO add tests for all of these
|
||||
|
||||
EQ_FUNCTION = lambda item_value, test_value: item_value == test_value # flake8: noqa
|
||||
|
|
@ -39,3 +41,452 @@ COMPARISON_FUNCS = {
|
|||
|
||||
def get_comparison_func(range_comparison):
|
||||
return COMPARISON_FUNCS.get(range_comparison)
|
||||
|
||||
|
||||
#
|
||||
def get_filter_expression(expr, names, values):
|
||||
# Examples
|
||||
# expr = 'Id > 5 AND attribute_exists(test) AND Id BETWEEN 5 AND 6 OR length < 6 AND contains(test, 1) AND 5 IN (4,5, 6) OR (Id < 5 AND 5 > Id)'
|
||||
# expr = 'Id > 5 AND Subs < 7'
|
||||
|
||||
# Need to do some dodgyness for NOT i think.
|
||||
if 'NOT' in expr:
|
||||
raise NotImplementedError('NOT not supported yet')
|
||||
|
||||
if names is None:
|
||||
names = {}
|
||||
if values is None:
|
||||
values = {}
|
||||
|
||||
# Do substitutions
|
||||
for key, value in names.items():
|
||||
expr = expr.replace(key, value)
|
||||
for key, value in values.items():
|
||||
if 'N' in value:
|
||||
expr.replace(key, float(value['N']))
|
||||
else:
|
||||
expr = expr.replace(key, value['S'])
|
||||
|
||||
# Remove all spaces, tbf we could just skip them in the next step.
|
||||
# The number of known options is really small so we can do a fair bit of cheating
|
||||
#expr = list(re.sub('\s', '', expr)) # 'Id>5ANDattribute_exists(test)ORNOTlength<6'
|
||||
expr = list(expr)
|
||||
|
||||
# DodgyTokenisation stage 1
|
||||
def is_value(val):
|
||||
return val not in ('<', '>', '=', '(', ')')
|
||||
|
||||
def contains_keyword(val):
|
||||
for kw in ('BETWEEN', 'IN', 'AND', 'OR', 'NOT'):
|
||||
if kw in val:
|
||||
return kw
|
||||
return None
|
||||
|
||||
def is_function(val):
|
||||
return val in ('attribute_exists', 'attribute_not_exists', 'attribute_type', 'begins_with', 'contains', 'size')
|
||||
|
||||
# Does the main part of splitting between sections of characters
|
||||
tokens = []
|
||||
stack = ''
|
||||
while len(expr) > 0:
|
||||
current_char = expr.pop(0)
|
||||
|
||||
if current_char == ' ':
|
||||
if len(stack) > 0:
|
||||
tokens.append(stack)
|
||||
stack = ''
|
||||
elif current_char == ',': # Split params ,
|
||||
if len(stack) > 0:
|
||||
tokens.append(stack)
|
||||
stack = ''
|
||||
elif is_value(current_char):
|
||||
stack += current_char
|
||||
|
||||
kw = contains_keyword(stack)
|
||||
if kw is not None:
|
||||
# We have a kw in the stack, could be AND or something like 5AND
|
||||
tmp = stack.replace(kw, '')
|
||||
if len(tmp) > 0:
|
||||
tokens.append(tmp)
|
||||
tokens.append(kw)
|
||||
stack = ''
|
||||
else:
|
||||
if len(stack) > 0:
|
||||
tokens.append(stack)
|
||||
tokens.append(current_char)
|
||||
stack = ''
|
||||
if len(stack) > 0:
|
||||
tokens.append(stack)
|
||||
|
||||
def is_op(val):
|
||||
return val in ('<', '>', '=', '>=', '<=', '<>', 'BETWEEN', 'IN', 'AND', 'OR', 'NOT')
|
||||
|
||||
# DodgyTokenisation stage 2, it groups together some elements to make RPN'ing it later easier.
|
||||
tokens2 = []
|
||||
token_iterator = iter(tokens)
|
||||
for token in token_iterator:
|
||||
if token == '(':
|
||||
tuple_list = []
|
||||
|
||||
next_token = six.next(token_iterator)
|
||||
while next_token != ')':
|
||||
try:
|
||||
next_token = int(next_token)
|
||||
except ValueError:
|
||||
try:
|
||||
next_token = float(next_token)
|
||||
except ValueError:
|
||||
pass
|
||||
tuple_list.append(next_token)
|
||||
next_token = six.next(token_iterator)
|
||||
|
||||
# Sigh, we only want to group a tuple if it doesnt contain operators
|
||||
if any([is_op(item) for item in tuple_list]):
|
||||
tokens2.append('(')
|
||||
tokens2.extend(tuple_list)
|
||||
tokens2.append(')')
|
||||
else:
|
||||
tokens2.append(tuple(tuple_list))
|
||||
elif token == 'BETWEEN':
|
||||
field = tokens2.pop()
|
||||
op1 = int(six.next(token_iterator))
|
||||
and_op = six.next(token_iterator)
|
||||
assert and_op == 'AND'
|
||||
op2 = int(six.next(token_iterator))
|
||||
tokens2.append(['between', field, op1, op2])
|
||||
|
||||
elif is_function(token):
|
||||
function_list = [token]
|
||||
|
||||
lbracket = six.next(token_iterator)
|
||||
assert lbracket == '('
|
||||
|
||||
next_token = six.next(token_iterator)
|
||||
while next_token != ')':
|
||||
function_list.append(next_token)
|
||||
next_token = six.next(token_iterator)
|
||||
|
||||
tokens2.append(function_list)
|
||||
|
||||
else:
|
||||
try:
|
||||
token = int(token)
|
||||
except ValueError:
|
||||
try:
|
||||
token = float(token)
|
||||
except ValueError:
|
||||
pass
|
||||
tokens2.append(token)
|
||||
|
||||
# Start of the Shunting-Yard algorithm. <-- Proper beast algorithm!
|
||||
def is_number(val):
|
||||
return val not in ('<', '>', '=', '>=', '<=', '<>', 'BETWEEN', 'IN', 'AND', 'OR', 'NOT')
|
||||
|
||||
OPS = {'<': 5, '>': 5, '=': 5, '>=': 5, '<=': 5, '<>': 5, 'IN': 8, 'AND': 11, 'OR': 12, 'NOT': 10, 'BETWEEN': 9, '(': 100, ')': 100}
|
||||
|
||||
def shunting_yard(token_list):
|
||||
output = []
|
||||
op_stack = []
|
||||
|
||||
# Basically takes in an infix notation calculation, converts it to a reverse polish notation where there is no
|
||||
# ambiguity on which order operators are applied.
|
||||
while len(token_list) > 0:
|
||||
token = token_list.pop(0)
|
||||
|
||||
if token == '(':
|
||||
op_stack.append(token)
|
||||
elif token == ')':
|
||||
while len(op_stack) > 0 and op_stack[-1] != '(':
|
||||
output.append(op_stack.pop())
|
||||
lbracket = op_stack.pop()
|
||||
assert lbracket == '('
|
||||
|
||||
elif is_number(token):
|
||||
output.append(token)
|
||||
else:
|
||||
# Must be operator kw
|
||||
while len(op_stack) > 0 and OPS[op_stack[-1]] <= OPS[token]:
|
||||
output.append(op_stack.pop())
|
||||
op_stack.append(token)
|
||||
while len(op_stack) > 0:
|
||||
output.append(op_stack.pop())
|
||||
|
||||
return output
|
||||
|
||||
output = shunting_yard(tokens2)
|
||||
|
||||
# Hacky function to convert dynamo functions (which are represented as lists) to their Class equivalent
|
||||
def to_func(val):
|
||||
if isinstance(val, list):
|
||||
func_name = val.pop(0)
|
||||
# Expand rest of the list to arguments
|
||||
val = FUNC_CLASS[func_name](*val)
|
||||
|
||||
return val
|
||||
|
||||
# Simple reverse polish notation execution. Builts up a nested filter object.
|
||||
# The filter object then takes a dynamo item and returns true/false
|
||||
stack = []
|
||||
for token in output:
|
||||
if is_op(token):
|
||||
op2 = stack.pop()
|
||||
op1 = stack.pop()
|
||||
|
||||
op_cls = OP_CLASS[token]
|
||||
stack.append(op_cls(op1, op2))
|
||||
else:
|
||||
stack.append(to_func(token))
|
||||
|
||||
result = stack.pop(0)
|
||||
if len(stack) > 0:
|
||||
raise ValueError('Malformed filter expression')
|
||||
|
||||
return result
|
||||
|
||||
|
||||
class Op(object):
|
||||
"""
|
||||
Base class for a FilterExpression operator
|
||||
"""
|
||||
OP = ''
|
||||
|
||||
def __init__(self, lhs, rhs):
|
||||
self.lhs = lhs
|
||||
self.rhs = rhs
|
||||
|
||||
def _lhs(self, item):
|
||||
"""
|
||||
:type item: moto.dynamodb2.models.Item
|
||||
"""
|
||||
lhs = self.lhs
|
||||
if isinstance(self.lhs, (Op, Func)):
|
||||
lhs = self.lhs.expr(item)
|
||||
elif isinstance(self.lhs, six.string_types):
|
||||
try:
|
||||
lhs = item.attrs[self.lhs].cast_value
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
return lhs
|
||||
|
||||
def _rhs(self, item):
|
||||
rhs = self.rhs
|
||||
if isinstance(self.rhs, (Op, Func)):
|
||||
rhs = self.rhs.expr(item)
|
||||
elif isinstance(self.rhs, six.string_types):
|
||||
try:
|
||||
rhs = item.attrs[self.rhs].cast_value
|
||||
except Exception:
|
||||
pass
|
||||
return rhs
|
||||
|
||||
def expr(self, item):
|
||||
return True
|
||||
|
||||
def __repr__(self):
|
||||
return '({0} {1} {2})'.format(self.lhs, self.OP, self.rhs)
|
||||
|
||||
|
||||
class Func(object):
|
||||
"""
|
||||
Base class for a FilterExpression function
|
||||
"""
|
||||
FUNC = 'Unknown'
|
||||
|
||||
def expr(self, item):
|
||||
return True
|
||||
|
||||
def __repr__(self):
|
||||
return 'Func(...)'.format(self.FUNC)
|
||||
|
||||
|
||||
class OpAnd(Op):
|
||||
OP = 'AND'
|
||||
|
||||
def expr(self, item):
|
||||
lhs = self._lhs(item)
|
||||
rhs = self._rhs(item)
|
||||
return lhs and rhs
|
||||
|
||||
|
||||
class OpLessThan(Op):
|
||||
OP = '<'
|
||||
|
||||
def expr(self, item):
|
||||
lhs = self._lhs(item)
|
||||
rhs = self._rhs(item)
|
||||
return lhs < rhs
|
||||
|
||||
|
||||
class OpGreaterThan(Op):
|
||||
OP = '>'
|
||||
|
||||
def expr(self, item):
|
||||
lhs = self._lhs(item)
|
||||
rhs = self._rhs(item)
|
||||
return lhs > rhs
|
||||
|
||||
|
||||
class OpEqual(Op):
|
||||
OP = '='
|
||||
|
||||
def expr(self, item):
|
||||
lhs = self._lhs(item)
|
||||
rhs = self._rhs(item)
|
||||
return lhs == rhs
|
||||
|
||||
|
||||
class OpNotEqual(Op):
|
||||
OP = '<>'
|
||||
|
||||
def expr(self, item):
|
||||
lhs = self._lhs(item)
|
||||
rhs = self._rhs(item)
|
||||
return lhs == rhs
|
||||
|
||||
|
||||
class OpLessThanOrEqual(Op):
|
||||
OP = '<='
|
||||
|
||||
def expr(self, item):
|
||||
lhs = self._lhs(item)
|
||||
rhs = self._rhs(item)
|
||||
return lhs <= rhs
|
||||
|
||||
|
||||
class OpGreaterThanOrEqual(Op):
|
||||
OP = '>='
|
||||
|
||||
def expr(self, item):
|
||||
lhs = self._lhs(item)
|
||||
rhs = self._rhs(item)
|
||||
return lhs >= rhs
|
||||
|
||||
|
||||
class OpOr(Op):
|
||||
OP = 'OR'
|
||||
|
||||
def expr(self, item):
|
||||
lhs = self._lhs(item)
|
||||
rhs = self._rhs(item)
|
||||
return lhs or rhs
|
||||
|
||||
|
||||
class OpIn(Op):
|
||||
OP = 'IN'
|
||||
|
||||
def expr(self, item):
|
||||
lhs = self._lhs(item)
|
||||
rhs = self._rhs(item)
|
||||
return lhs in rhs
|
||||
|
||||
|
||||
class FuncAttrExists(Func):
|
||||
FUNC = 'attribute_exists'
|
||||
|
||||
def __init__(self, attribute):
|
||||
self.attr = attribute
|
||||
|
||||
def expr(self, item):
|
||||
return self.attr in item.attrs
|
||||
|
||||
|
||||
class FuncAttrNotExists(Func):
|
||||
FUNC = 'attribute_not_exists'
|
||||
|
||||
def __init__(self, attribute):
|
||||
self.attr = attribute
|
||||
|
||||
def expr(self, item):
|
||||
return self.attr not in item.attrs
|
||||
|
||||
|
||||
class FuncAttrType(Func):
|
||||
FUNC = 'attribute_type'
|
||||
|
||||
def __init__(self, attribute, _type):
|
||||
self.attr = attribute
|
||||
self.type = _type
|
||||
|
||||
def expr(self, item):
|
||||
return self.attr in item.attrs and item.attrs[self.attr].type == self.type
|
||||
|
||||
|
||||
class FuncBeginsWith(Func):
|
||||
FUNC = 'begins_with'
|
||||
|
||||
def __init__(self, attribute, substr):
|
||||
self.attr = attribute
|
||||
self.substr = substr
|
||||
|
||||
def expr(self, item):
|
||||
return self.attr in item.attrs and item.attrs[self.attr].type == 'S' and item.attrs[self.attr].value.startswith(self.substr)
|
||||
|
||||
|
||||
class FuncContains(Func):
|
||||
FUNC = 'contains'
|
||||
|
||||
def __init__(self, attribute, operand):
|
||||
self.attr = attribute
|
||||
self.operand = operand
|
||||
|
||||
def expr(self, item):
|
||||
if self.attr not in item.attrs:
|
||||
return False
|
||||
|
||||
if item.attrs[self.attr].type in ('S', 'SS', 'NS', 'BS', 'L', 'M'):
|
||||
return self.operand in item.attrs[self.attr].value
|
||||
return False
|
||||
|
||||
|
||||
class FuncSize(Func):
|
||||
FUNC = 'contains'
|
||||
|
||||
def __init__(self, attribute):
|
||||
self.attr = attribute
|
||||
|
||||
def expr(self, item):
|
||||
if self.attr not in item.attrs:
|
||||
raise ValueError('Invalid attribute name {0}'.format(self.attr))
|
||||
|
||||
if item.attrs[self.attr].type in ('S', 'SS', 'NS', 'B', 'BS', 'L', 'M'):
|
||||
return len(item.attrs[self.attr].value)
|
||||
raise ValueError('Invalid filter expression')
|
||||
|
||||
|
||||
class FuncBetween(Func):
|
||||
FUNC = 'between'
|
||||
|
||||
def __init__(self, attribute, start, end):
|
||||
self.attr = attribute
|
||||
self.start = start
|
||||
self.end = end
|
||||
|
||||
def expr(self, item):
|
||||
if self.attr not in item.attrs:
|
||||
raise ValueError('Invalid attribute name {0}'.format(self.attr))
|
||||
|
||||
return self.start <= item.attrs[self.attr].cast_value <= self.end
|
||||
|
||||
|
||||
OP_CLASS = {
|
||||
'AND': OpAnd,
|
||||
'OR': OpOr,
|
||||
'IN': OpIn,
|
||||
'<': OpLessThan,
|
||||
'>': OpGreaterThan,
|
||||
'<=': OpLessThanOrEqual,
|
||||
'>=': OpGreaterThanOrEqual,
|
||||
'=': OpEqual,
|
||||
'<>': OpNotEqual
|
||||
}
|
||||
|
||||
FUNC_CLASS = {
|
||||
'attribute_exists': FuncAttrExists,
|
||||
'attribute_not_exists': FuncAttrNotExists,
|
||||
'attribute_type': FuncAttrType,
|
||||
'begins_with': FuncBeginsWith,
|
||||
'contains': FuncContains,
|
||||
'size': FuncSize,
|
||||
'between': FuncBetween
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ import re
|
|||
from moto.compat import OrderedDict
|
||||
from moto.core import BaseBackend, BaseModel
|
||||
from moto.core.utils import unix_time
|
||||
from .comparisons import get_comparison_func
|
||||
from .comparisons import get_comparison_func, get_filter_expression, Op
|
||||
|
||||
|
||||
class DynamoJsonEncoder(json.JSONEncoder):
|
||||
|
|
@ -508,15 +508,15 @@ class Table(BaseModel):
|
|||
else:
|
||||
yield hash_set
|
||||
|
||||
def scan(self, filters, limit, exclusive_start_key):
|
||||
def scan(self, filters, limit, exclusive_start_key, filter_expression=None):
|
||||
results = []
|
||||
scanned_count = 0
|
||||
|
||||
for result in self.all_items():
|
||||
for item in self.all_items():
|
||||
scanned_count += 1
|
||||
passes_all_conditions = True
|
||||
for attribute_name, (comparison_operator, comparison_objs) in filters.items():
|
||||
attribute = result.attrs.get(attribute_name)
|
||||
attribute = item.attrs.get(attribute_name)
|
||||
|
||||
if attribute:
|
||||
# Attribute found
|
||||
|
|
@ -532,8 +532,11 @@ class Table(BaseModel):
|
|||
passes_all_conditions = False
|
||||
break
|
||||
|
||||
if filter_expression is not None:
|
||||
passes_all_conditions &= filter_expression.expr(item)
|
||||
|
||||
if passes_all_conditions:
|
||||
results.append(result)
|
||||
results.append(item)
|
||||
|
||||
results, last_evaluated_key = self._trim_results(results, limit,
|
||||
exclusive_start_key)
|
||||
|
|
@ -698,7 +701,7 @@ class DynamoDBBackend(BaseBackend):
|
|||
return table.query(hash_key, range_comparison, range_values, limit,
|
||||
exclusive_start_key, scan_index_forward, projection_expression, index_name, **filter_kwargs)
|
||||
|
||||
def scan(self, table_name, filters, limit, exclusive_start_key):
|
||||
def scan(self, table_name, filters, limit, exclusive_start_key, filter_expression, expr_names, expr_values):
|
||||
table = self.tables.get(table_name)
|
||||
if not table:
|
||||
return None, None, None
|
||||
|
|
@ -708,7 +711,12 @@ class DynamoDBBackend(BaseBackend):
|
|||
dynamo_types = [DynamoType(value) for value in comparison_values]
|
||||
scan_filters[key] = (comparison_operator, dynamo_types)
|
||||
|
||||
return table.scan(scan_filters, limit, exclusive_start_key)
|
||||
if filter_expression is not None:
|
||||
filter_expression = get_filter_expression(filter_expression, expr_names, expr_values)
|
||||
else:
|
||||
filter_expression = Op(None, None) # Will always eval to true
|
||||
|
||||
return table.scan(scan_filters, limit, exclusive_start_key, filter_expression)
|
||||
|
||||
def update_item(self, table_name, key, update_expression, attribute_updates, expression_attribute_names,
|
||||
expression_attribute_values, expected=None):
|
||||
|
|
|
|||
|
|
@ -432,13 +432,29 @@ class DynamoHandler(BaseResponse):
|
|||
comparison_values = scan_filter.get("AttributeValueList", [])
|
||||
filters[attribute_name] = (comparison_operator, comparison_values)
|
||||
|
||||
filter_expression = self.body.get('FilterExpression')
|
||||
expression_attribute_values = self.body.get('ExpressionAttributeValues', {})
|
||||
expression_attribute_names = self.body.get('ExpressionAttributeNames', {})
|
||||
|
||||
exclusive_start_key = self.body.get('ExclusiveStartKey')
|
||||
limit = self.body.get("Limit")
|
||||
|
||||
items, scanned_count, last_evaluated_key = dynamodb_backend2.scan(name, filters,
|
||||
limit,
|
||||
exclusive_start_key)
|
||||
try:
|
||||
items, scanned_count, last_evaluated_key = dynamodb_backend2.scan(name, filters,
|
||||
limit,
|
||||
exclusive_start_key,
|
||||
filter_expression,
|
||||
expression_attribute_names,
|
||||
expression_attribute_values)
|
||||
except ValueError as err:
|
||||
er = 'com.amazonaws.dynamodb.v20111205#ValidationError'
|
||||
return self.error(er, 'Bad Filter Expression: {0}'.format(err))
|
||||
except Exception as err:
|
||||
er = 'com.amazonaws.dynamodb.v20111205#InternalFailure'
|
||||
return self.error(er, 'Internal error. {0}'.format(err))
|
||||
|
||||
# Items should be a list, at least an empty one. Is None if table does not exist.
|
||||
# Should really check this at the beginning
|
||||
if items is None:
|
||||
er = 'com.amazonaws.dynamodb.v20111205#ResourceNotFoundException'
|
||||
return self.error(er, 'Requested resource not found')
|
||||
|
|
|
|||
|
|
@ -109,6 +109,7 @@ from .utils import (
|
|||
random_vpn_connection_id,
|
||||
random_customer_gateway_id,
|
||||
is_tag_filter,
|
||||
tag_filter_matches,
|
||||
)
|
||||
|
||||
RESOURCES_DIR = os.path.join(os.path.dirname(__file__), 'resources')
|
||||
|
|
@ -374,6 +375,7 @@ class Instance(TaggedEC2Resource, BotoInstance):
|
|||
self.source_dest_check = "true"
|
||||
self.launch_time = utc_date_and_time()
|
||||
self.disable_api_termination = kwargs.get("disable_api_termination", False)
|
||||
self._spot_fleet_id = kwargs.get("spot_fleet_id", None)
|
||||
associate_public_ip = kwargs.get("associate_public_ip", False)
|
||||
if in_ec2_classic:
|
||||
# If we are in EC2-Classic, autoassign a public IP
|
||||
|
|
@ -511,6 +513,14 @@ class Instance(TaggedEC2Resource, BotoInstance):
|
|||
|
||||
self.teardown_defaults()
|
||||
|
||||
if self._spot_fleet_id:
|
||||
spot_fleet = self.ec2_backend.get_spot_fleet_request(self._spot_fleet_id)
|
||||
for spec in spot_fleet.launch_specs:
|
||||
if spec.instance_type == self.instance_type and spec.subnet_id == self.subnet_id:
|
||||
break
|
||||
spot_fleet.fulfilled_capacity -= spec.weighted_capacity
|
||||
spot_fleet.spot_requests = [req for req in spot_fleet.spot_requests if req.instance != self]
|
||||
|
||||
self._state.name = "terminated"
|
||||
self._state.code = 48
|
||||
|
||||
|
|
@ -1300,7 +1310,7 @@ class SecurityGroup(TaggedEC2Resource):
|
|||
elif is_tag_filter(key):
|
||||
tag_value = self.get_filter_value(key)
|
||||
if isinstance(filter_value, list):
|
||||
return any(v in tag_value for v in filter_value)
|
||||
return tag_filter_matches(self, key, filter_value)
|
||||
return tag_value in filter_value
|
||||
else:
|
||||
attr_name = to_attr(key)
|
||||
|
|
@ -2623,7 +2633,7 @@ class SpotInstanceRequest(BotoSpotRequest, TaggedEC2Resource):
|
|||
def __init__(self, ec2_backend, spot_request_id, price, image_id, type,
|
||||
valid_from, valid_until, launch_group, availability_zone_group,
|
||||
key_name, security_groups, user_data, instance_type, placement,
|
||||
kernel_id, ramdisk_id, monitoring_enabled, subnet_id,
|
||||
kernel_id, ramdisk_id, monitoring_enabled, subnet_id, spot_fleet_id,
|
||||
**kwargs):
|
||||
super(SpotInstanceRequest, self).__init__(**kwargs)
|
||||
ls = LaunchSpecification()
|
||||
|
|
@ -2646,6 +2656,7 @@ class SpotInstanceRequest(BotoSpotRequest, TaggedEC2Resource):
|
|||
ls.placement = placement
|
||||
ls.monitored = monitoring_enabled
|
||||
ls.subnet_id = subnet_id
|
||||
self.spot_fleet_id = spot_fleet_id
|
||||
|
||||
if security_groups:
|
||||
for group_name in security_groups:
|
||||
|
|
@ -2678,6 +2689,7 @@ class SpotInstanceRequest(BotoSpotRequest, TaggedEC2Resource):
|
|||
key_name=self.launch_specification.key_name,
|
||||
security_group_names=[],
|
||||
security_group_ids=self.launch_specification.groups,
|
||||
spot_fleet_id=self.spot_fleet_id,
|
||||
)
|
||||
instance = reservation.instances[0]
|
||||
return instance
|
||||
|
|
@ -2693,7 +2705,7 @@ class SpotRequestBackend(object):
|
|||
valid_until, launch_group, availability_zone_group,
|
||||
key_name, security_groups, user_data,
|
||||
instance_type, placement, kernel_id, ramdisk_id,
|
||||
monitoring_enabled, subnet_id):
|
||||
monitoring_enabled, subnet_id, spot_fleet_id=None):
|
||||
requests = []
|
||||
for _ in range(count):
|
||||
spot_request_id = random_spot_request_id()
|
||||
|
|
@ -2701,7 +2713,7 @@ class SpotRequestBackend(object):
|
|||
spot_request_id, price, image_id, type, valid_from, valid_until,
|
||||
launch_group, availability_zone_group, key_name, security_groups,
|
||||
user_data, instance_type, placement, kernel_id, ramdisk_id,
|
||||
monitoring_enabled, subnet_id)
|
||||
monitoring_enabled, subnet_id, spot_fleet_id)
|
||||
self.spot_instance_requests[spot_request_id] = request
|
||||
requests.append(request)
|
||||
return requests
|
||||
|
|
@ -2747,7 +2759,7 @@ class SpotFleetRequest(TaggedEC2Resource):
|
|||
self.iam_fleet_role = iam_fleet_role
|
||||
self.allocation_strategy = allocation_strategy
|
||||
self.state = "active"
|
||||
self.fulfilled_capacity = self.target_capacity
|
||||
self.fulfilled_capacity = 0.0
|
||||
|
||||
self.launch_specs = []
|
||||
for spec in launch_specs:
|
||||
|
|
@ -2768,7 +2780,7 @@ class SpotFleetRequest(TaggedEC2Resource):
|
|||
)
|
||||
|
||||
self.spot_requests = []
|
||||
self.create_spot_requests()
|
||||
self.create_spot_requests(self.target_capacity)
|
||||
|
||||
@property
|
||||
def physical_resource_id(self):
|
||||
|
|
@ -2798,31 +2810,32 @@ class SpotFleetRequest(TaggedEC2Resource):
|
|||
|
||||
return spot_fleet_request
|
||||
|
||||
def get_launch_spec_counts(self):
|
||||
def get_launch_spec_counts(self, weight_to_add):
|
||||
weight_map = defaultdict(int)
|
||||
|
||||
weight_so_far = 0
|
||||
if self.allocation_strategy == 'diversified':
|
||||
weight_so_far = 0
|
||||
launch_spec_index = 0
|
||||
while True:
|
||||
launch_spec = self.launch_specs[
|
||||
launch_spec_index % len(self.launch_specs)]
|
||||
weight_map[launch_spec] += 1
|
||||
weight_so_far += launch_spec.weighted_capacity
|
||||
if weight_so_far >= self.target_capacity:
|
||||
if weight_so_far >= weight_to_add:
|
||||
break
|
||||
launch_spec_index += 1
|
||||
else: # lowestPrice
|
||||
cheapest_spec = sorted(
|
||||
self.launch_specs, key=lambda spec: float(spec.spot_price))[0]
|
||||
extra = 1 if self.target_capacity % cheapest_spec.weighted_capacity else 0
|
||||
weight_so_far = weight_to_add + (weight_to_add % cheapest_spec.weighted_capacity)
|
||||
weight_map[cheapest_spec] = int(
|
||||
self.target_capacity // cheapest_spec.weighted_capacity) + extra
|
||||
weight_so_far // cheapest_spec.weighted_capacity)
|
||||
|
||||
return weight_map.items()
|
||||
return weight_map, weight_so_far
|
||||
|
||||
def create_spot_requests(self):
|
||||
for launch_spec, count in self.get_launch_spec_counts():
|
||||
def create_spot_requests(self, weight_to_add):
|
||||
weight_map, added_weight = self.get_launch_spec_counts(weight_to_add)
|
||||
for launch_spec, count in weight_map.items():
|
||||
requests = self.ec2_backend.request_spot_instances(
|
||||
price=launch_spec.spot_price,
|
||||
image_id=launch_spec.image_id,
|
||||
|
|
@ -2841,12 +2854,28 @@ class SpotFleetRequest(TaggedEC2Resource):
|
|||
ramdisk_id=None,
|
||||
monitoring_enabled=launch_spec.monitoring,
|
||||
subnet_id=launch_spec.subnet_id,
|
||||
spot_fleet_id=self.id,
|
||||
)
|
||||
self.spot_requests.extend(requests)
|
||||
self.fulfilled_capacity += added_weight
|
||||
return self.spot_requests
|
||||
|
||||
def terminate_instances(self):
|
||||
pass
|
||||
instance_ids = []
|
||||
new_fulfilled_capacity = self.fulfilled_capacity
|
||||
for req in self.spot_requests:
|
||||
instance = req.instance
|
||||
for spec in self.launch_specs:
|
||||
if spec.instance_type == instance.instance_type and spec.subnet_id == instance.subnet_id:
|
||||
break
|
||||
|
||||
if new_fulfilled_capacity - spec.weighted_capacity < self.target_capacity:
|
||||
continue
|
||||
new_fulfilled_capacity -= spec.weighted_capacity
|
||||
instance_ids.append(instance.id)
|
||||
|
||||
self.spot_requests = [req for req in self.spot_requests if req.instance.id not in instance_ids]
|
||||
self.ec2_backend.terminate_instances(instance_ids)
|
||||
|
||||
|
||||
class SpotFleetBackend(object):
|
||||
|
|
@ -2882,12 +2911,26 @@ class SpotFleetBackend(object):
|
|||
def cancel_spot_fleet_requests(self, spot_fleet_request_ids, terminate_instances):
|
||||
spot_requests = []
|
||||
for spot_fleet_request_id in spot_fleet_request_ids:
|
||||
spot_fleet = self.spot_fleet_requests.pop(spot_fleet_request_id)
|
||||
spot_fleet = self.spot_fleet_requests[spot_fleet_request_id]
|
||||
if terminate_instances:
|
||||
spot_fleet.target_capacity = 0
|
||||
spot_fleet.terminate_instances()
|
||||
spot_requests.append(spot_fleet)
|
||||
del self.spot_fleet_requests[spot_fleet_request_id]
|
||||
return spot_requests
|
||||
|
||||
def modify_spot_fleet_request(self, spot_fleet_request_id, target_capacity, terminate_instances):
|
||||
if target_capacity < 0:
|
||||
raise ValueError('Cannot reduce spot fleet capacity below 0')
|
||||
spot_fleet_request = self.spot_fleet_requests[spot_fleet_request_id]
|
||||
delta = target_capacity - spot_fleet_request.fulfilled_capacity
|
||||
spot_fleet_request.target_capacity = target_capacity
|
||||
if delta > 0:
|
||||
spot_fleet_request.create_spot_requests(delta)
|
||||
elif delta < 0 and terminate_instances == 'Default':
|
||||
spot_fleet_request.terminate_instances()
|
||||
return True
|
||||
|
||||
|
||||
class ElasticAddress(object):
|
||||
def __init__(self, domain):
|
||||
|
|
|
|||
|
|
@ -29,6 +29,15 @@ class SpotFleets(BaseResponse):
|
|||
template = self.response_template(DESCRIBE_SPOT_FLEET_TEMPLATE)
|
||||
return template.render(requests=requests)
|
||||
|
||||
def modify_spot_fleet_request(self):
|
||||
spot_fleet_request_id = self._get_param("SpotFleetRequestId")
|
||||
target_capacity = self._get_int_param("TargetCapacity")
|
||||
terminate_instances = self._get_param("ExcessCapacityTerminationPolicy", if_none="Default")
|
||||
successful = self.ec2_backend.modify_spot_fleet_request(
|
||||
spot_fleet_request_id, target_capacity, terminate_instances)
|
||||
template = self.response_template(MODIFY_SPOT_FLEET_REQUEST_TEMPLATE)
|
||||
return template.render(successful=successful)
|
||||
|
||||
def request_spot_fleet(self):
|
||||
spot_config = self._get_dict_param("SpotFleetRequestConfig.")
|
||||
spot_price = spot_config['spot_price']
|
||||
|
|
@ -56,6 +65,11 @@ REQUEST_SPOT_FLEET_TEMPLATE = """<RequestSpotFleetResponse xmlns="http://ec2.ama
|
|||
<spotFleetRequestId>{{ request.id }}</spotFleetRequestId>
|
||||
</RequestSpotFleetResponse>"""
|
||||
|
||||
MODIFY_SPOT_FLEET_REQUEST_TEMPLATE = """<ModifySpotFleetRequestResponse xmlns="http://ec2.amazonaws.com/doc/2016-09-15/">
|
||||
<requestId>21681fea-9987-aef3-2121-example</requestId>
|
||||
<return>{{ 'true' if successful else 'false' }}</return>
|
||||
</ModifySpotFleetRequestResponse>"""
|
||||
|
||||
DESCRIBE_SPOT_FLEET_TEMPLATE = """<DescribeSpotFleetRequestsResponse xmlns="http://ec2.amazonaws.com/doc/2016-09-15/">
|
||||
<requestId>4d68a6cc-8f2e-4be1-b425-example</requestId>
|
||||
<spotFleetRequestConfigSet>
|
||||
|
|
|
|||
|
|
@ -51,7 +51,7 @@ def random_ami_id():
|
|||
|
||||
|
||||
def random_instance_id():
|
||||
return random_id(prefix=EC2_RESOURCE_TO_PREFIX['instance'])
|
||||
return random_id(prefix=EC2_RESOURCE_TO_PREFIX['instance'], size=17)
|
||||
|
||||
|
||||
def random_reservation_id():
|
||||
|
|
|
|||
|
|
@ -1159,9 +1159,7 @@ CREATE_ACCESS_KEY_TEMPLATE = """<CreateAccessKeyResponse>
|
|||
<UserName>{{ key.user_name }}</UserName>
|
||||
<AccessKeyId>{{ key.access_key_id }}</AccessKeyId>
|
||||
<Status>{{ key.status }}</Status>
|
||||
<SecretAccessKey>
|
||||
{{ key.secret_access_key }}
|
||||
</SecretAccessKey>
|
||||
<SecretAccessKey>{{ key.secret_access_key }}</SecretAccessKey>
|
||||
</AccessKey>
|
||||
</CreateAccessKeyResult>
|
||||
<ResponseMetadata>
|
||||
|
|
|
|||
|
|
@ -103,6 +103,12 @@ try: # pragma: no cover
|
|||
except ImportError: # pragma: no cover
|
||||
ssl = None
|
||||
|
||||
try: # pragma: no cover
|
||||
from requests.packages.urllib3.contrib.pyopenssl import inject_into_urllib3, extract_from_urllib3
|
||||
pyopenssl_override = True
|
||||
except:
|
||||
pyopenssl_override = False
|
||||
|
||||
|
||||
DEFAULT_HTTP_PORTS = frozenset([80])
|
||||
POTENTIAL_HTTP_PORTS = set(DEFAULT_HTTP_PORTS)
|
||||
|
|
@ -1013,6 +1019,9 @@ class httpretty(HttpBaseClass):
|
|||
ssl.sslwrap_simple = old_sslwrap_simple
|
||||
ssl.__dict__['sslwrap_simple'] = old_sslwrap_simple
|
||||
|
||||
if pyopenssl_override:
|
||||
inject_into_urllib3()
|
||||
|
||||
@classmethod
|
||||
def is_enabled(cls):
|
||||
return cls._is_enabled
|
||||
|
|
@ -1056,6 +1065,9 @@ class httpretty(HttpBaseClass):
|
|||
ssl.sslwrap_simple = fake_wrap_socket
|
||||
ssl.__dict__['sslwrap_simple'] = fake_wrap_socket
|
||||
|
||||
if pyopenssl_override:
|
||||
extract_from_urllib3()
|
||||
|
||||
|
||||
def httprettified(test):
|
||||
"A decorator tests that use HTTPretty"
|
||||
|
|
|
|||
|
|
@ -16,3 +16,8 @@ class MessageAttributesInvalid(Exception):
|
|||
|
||||
def __init__(self, description):
|
||||
self.description = description
|
||||
|
||||
|
||||
class QueueDoesNotExist(Exception):
|
||||
status_code = 404
|
||||
description = "The specified queue does not exist for this wsdl version."
|
||||
|
|
|
|||
|
|
@ -12,7 +12,12 @@ import boto.sqs
|
|||
from moto.core import BaseBackend, BaseModel
|
||||
from moto.core.utils import camelcase_to_underscores, get_random_message_id, unix_time, unix_time_millis
|
||||
from .utils import generate_receipt_handle
|
||||
from .exceptions import ReceiptHandleIsInvalid, MessageNotInflight, MessageAttributesInvalid
|
||||
from .exceptions import (
|
||||
MessageAttributesInvalid,
|
||||
MessageNotInflight,
|
||||
QueueDoesNotExist,
|
||||
ReceiptHandleIsInvalid,
|
||||
)
|
||||
|
||||
DEFAULT_ACCOUNT_ID = 123456789012
|
||||
DEFAULT_SENDER_ID = "AIDAIT2UOQQY3AUEKVGXU"
|
||||
|
|
@ -304,7 +309,10 @@ class SQSBackend(BaseBackend):
|
|||
return qs
|
||||
|
||||
def get_queue(self, queue_name):
|
||||
return self.queues.get(queue_name, None)
|
||||
queue = self.queues.get(queue_name)
|
||||
if queue is None:
|
||||
raise QueueDoesNotExist()
|
||||
return queue
|
||||
|
||||
def delete_queue(self, queue_name):
|
||||
if queue_name in self.queues:
|
||||
|
|
|
|||
|
|
@ -8,7 +8,8 @@ from .models import sqs_backends
|
|||
from .exceptions import (
|
||||
MessageAttributesInvalid,
|
||||
MessageNotInflight,
|
||||
ReceiptHandleIsInvalid
|
||||
QueueDoesNotExist,
|
||||
ReceiptHandleIsInvalid,
|
||||
)
|
||||
|
||||
MAXIMUM_VISIBILTY_TIMEOUT = 43200
|
||||
|
|
@ -76,7 +77,12 @@ class SQSResponse(BaseResponse):
|
|||
def get_queue_url(self):
|
||||
request_url = urlparse(self.uri)
|
||||
queue_name = self._get_param("QueueName")
|
||||
queue = self.sqs_backend.get_queue(queue_name)
|
||||
|
||||
try:
|
||||
queue = self.sqs_backend.get_queue(queue_name)
|
||||
except QueueDoesNotExist as e:
|
||||
return self._error('QueueDoesNotExist', e.description)
|
||||
|
||||
if queue:
|
||||
template = self.response_template(GET_QUEUE_URL_RESPONSE)
|
||||
return template.render(queue=queue, request_url=request_url)
|
||||
|
|
@ -113,7 +119,11 @@ class SQSResponse(BaseResponse):
|
|||
|
||||
def get_queue_attributes(self):
|
||||
queue_name = self._get_queue_name()
|
||||
queue = self.sqs_backend.get_queue(queue_name)
|
||||
try:
|
||||
queue = self.sqs_backend.get_queue(queue_name)
|
||||
except QueueDoesNotExist as e:
|
||||
return self._error('QueueDoesNotExist', e.description)
|
||||
|
||||
template = self.response_template(GET_QUEUE_ATTRIBUTES_RESPONSE)
|
||||
return template.render(queue=queue)
|
||||
|
||||
|
|
@ -250,7 +260,11 @@ class SQSResponse(BaseResponse):
|
|||
|
||||
def receive_message(self):
|
||||
queue_name = self._get_queue_name()
|
||||
queue = self.sqs_backend.get_queue(queue_name)
|
||||
|
||||
try:
|
||||
queue = self.sqs_backend.get_queue(queue_name)
|
||||
except QueueDoesNotExist as e:
|
||||
return self._error('QueueDoesNotExist', e.description)
|
||||
|
||||
try:
|
||||
message_count = int(self.querystring.get("MaxNumberOfMessages")[0])
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue