catalin/moto
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Completed SQS Implementation

Browse source
This commit is contained in:
Terry Cain 2017-10-29 18:59:40 +00:00
commit 75f0deb828
5 changed files with 463 additions and 34 deletions
Download patch file Download diff file Expand all files Collapse all files

85
moto/sqs/models.py
View file

@ -2,6 +2,7 @@ from __future__ import unicode_literals
import base64
import hashlib
import json
import re
import six
import struct
@ -9,6 +10,7 @@ from xml.sax.saxutils import escape
import boto.sqs
from moto.core.exceptions import RESTError
from moto.core import BaseBackend, BaseModel
from moto.core.utils import camelcase_to_underscores, get_random_message_id, unix_time, unix_time_millis
from .utils import generate_receipt_handle
@ -166,11 +168,14 @@ class Queue(BaseModel):
'ReceiveMessageWaitTimeSeconds',
'VisibilityTimeout',
'WaitTimeSeconds']
ALLOWED_PERMISSIONS = ('*', 'ChangeMessageVisibility', 'DeleteMessage', 'GetQueueAttributes',
'GetQueueUrl', 'ReceiveMessage', 'SendMessage')
def __init__(self, name, region, **kwargs):
self.name = name
self.visibility_timeout = int(kwargs.get('VisibilityTimeout', 30))
self.region = region
self.tags = {}
self._messages = []
@ -189,14 +194,42 @@ class Queue(BaseModel):
self.message_retention_period = int(kwargs.get('MessageRetentionPeriod', 86400 * 4)) # four days
self.queue_arn = 'arn:aws:sqs:{0}:123456789012:{1}'.format(self.region, self.name)
self.receive_message_wait_time_seconds = int(kwargs.get('ReceiveMessageWaitTimeSeconds', 0))
self.permissions = {}
# wait_time_seconds will be set to immediate return messages
self.wait_time_seconds = int(kwargs.get('WaitTimeSeconds', 0))
self.redrive_policy = {}
self.dead_letter_queue = None
if 'RedrivePolicy' in kwargs:
self._setup_dlq(kwargs['RedrivePolicy'])
# Check some conditions
if self.fifo_queue and not self.name.endswith('.fifo'):
raise MessageAttributesInvalid('Queue name must end in .fifo for FIFO queues')
def _setup_dlq(self, policy_json):
try:
self.redrive_policy = json.loads(policy_json)
except ValueError:
raise RESTError('InvalidParameterValue', 'Redrive policy does not contain valid json')
if 'deadLetterTargetArn' not in self.redrive_policy:
raise RESTError('InvalidParameterValue', 'Redrive policy does not contain deadLetterTargetArn')
if 'maxReceiveCount' not in self.redrive_policy:
raise RESTError('InvalidParameterValue', 'Redrive policy does not contain maxReceiveCount')
for queue in sqs_backends[self.region].queues.values():
if queue.queue_arn == self.redrive_policy['deadLetterTargetArn']:
self.dead_letter_queue = queue
if self.fifo_queue and not queue.fifo_queue:
raise RESTError('InvalidParameterCombination', 'Fifo queues cannot use non fifo dead letter queues')
break
else:
raise RESTError('AWS.SimpleQueueService.NonExistentQueue', 'Could not find DLQ for {0}'.format(self.redrive_policy['deadLetterTargetArn']))
@classmethod
def create_from_cloudformation_json(cls, resource_name, cloudformation_json, region_name):
properties = cloudformation_json['Properties']
@ -382,9 +415,14 @@ class SQSBackend(BaseBackend):
time.sleep(0.001)
continue
messages_to_dlq = []
for message in queue.messages:
if not message.visible:
continue
if queue.dead_letter_queue is not None and message.approximate_receive_count >= queue.redrive_policy['maxReceiveCount']:
messages_to_dlq.append(message)
continue
message.mark_received(
visibility_timeout=visibility_timeout
)
@ -392,6 +430,10 @@ class SQSBackend(BaseBackend):
if len(result) >= count:
break
for message in messages_to_dlq:
queue._messages.remove(message)
queue.dead_letter_queue.add_message(message)
return result
def delete_message(self, queue_name, receipt_handle):
@ -419,6 +461,49 @@ class SQSBackend(BaseBackend):
queue = self.get_queue(queue_name)
queue._messages = []
def list_dead_letter_source_queues(self, queue_name):
dlq = self.get_queue(queue_name)
queues = []
for queue in self.queues.values():
if queue.dead_letter_queue is dlq:
queues.append(queue)
return queues
def add_permission(self, queue_name, actions, account_ids, label):
queue = self.get_queue(queue_name)
if actions is None or len(actions) == 0:
raise RESTError('InvalidParameterValue', 'Need at least one Action')
if account_ids is None or len(account_ids) == 0:
raise RESTError('InvalidParameterValue', 'Need at least one Account ID')
if not all([item in Queue.ALLOWED_PERMISSIONS for item in actions]):
raise RESTError('InvalidParameterValue', 'Invalid permissions')
queue.permissions[label] = (account_ids, actions)
def remove_permission(self, queue_name, label):
queue = self.get_queue(queue_name)
if label not in queue.permissions:
raise RESTError('InvalidParameterValue', 'Permission doesnt exist for the given label')
del queue.permissions[label]
def tag_queue(self, queue_name, tags):
queue = self.get_queue(queue_name)
queue.tags.update(tags)
def untag_queue(self, queue_name, tag_keys):
queue = self.get_queue(queue_name)
for key in tag_keys:
try:
del queue.tags[key]
except KeyError:
pass
sqs_backends = {}
for region in boto.sqs.regions():

202
moto/sqs/responses.py
View file

@ -40,12 +40,15 @@ class SQSResponse(BaseResponse):
queue_name = self.path.split("/")[-1]
return queue_name
def _get_validated_visibility_timeout(self):
def _get_validated_visibility_timeout(self, timeout=None):
"""
:raises ValueError: If specified visibility timeout exceeds MAXIMUM_VISIBILTY_TIMEOUT
:raises TypeError: If visibility timeout was not specified
"""
visibility_timeout = int(self.querystring.get("VisibilityTimeout")[0])
if timeout is not None:
visibility_timeout = int(timeout)
else:
visibility_timeout = int(self.querystring.get("VisibilityTimeout")[0])
if visibility_timeout > MAXIMUM_VISIBILTY_TIMEOUT:
raise ValueError
@ -119,6 +122,49 @@ class SQSResponse(BaseResponse):
template = self.response_template(CHANGE_MESSAGE_VISIBILITY_RESPONSE)
return template.render()
def change_message_visibility_batch(self):
queue_name = self._get_queue_name()
entries = self._get_list_prefix('ChangeMessageVisibilityBatchRequestEntry')
success = []
error = []
for entry in entries:
try:
visibility_timeout = self._get_validated_visibility_timeout(entry['visibility_timeout'])
except ValueError:
error.append({
'Id': entry['id'],
'SenderFault': 'true',
'Code': 'InvalidParameterValue',
'Message': 'Visibility timeout invalid'
})
continue
try:
self.sqs_backend.change_message_visibility(
queue_name=queue_name,
receipt_handle=entry['receipt_handle'],
visibility_timeout=visibility_timeout
)
success.append(entry['id'])
except ReceiptHandleIsInvalid as e:
error.append({
'Id': entry['id'],
'SenderFault': 'true',
'Code': 'ReceiptHandleIsInvalid',
'Message': e.description
})
except MessageNotInflight as e:
error.append({
'Id': entry['id'],
'SenderFault': 'false',
'Code': 'AWS.SimpleQueueService.MessageNotInflight',
'Message': e.description
})
template = self.response_template(CHANGE_MESSAGE_VISIBILITY_BATCH_RESPONSE)
return template.render(success=success, errors=error)
def get_queue_attributes(self):
queue_name = self._get_queue_name()
try:
@ -288,8 +334,62 @@ class SQSResponse(BaseResponse):
messages = self.sqs_backend.receive_messages(
queue_name, message_count, wait_time, visibility_timeout)
template = self.response_template(RECEIVE_MESSAGE_RESPONSE)
output = template.render(messages=messages)
return output
return template.render(messages=messages)
def list_dead_letter_source_queues(self):
request_url = urlparse(self.uri)
queue_name = self._get_queue_name()
source_queue_urls = self.sqs_backend.list_dead_letter_source_queues(queue_name)
template = self.response_template(LIST_DEAD_LETTER_SOURCE_QUEUES_RESPONSE)
return template.render(queues=source_queue_urls, request_url=request_url)
def add_permission(self):
queue_name = self._get_queue_name()
actions = self._get_multi_param('ActionName')
account_ids = self._get_multi_param('AWSAccountId')
label = self._get_param('Label')
self.sqs_backend.add_permission(queue_name, actions, account_ids, label)
template = self.response_template(ADD_PERMISSION_RESPONSE)
return template.render()
def remove_permission(self):
queue_name = self._get_queue_name()
label = self._get_param('Label')
self.sqs_backend.remove_permission(queue_name, label)
template = self.response_template(REMOVE_PERMISSION_RESPONSE)
return template.render()
def tag_queue(self):
queue_name = self._get_queue_name()
tags = self._get_map_prefix('Tag', key_end='.Key', value_end='.Value')
self.sqs_backend.tag_queue(queue_name, tags)
template = self.response_template(TAG_QUEUE_RESPONSE)
return template.render()
def untag_queue(self):
queue_name = self._get_queue_name()
tag_keys = self._get_multi_param('TagKey')
self.sqs_backend.untag_queue(queue_name, tag_keys)
template = self.response_template(UNTAG_QUEUE_RESPONSE)
return template.render()
def list_queue_tags(self):
queue_name = self._get_queue_name()
queue = self.sqs_backend.get_queue(queue_name)
template = self.response_template(LIST_QUEUE_TAGS_RESPONSE)
return template.render(tags=queue.tags)
CREATE_QUEUE_RESPONSE = """<CreateQueueResponse>
@ -307,7 +407,7 @@ GET_QUEUE_URL_RESPONSE = """<GetQueueUrlResponse>
<QueueUrl>{{ queue.url(request_url) }}</QueueUrl>
</GetQueueUrlResult>
<ResponseMetadata>
<RequestId>470a6f13-2ed9-4181-ad8a-2fdea142988e</RequestId>
<RequestId>{{ requestid }}</RequestId>
</ResponseMetadata>
</GetQueueUrlResponse>"""
@ -318,13 +418,13 @@ LIST_QUEUES_RESPONSE = """<ListQueuesResponse>
{% endfor %}
</ListQueuesResult>
<ResponseMetadata>
<RequestId>725275ae-0b9b-4762-b238-436d7c65a1ac</RequestId>
<RequestId>{{ requestid }}</RequestId>
</ResponseMetadata>
</ListQueuesResponse>"""
DELETE_QUEUE_RESPONSE = """<DeleteQueueResponse>
<ResponseMetadata>
<RequestId>6fde8d1e-52cd-4581-8cd9-c512f4c64223</RequestId>
<RequestId>{{ requestid }}</RequestId>
</ResponseMetadata>
</DeleteQueueResponse>"""
@ -338,13 +438,13 @@ GET_QUEUE_ATTRIBUTES_RESPONSE = """<GetQueueAttributesResponse>
{% endfor %}
</GetQueueAttributesResult>
<ResponseMetadata>
<RequestId>1ea71be5-b5a2-4f9d-b85a-945d8d08cd0b</RequestId>
<RequestId>{{ requestid }}</RequestId>
</ResponseMetadata>
</GetQueueAttributesResponse>"""
SET_QUEUE_ATTRIBUTE_RESPONSE = """<SetQueueAttributesResponse>
<ResponseMetadata>
<RequestId>e5cca473-4fc0-4198-a451-8abb94d02c75</RequestId>
<RequestId>{{ requestid }}</RequestId>
</ResponseMetadata>
</SetQueueAttributesResponse>"""
@ -361,7 +461,7 @@ SEND_MESSAGE_RESPONSE = """<SendMessageResponse>
</MessageId>
</SendMessageResult>
<ResponseMetadata>
<RequestId>27daac76-34dd-47df-bd01-1f6e873584a0</RequestId>
<RequestId>{{ requestid }}</RequestId>
</ResponseMetadata>
</SendMessageResponse>"""
@ -409,7 +509,7 @@ RECEIVE_MESSAGE_RESPONSE = """<ReceiveMessageResponse>
{% endfor %}
</ReceiveMessageResult>
<ResponseMetadata>
<RequestId>b6633655-283d-45b4-aee4-4e84e0ae6afa</RequestId>
<RequestId>{{ requestid }}</RequestId>
</ResponseMetadata>
</ReceiveMessageResponse>"""
@ -427,13 +527,13 @@ SEND_MESSAGE_BATCH_RESPONSE = """<SendMessageBatchResponse>
{% endfor %}
</SendMessageBatchResult>
<ResponseMetadata>
<RequestId>ca1ad5d0-8271-408b-8d0f-1351bf547e74</RequestId>
<RequestId>{{ requestid }}</RequestId>
</ResponseMetadata>
</SendMessageBatchResponse>"""
DELETE_MESSAGE_RESPONSE = """<DeleteMessageResponse>
<ResponseMetadata>
<RequestId>b5293cb5-d306-4a17-9048-b263635abe42</RequestId>
<RequestId>{{ requestid }}</RequestId>
</ResponseMetadata>
</DeleteMessageResponse>"""
@ -446,22 +546,92 @@ DELETE_MESSAGE_BATCH_RESPONSE = """<DeleteMessageBatchResponse>
{% endfor %}
</DeleteMessageBatchResult>
<ResponseMetadata>
<RequestId>d6f86b7a-74d1-4439-b43f-196a1e29cd85</RequestId>
<RequestId>{{ requestid }}</RequestId>
</ResponseMetadata>
</DeleteMessageBatchResponse>"""
CHANGE_MESSAGE_VISIBILITY_RESPONSE = """<ChangeMessageVisibilityResponse>
<ResponseMetadata>
<RequestId>6a7a282a-d013-4a59-aba9-335b0fa48bed</RequestId>
<RequestId>{{ requestid }}</RequestId>
</ResponseMetadata>
</ChangeMessageVisibilityResponse>"""
CHANGE_MESSAGE_VISIBILITY_BATCH_RESPONSE = """<ChangeMessageVisibilityBatchResponse>
<ChangeMessageVisibilityBatchResult>
{% for success_id in success %}
<ChangeMessageVisibilityBatchResultEntry>
<Id>{{ success_id }}</Id>
</ChangeMessageVisibilityBatchResultEntry>
{% endfor %}
{% for error_dict in errors %}
<BatchResultErrorEntry>
<Id>{{ error_dict['Id'] }}</Id>
<Code>{{ error_dict['Code'] }}</Code>
<Message>{{ error_dict['Message'] }}</Message>
<SenderFault>{{ error_dict['SenderFault'] }}</SenderFault>
</BatchResultErrorEntry>
{% endfor %}
</ChangeMessageVisibilityBatchResult>
<ResponseMetadata>
<RequestId>{{ request_id }}</RequestId>
</ResponseMetadata>
</ChangeMessageVisibilityBatchResponse>"""
PURGE_QUEUE_RESPONSE = """<PurgeQueueResponse>
<ResponseMetadata>
<RequestId>6fde8d1e-52cd-4581-8cd9-c512f4c64223</RequestId>
<RequestId>{{ requestid }}</RequestId>
</ResponseMetadata>
</PurgeQueueResponse>"""
LIST_DEAD_LETTER_SOURCE_QUEUES_RESPONSE = """<ListDeadLetterSourceQueuesResponse xmlns="http://queue.amazonaws.com/doc/2012-11-05/">
<ListDeadLetterSourceQueuesResult>
{% for queue in queues %}
<QueueUrl>{{ queue.url(request_url) }}</QueueUrl>
{% endfor %}
</ListDeadLetterSourceQueuesResult>
<ResponseMetadata>
<RequestId>8ffb921f-b85e-53d9-abcf-d8d0057f38fc</RequestId>
</ResponseMetadata>
</ListDeadLetterSourceQueuesResponse>"""
ADD_PERMISSION_RESPONSE = """<AddPermissionResponse>
<ResponseMetadata>
<RequestId>{{ request_id }}</RequestId>
</ResponseMetadata>
</AddPermissionResponse>"""
REMOVE_PERMISSION_RESPONSE = """<RemovePermissionResponse>
<ResponseMetadata>
<RequestId>{{ request_id }}</RequestId>
</ResponseMetadata>
</RemovePermissionResponse>"""
TAG_QUEUE_RESPONSE = """<TagQueueResponse>
<ResponseMetadata>
<RequestId>{{ request_id }}</RequestId>
</ResponseMetadata>
</TagQueueResponse>"""
UNTAG_QUEUE_RESPONSE = """<UntagQueueResponse>
<ResponseMetadata>
<RequestId>{{ request_id }}</RequestId>
</ResponseMetadata>
</UntagQueueResponse>"""
LIST_QUEUE_TAGS_RESPONSE = """<ListQueueTagsResponse>
<ListQueueTagsResult>
{% for key, value in tags.items() %}
<Tag>
<Key>{{ key }}</Key>
<Value>{{ value }}</Value>
</Tag>
{% endfor %}
</ListQueueTagsResult>
<ResponseMetadata>
<RequestId>{{ request_id }}</RequestId>
</ResponseMetadata>
</ListQueueTagsResponse>"""
ERROR_TOO_LONG_RESPONSE = """<ErrorResponse xmlns="http://queue.amazonaws.com/doc/2012-11-05/">
<Error>
<Type>Sender</Type>