fix Fifo queue delivers duplicate message(s) (#3538)

* fix https://github.com/localstack/localstack/issues/3339

* fixe lint issues

* Fix review comments
- move deduplication time to constants
- make tests parameterized
- update tests as per review comments

* change variable name expectedCount => expected_count

* fix tests for python 2.7
increase deduplication mock config to account for delays

* ignore time mocking test in server mode

moto/sqs/models.py

@@ -63,6 +63,8 @@ BINARY_LIST_TYPE_FIELD_INDEX = 4
 # https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html
 ATTRIBUTE_NAME_PATTERN = re.compile("^([a-z]|[A-Z]|[0-9]|[_.\\-])+$")
 
+DEDUPLICATION_TIME_IN_SECONDS = 300
+
 
 class Message(BaseModel):
     def __init__(self, message_id, body):
@@ -478,6 +480,18 @@ class Queue(CloudFormationModel):
         ]
 
     def add_message(self, message):
+        if (
+            self.fifo_queue
+            and self.attributes.get("ContentBasedDeduplication") == "true"
+        ):
+            for m in self._messages:
+                if m.deduplication_id == message.deduplication_id:
+                    diff = message.sent_timestamp - m.sent_timestamp
+                    # if a duplicate message is received within the deduplication time then it should
+                    # not be added to the queue
+                    if diff / 1000 < DEDUPLICATION_TIME_IN_SECONDS:
+                        return
+
         self._messages.append(message)
         from moto.awslambda import lambda_backends
 
@@ -675,6 +689,13 @@ class SQSBackend(BaseBackend):
         message_id = get_random_message_id()
         message = Message(message_id, message_body)
 
+        # if content based deduplication is set then set sha256 hash of the message
+        # as the deduplication_id
+        if queue.attributes.get("ContentBasedDeduplication") == "true":
+            sha256 = hashlib.sha256()
+            sha256.update(message_body.encode("utf-8"))
+            message.deduplication_id = sha256.hexdigest()
+
         # Attributes, but not *message* attributes
         if deduplication_id is not None:
             message.deduplication_id = deduplication_id