RFC/Draft for configuration utility

Author: BeyondEvil

.travis.yml

@@ -24,7 +24,16 @@ install:
     else
       PYTHON_DOCKER_TAG=${TRAVIS_PYTHON_VERSION}-stretch
     fi
-    docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:${PYTHON_DOCKER_TAG} /moto/travis_moto_server.sh &
+    docker run --rm -t \
+      --name motoserver \
+      -e MOTO_ENVIRONMENT=development \
+      -e TEST_SERVER_MODE=true \
+      -e AWS_SECRET_ACCESS_KEY=server_secret \
+      -e AWS_ACCESS_KEY_ID=server_key \
+      -v `pwd`:/moto \
+      -v /var/run/docker.sock:/var/run/docker.sock \
+      -p 5000:5000 \
+      python:${PYTHON_DOCKER_TAG} /moto/travis_moto_server.sh &
   fi
   travis_retry pip install -r requirements-dev.txt
   travis_retry pip install docker>=2.5.1

MANIFEST.in

@@ -5,5 +5,6 @@ include moto/ec2/resources/instance_type_offerings/*/*.json
 include moto/ec2/resources/amis.json
 include moto/cognitoidp/resources/*.json
 include moto/dynamodb2/parsing/reserved_keywords.txt
+include moto/configuration.ini
 recursive-include moto/templates *
 recursive-include tests *

Makefile

@@ -26,12 +26,12 @@ format:
 test-only:
 	rm -f .coverage
 	rm -rf cover
-	@pytest -sv --cov=moto --cov-report html ./tests/ $(TEST_EXCLUDE)
+	@MOTO_ENVIRONMENT=development pytest -sv --cov=moto --cov-report html ./tests/ $(TEST_EXCLUDE)
 
 test: lint test-only
 
 test_server:
-	@TEST_SERVER_MODE=true pytest -sv --cov=moto --cov-report html ./tests/
+	@TEST_SERVER_MODE=true @MOTO_ENVIRONMENT=development pytest -sv --cov=moto --cov-report html ./tests/
 
 aws_managed_policies:
 	scripts/update_managed_policies.py

moto/configuration.ini

@@ -0,0 +1,5 @@
+[production]
+DeduplicationTimeInSeconds = 300
+
+[development]
+DeduplicationTimeInSeconds = 1

moto/sqs/models.py

@@ -20,7 +20,7 @@
     unix_time_millis,
     tags_from_cloudformation_tags_list,
 )
-from .utils import generate_receipt_handle
+from .utils import generate_receipt_handle, read_config
 from .exceptions import (
     MessageAttributesInvalid,
     MessageNotInflight,
@@ -63,8 +63,6 @@
 # https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html
 ATTRIBUTE_NAME_PATTERN = re.compile("^([a-z]|[A-Z]|[0-9]|[_.\\-])+$")
 
-DEDUPLICATION_TIME_IN_SECONDS = 300
-
 
 class Message(BaseModel):
     def __init__(self, message_id, body):
@@ -231,9 +229,10 @@ class Queue(CloudFormationModel):
         "SendMessage",
     )
 
-    def __init__(self, name, region, **kwargs):
+    def __init__(self, name, region, config, **kwargs):
         self.name = name
         self.region = region
+        self.config = config
         self.tags = {}
         self.permissions = {}
 
@@ -489,7 +488,7 @@ def add_message(self, message):
                     diff = message.sent_timestamp - m.sent_timestamp
                     # if a duplicate message is received within the deduplication time then it should
                     # not be added to the queue
-                    if diff / 1000 < DEDUPLICATION_TIME_IN_SECONDS:
+                    if diff / 1000 < int(self.config["DeduplicationTimeInSeconds"]):
                         return
 
         self._messages.append(message)
@@ -561,16 +560,18 @@ def _filter_message_attributes(message, input_message_attributes):
 
 
 class SQSBackend(BaseBackend):
-    def __init__(self, region_name):
+    def __init__(self, region_name, config):
         self.region_name = region_name
+        self.config = config
         self.queues = {}
         super(SQSBackend, self).__init__()
 
     def reset(self):
         region_name = self.region_name
+        config = self.config
         self._reset_model_refs()
         self.__dict__ = {}
-        self.__init__(region_name)
+        self.__init__(region_name, config)
 
     def create_queue(self, name, tags=None, **kwargs):
         queue = self.queues.get(name)
@@ -580,7 +581,9 @@ def create_queue(self, name, tags=None, **kwargs):
             except KeyError:
                 pass
 
-            new_queue = Queue(name, region=self.region_name, **kwargs)
+            new_queue = Queue(
+                name, region=self.region_name, config=self.config, **kwargs
+            )
 
             queue_attributes = queue.attributes
             new_queue_attributes = new_queue.attributes
@@ -595,7 +598,7 @@ def create_queue(self, name, tags=None, **kwargs):
                 kwargs.pop("region")
             except KeyError:
                 pass
-            queue = Queue(name, region=self.region_name, **kwargs)
+            queue = Queue(name, region=self.region_name, config=self.config, **kwargs)
             self.queues[name] = queue
 
         if tags:
@@ -1008,9 +1011,10 @@ def list_queue_tags(self, queue_name):
 
 
 sqs_backends = {}
+config = read_config()
 for region in Session().get_available_regions("sqs"):
-    sqs_backends[region] = SQSBackend(region)
+    sqs_backends[region] = SQSBackend(region, config)
 for region in Session().get_available_regions("sqs", partition_name="aws-us-gov"):
-    sqs_backends[region] = SQSBackend(region)
+    sqs_backends[region] = SQSBackend(region, config)
 for region in Session().get_available_regions("sqs", partition_name="aws-cn"):
-    sqs_backends[region] = SQSBackend(region)
+    sqs_backends[region] = SQSBackend(region, config)

moto/sqs/utils.py

@@ -1,8 +1,17 @@
 from __future__ import unicode_literals
+import os
 import random
 import string
+import moto
 
 from .exceptions import MessageAttributesInvalid
+from moto.settings import TEST_SERVER_MODE
+
+try:
+    import configparser
+except ModuleNotFoundError:
+    # why are you using python 2.7?
+    import ConfigParser as configparser
 
 
 def generate_receipt_handle():
@@ -49,11 +58,7 @@ def parse_message_attributes(querystring, base="", value_namespace="Value."):
             )
 
         data_type_parts = data_type[0].split(".")
-        if data_type_parts[0] not in [
-            "String",
-            "Binary",
-            "Number",
-        ]:
+        if data_type_parts[0] not in ["String", "Binary", "Number"]:
             raise MessageAttributesInvalid(
                 "The message attribute '{0}' has an invalid message attribute type, the set of supported type prefixes is Binary, Number, and String.".format(
                     name[0]
@@ -83,3 +88,17 @@ def parse_message_attributes(querystring, base="", value_namespace="Value."):
         index += 1
 
     return message_attributes
+
+
+def read_config():
+    section = os.getenv("MOTO_ENVIRONMENT", "production")
+    parser = configparser.ConfigParser()
+    filename = "configuration.ini"
+    config_path = os.path.join(os.getcwd(), "moto", filename)
+
+    if TEST_SERVER_MODE:
+        # in server mode the config resides in the package
+        config_path = os.path.join(os.path.dirname(moto.__file__), filename)
+
+    parser.read(config_path)
+    return parser[section]

tests/test_sqs/test_sqs.py

@@ -20,11 +20,7 @@
 from moto import mock_sqs, mock_sqs_deprecated, mock_lambda, mock_logs, settings
 from unittest import SkipTest
 
-if sys.version_info[0] < 3:
-    import mock
-    from unittest import SkipTest
-else:
-    from unittest import SkipTest, mock
+
 import pytest
 from tests.helpers import requires_boto_gte
 from tests.test_awslambda.test_lambda import get_test_zip_file1, get_role_name
@@ -54,8 +50,6 @@
 }
 """
 
-MOCK_DEDUPLICATION_TIME_IN_SECONDS = 5
-
 
 @mock_sqs
 def test_create_fifo_queue_fail():
@@ -2343,22 +2337,16 @@ def test_fifo_queue_deduplication_withoutid(msg_1, msg_2, expected_count):
     messages.should.have.length_of(expected_count)
 
 
-@mock.patch(
-    "moto.sqs.models.DEDUPLICATION_TIME_IN_SECONDS", MOCK_DEDUPLICATION_TIME_IN_SECONDS
-)
 @mock_sqs
 def test_fifo_queue_send_duplicate_messages_after_deduplication_time_limit():
-    if settings.TEST_SERVER_MODE:
-        raise SkipTest("Cant manipulate time in server mode")
-
     sqs = boto3.resource("sqs", region_name="us-east-1")
     msg_queue = sqs.create_queue(
         QueueName="test-queue-dlq.fifo",
         Attributes={"FifoQueue": "true", "ContentBasedDeduplication": "true"},
     )
 
     msg_queue.send_message(MessageBody="first", MessageGroupId="1")
-    time.sleep(MOCK_DEDUPLICATION_TIME_IN_SECONDS + 5)
+    time.sleep(5)  # in tests deduplication time is 1 second
     msg_queue.send_message(MessageBody="first", MessageGroupId="2")
     messages = msg_queue.receive_messages(MaxNumberOfMessages=2)
     messages.should.have.length_of(2)