From 218cb36e0aa1d14cd33ad45bc898a1c1cf76dab4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jim=20Br=C3=A4nnlund?= Date: Mon, 14 Dec 2020 00:42:45 +0100 Subject: [PATCH] RFC/Draft for configuration utility --- .travis.yml | 11 ++++++++++- MANIFEST.in | 1 + Makefile | 4 ++-- moto/configuration.ini | 5 +++++ moto/sqs/models.py | 28 ++++++++++++++++------------ moto/sqs/utils.py | 29 ++++++++++++++++++++++++----- tests/test_sqs/test_sqs.py | 16 ++-------------- 7 files changed, 60 insertions(+), 34 deletions(-) create mode 100644 moto/configuration.ini diff --git a/.travis.yml b/.travis.yml index 824eb0edcf66..677b84c578c8 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,16 @@ install: else PYTHON_DOCKER_TAG=${TRAVIS_PYTHON_VERSION}-stretch fi - docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:${PYTHON_DOCKER_TAG} /moto/travis_moto_server.sh & + docker run --rm -t \ + --name motoserver \ + -e MOTO_ENVIRONMENT=development \ + -e TEST_SERVER_MODE=true \ + -e AWS_SECRET_ACCESS_KEY=server_secret \ + -e AWS_ACCESS_KEY_ID=server_key \ + -v `pwd`:/moto \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -p 5000:5000 \ + python:${PYTHON_DOCKER_TAG} /moto/travis_moto_server.sh & fi travis_retry pip install -r requirements-dev.txt travis_retry pip install docker>=2.5.1 diff --git a/MANIFEST.in b/MANIFEST.in index 9e3dc7cde0c1..74697c4d4707 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -5,5 +5,6 @@ include moto/ec2/resources/instance_type_offerings/*/*.json include moto/ec2/resources/amis.json include moto/cognitoidp/resources/*.json include moto/dynamodb2/parsing/reserved_keywords.txt +include moto/configuration.ini recursive-include moto/templates * recursive-include tests * diff --git a/Makefile b/Makefile index b155b6f8eb03..9c12e417adaa 100644 --- a/Makefile +++ b/Makefile @@ -26,12 +26,12 @@ format: test-only: rm -f .coverage rm -rf cover - @pytest -sv --cov=moto --cov-report html ./tests/ $(TEST_EXCLUDE) + @MOTO_ENVIRONMENT=development pytest -sv --cov=moto --cov-report html ./tests/ $(TEST_EXCLUDE) test: lint test-only test_server: - @TEST_SERVER_MODE=true pytest -sv --cov=moto --cov-report html ./tests/ + @TEST_SERVER_MODE=true @MOTO_ENVIRONMENT=development pytest -sv --cov=moto --cov-report html ./tests/ aws_managed_policies: scripts/update_managed_policies.py diff --git a/moto/configuration.ini b/moto/configuration.ini new file mode 100644 index 000000000000..e9185892d07c --- /dev/null +++ b/moto/configuration.ini @@ -0,0 +1,5 @@ +[production] +DeduplicationTimeInSeconds = 300 + +[development] +DeduplicationTimeInSeconds = 1 diff --git a/moto/sqs/models.py b/moto/sqs/models.py index 421a1e5c7951..1c57166fe7ef 100644 --- a/moto/sqs/models.py +++ b/moto/sqs/models.py @@ -20,7 +20,7 @@ unix_time_millis, tags_from_cloudformation_tags_list, ) -from .utils import generate_receipt_handle +from .utils import generate_receipt_handle, read_config from .exceptions import ( MessageAttributesInvalid, MessageNotInflight, @@ -63,8 +63,6 @@ # https://docs.aws.amazon.com/AWSSimpleQueueService/latest/SQSDeveloperGuide/sqs-message-metadata.html ATTRIBUTE_NAME_PATTERN = re.compile("^([a-z]|[A-Z]|[0-9]|[_.\\-])+$") -DEDUPLICATION_TIME_IN_SECONDS = 300 - class Message(BaseModel): def __init__(self, message_id, body): @@ -231,9 +229,10 @@ class Queue(CloudFormationModel): "SendMessage", ) - def __init__(self, name, region, **kwargs): + def __init__(self, name, region, config, **kwargs): self.name = name self.region = region + self.config = config self.tags = {} self.permissions = {} @@ -489,7 +488,7 @@ def add_message(self, message): diff = message.sent_timestamp - m.sent_timestamp # if a duplicate message is received within the deduplication time then it should # not be added to the queue - if diff / 1000 < DEDUPLICATION_TIME_IN_SECONDS: + if diff / 1000 < int(self.config["DeduplicationTimeInSeconds"]): return self._messages.append(message) @@ -561,16 +560,18 @@ def _filter_message_attributes(message, input_message_attributes): class SQSBackend(BaseBackend): - def __init__(self, region_name): + def __init__(self, region_name, config): self.region_name = region_name + self.config = config self.queues = {} super(SQSBackend, self).__init__() def reset(self): region_name = self.region_name + config = self.config self._reset_model_refs() self.__dict__ = {} - self.__init__(region_name) + self.__init__(region_name, config) def create_queue(self, name, tags=None, **kwargs): queue = self.queues.get(name) @@ -580,7 +581,9 @@ def create_queue(self, name, tags=None, **kwargs): except KeyError: pass - new_queue = Queue(name, region=self.region_name, **kwargs) + new_queue = Queue( + name, region=self.region_name, config=self.config, **kwargs + ) queue_attributes = queue.attributes new_queue_attributes = new_queue.attributes @@ -595,7 +598,7 @@ def create_queue(self, name, tags=None, **kwargs): kwargs.pop("region") except KeyError: pass - queue = Queue(name, region=self.region_name, **kwargs) + queue = Queue(name, region=self.region_name, config=self.config, **kwargs) self.queues[name] = queue if tags: @@ -1008,9 +1011,10 @@ def list_queue_tags(self, queue_name): sqs_backends = {} +config = read_config() for region in Session().get_available_regions("sqs"): - sqs_backends[region] = SQSBackend(region) + sqs_backends[region] = SQSBackend(region, config) for region in Session().get_available_regions("sqs", partition_name="aws-us-gov"): - sqs_backends[region] = SQSBackend(region) + sqs_backends[region] = SQSBackend(region, config) for region in Session().get_available_regions("sqs", partition_name="aws-cn"): - sqs_backends[region] = SQSBackend(region) + sqs_backends[region] = SQSBackend(region, config) diff --git a/moto/sqs/utils.py b/moto/sqs/utils.py index 876d6b40ef35..f648eadf3106 100644 --- a/moto/sqs/utils.py +++ b/moto/sqs/utils.py @@ -1,8 +1,17 @@ from __future__ import unicode_literals +import os import random import string +import moto from .exceptions import MessageAttributesInvalid +from moto.settings import TEST_SERVER_MODE + +try: + import configparser +except ModuleNotFoundError: + # why are you using python 2.7? + import ConfigParser as configparser def generate_receipt_handle(): @@ -49,11 +58,7 @@ def parse_message_attributes(querystring, base="", value_namespace="Value."): ) data_type_parts = data_type[0].split(".") - if data_type_parts[0] not in [ - "String", - "Binary", - "Number", - ]: + if data_type_parts[0] not in ["String", "Binary", "Number"]: raise MessageAttributesInvalid( "The message attribute '{0}' has an invalid message attribute type, the set of supported type prefixes is Binary, Number, and String.".format( name[0] @@ -83,3 +88,17 @@ def parse_message_attributes(querystring, base="", value_namespace="Value."): index += 1 return message_attributes + + +def read_config(): + section = os.getenv("MOTO_ENVIRONMENT", "production") + parser = configparser.ConfigParser() + filename = "configuration.ini" + config_path = os.path.join(os.getcwd(), "moto", filename) + + if TEST_SERVER_MODE: + # in server mode the config resides in the package + config_path = os.path.join(os.path.dirname(moto.__file__), filename) + + parser.read(config_path) + return parser[section] diff --git a/tests/test_sqs/test_sqs.py b/tests/test_sqs/test_sqs.py index b7cbc793a98c..a652ad2a2226 100644 --- a/tests/test_sqs/test_sqs.py +++ b/tests/test_sqs/test_sqs.py @@ -20,11 +20,7 @@ from moto import mock_sqs, mock_sqs_deprecated, mock_lambda, mock_logs, settings from unittest import SkipTest -if sys.version_info[0] < 3: - import mock - from unittest import SkipTest -else: - from unittest import SkipTest, mock + import pytest from tests.helpers import requires_boto_gte from tests.test_awslambda.test_lambda import get_test_zip_file1, get_role_name @@ -54,8 +50,6 @@ } """ -MOCK_DEDUPLICATION_TIME_IN_SECONDS = 5 - @mock_sqs def test_create_fifo_queue_fail(): @@ -2343,14 +2337,8 @@ def test_fifo_queue_deduplication_withoutid(msg_1, msg_2, expected_count): messages.should.have.length_of(expected_count) -@mock.patch( - "moto.sqs.models.DEDUPLICATION_TIME_IN_SECONDS", MOCK_DEDUPLICATION_TIME_IN_SECONDS -) @mock_sqs def test_fifo_queue_send_duplicate_messages_after_deduplication_time_limit(): - if settings.TEST_SERVER_MODE: - raise SkipTest("Cant manipulate time in server mode") - sqs = boto3.resource("sqs", region_name="us-east-1") msg_queue = sqs.create_queue( QueueName="test-queue-dlq.fifo", @@ -2358,7 +2346,7 @@ def test_fifo_queue_send_duplicate_messages_after_deduplication_time_limit(): ) msg_queue.send_message(MessageBody="first", MessageGroupId="1") - time.sleep(MOCK_DEDUPLICATION_TIME_IN_SECONDS + 5) + time.sleep(5) # in tests deduplication time is 1 second msg_queue.send_message(MessageBody="first", MessageGroupId="2") messages = msg_queue.receive_messages(MaxNumberOfMessages=2) messages.should.have.length_of(2)