SQS Batched CloudAux watcher

Author: Mike Grima

docs/iam_aws.md

@@ -204,7 +204,9 @@ You will need to create this role in all AWS accounts that you want to monitor.
                     "sns:listsubscriptionsbytopic",
                     "sns:listtopics",
                     "sqs:getqueueattributes",
-                    "sqs:listqueues"
+                    "sqs:listqueues",
+                    "sqs:listqueuetags", 
+                    "sqs:listdeadlettersourcequeues"
                 ],
                 "Effect": "Allow",
                 "Resource": "*"

docs/quickstart.md

@@ -1,6 +1,13 @@
 Quick Start Guide
 =================
 
+What does the Security Monkey architecture look like?
+---------------------------
+Security Monkey operates in a hub-spoke type of model where Security Monkey lives in one account,
+but then "reaches into" other accounts to describe and collect details.
+
+More details on this is outlined in the IAM section below for each respective infrastructure.
+
 Setup on AWS, GCP, or OpenStack
 -------------------------------
 

scripts/secmonkey_role_setup.py

@@ -169,7 +169,9 @@
            "sns:listsubscriptionsbytopic",
            "sns:listtopics",
            "sqs:getqueueattributes",
-           "sqs:listqueues"
+           "sqs:listqueues",
+           "sqs:listqueuetags", 
+           "sqs:listdeadlettersourcequeues"
       ],
       "Effect": "Allow",
       "Resource": "*"

security_monkey/cloudaux_batched_watcher.py

@@ -1,3 +1,4 @@
+from security_monkey import app
 from security_monkey.cloudaux_watcher import CloudAuxWatcher
 from security_monkey.cloudaux_watcher import CloudAuxChangeItem
 from security_monkey.decorators import record_exception
@@ -55,6 +56,11 @@ def slurp_items(**kwargs):
                         self.done_slurping = True
                     continue
 
+                app.logger.debug("Account: {account}, Batched Watcher: {watcher}, Fetching item: "
+                                 "{item}/{region}".format(account=kwargs["account_name"],
+                                                          watcher=self.index,
+                                                          item=item_name,
+                                                          region=kwargs["conn_dict"]["region"]))
                 item_details = invoke_get_method(cursor, name=item_name, **kwargs)
                 if item_details:
                     # Determine which region to record the item into.

security_monkey/watchers/sqs.py

@@ -1,4 +1,4 @@
-#     Copyright 2014 Netflix, Inc.
+#     Copyright 2018 Netflix, Inc.
 #
 #     Licensed under the Apache License, Version 2.0 (the "License");
 #     you may not use this file except in compliance with the License.
@@ -16,104 +16,39 @@
     :platform: Unix
 
 .. version:: $$VERSION$$
-.. moduleauthor:: Patrick Kelley <[email protected]> @monkeysecurity
-
+.. moduleauthor:: Mike Grima <[email protected]>
 """
+from cloudaux.aws.sqs import list_queues
+from cloudaux.orchestration.aws.sqs import get_queue
 
-from security_monkey.watcher import Watcher
-from security_monkey.watcher import ChangeItem
-from security_monkey.constants import TROUBLE_REGIONS
-from security_monkey.exceptions import InvalidAWSJSON
-from security_monkey.exceptions import BotoConnectionIssue
-from security_monkey.datastore import Account
-from security_monkey import app, ARN_PREFIX
-
-import json
-import boto
-from boto.sqs import regions
+from security_monkey.cloudaux_batched_watcher import CloudAuxBatchedWatcher
 
 
-class SQS(Watcher):
+class SQS(CloudAuxBatchedWatcher):
     index = 'sqs'
     i_am_singular = 'SQS Policy'
     i_am_plural = 'SQS Policies'
 
-    def __init__(self, accounts=None, debug=False):
-        super(SQS, self).__init__(accounts=accounts, debug=debug)
+    def __init__(self, **kwargs):
+        super(SQS, self).__init__(**kwargs)
         self.honor_ephemerals = True
         self.ephemeral_paths = [
-            'LastModifiedTimestamp',
-            'ApproximateNumberOfMessagesNotVisible',
-            'ApproximateNumberOfMessages',
-            'ApproximateNumberOfMessagesDelayed']
-
-    def slurp(self):
-        """
-        :returns: item_list - list of SQS Policies.
-        :returns: exception_map - A dict where the keys are a tuple containing the
-            location of the exception and the value is the actual exception
-
-        """
-        self.prep_for_slurp()
-
-        item_list = []
-        exception_map = {}
-        from security_monkey.common.sts_connect import connect
-        for account in self.accounts:
-            account_db = Account.query.filter(Account.name == account).first()
-            account_number = account_db.identifier
-            for region in regions():
-                app.logger.debug("Checking {}/{}/{}".format(SQS.index, account, region.name))
-                try:
-                    sqs = connect(account, 'sqs', region=region)
-                    all_queues = self.wrap_aws_rate_limited_call(
-                        sqs.get_all_queues
-                    )
-                except Exception as e:
-                    if region.name not in TROUBLE_REGIONS:
-                        exc = BotoConnectionIssue(str(e), 'sqs', account, region.name)
-                        self.slurp_exception((self.index, account, region.name), exc, exception_map,
-                                             source="{}-watcher".format(self.index))
-                    continue
-                app.logger.debug("Found {} {}".format(len(all_queues), SQS.i_am_plural))
-                for q in all_queues:
-
-                    if self.check_ignore_list(q.name):
-                        continue
-
-                    try:
-                        attrs = self.wrap_aws_rate_limited_call(
-                            q.get_attributes,
-                            attributes='All'
-                        )
+            '_version',
+            'Attributes$*$LastModifiedTimestamp',
+            'Attributes$*$ApproximateNumberOfMessagesNotVisible',
+            'Attributes$*$ApproximateNumberOfMessages',
+            'Attributes$*$ApproximateNumberOfMessagesDelayed'
+        ]
+        self.batched_size = 200
 
-                        try:
-                            if 'Policy' in attrs:
-                                json_str = attrs['Policy']
-                                attrs['Policy'] = json.loads(json_str)
-                            else:
-                                attrs['Policy'] = {}
+    def get_name_from_list_output(self, item):
+        # SQS returns URLs. Need to deconstruct the URL to pull out the name :/
+        name = item.split("{}/".format(self.account_identifiers[0]))[1]
 
-                            item = SQSItem(region=region.name, account=account, name=q.name, arn=attrs['QueueArn'],
-                                           config=dict(attrs), source_watcher=self)
-                            item_list.append(item)
-                        except:
-                            self.slurp_exception((self.index, account, region, q.name), InvalidAWSJSON(json_str),
-                                                 exception_map, source="{}-watcher".format(self.index))
-                    except boto.exception.SQSError:
-                        # A number of Queues are so ephemeral that they may be gone by the time
-                        # the code reaches here.  Just ignore them and move on.
-                        pass
-        return item_list, exception_map
+        return name
 
+    def list_method(self, **kwargs):
+        return list_queues(**kwargs)
 
-class SQSItem(ChangeItem):
-    def __init__(self, region=None, account=None, name=None, arn=None, config=None, source_watcher=None):
-        super(SQSItem, self).__init__(
-            index=SQS.index,
-            region=region,
-            account=account,
-            name=name,
-            arn=arn,
-            new_config=config if config else {},
-            source_watcher=source_watcher)
+    def get_method(self, item, **kwargs):
+        return get_queue(item, **kwargs)