Forbid extra fields in YMLs

contentctl/actions/build.py

@@ -51,7 +51,9 @@ def execute(self, input_dto: BuildInputDto) -> DirectorOutputDto:
             updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.lookups, SecurityContentType.lookups))
             updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.macros, SecurityContentType.macros))
             updated_conf_files.update(conf_output.writeObjects(input_dto.director_output_dto.dashboards, SecurityContentType.dashboards))
-            updated_conf_files.update(conf_output.writeAppConf())
+            updated_conf_files.update(conf_output.writeMiscellaneousAppFiles())
+
+
 
             #Ensure that the conf file we just generated/update is syntactically valid
             for conf_file in updated_conf_files:

contentctl/actions/deploy_acs.py

@@ -1,38 +1,55 @@
-from dataclasses import dataclass
-from contentctl.input.director import DirectorInputDto
-from contentctl.output.conf_output import ConfOutput
-
-
-from typing import Union
-
-@dataclass(frozen=True)
-class ACSDeployInputDto:
-    director_input_dto: DirectorInputDto
-    splunk_api_username: str
-    splunk_api_password: str
-    splunk_cloud_jwt_token: str
-    splunk_cloud_stack: str
-    stack_type: str
+from contentctl.objects.config import deploy_acs, StackType
+from requests import post
+import pprint
 
 
 class Deploy:
-    def execute(self, input_dto: ACSDeployInputDto) -> None:
-
-        conf_output = ConfOutput(input_dto.director_input_dto.input_path, input_dto.director_input_dto.config)
+    def execute(self, config: deploy_acs, appinspect_token:str) -> None:
 
-        appinspect_token = conf_output.inspectAppAPI(input_dto.splunk_api_username, input_dto.splunk_api_password, input_dto.stack_type)
+        #The following common headers are used by both Clasic and Victoria
+        headers = {
+            'Authorization': f'Bearer {config.splunk_cloud_jwt_token}',
+            'ACS-Legal-Ack': 'Y'
+        }
+        try:
+
+            with open(config.getPackageFilePath(include_version=False),'rb') as app_data:
+                #request_data = app_data.read()
+                if config.stack_type == StackType.classic:
+                    # Classic instead uses a form to store token and package
+                    # https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Classic_Experience
+                    address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps"
+
+                    form_data = {
+                        'token': (None, appinspect_token),
+                        'package': app_data
+                    }
+                    res = post(address, headers=headers, files = form_data)
+                elif config.stack_type == StackType.victoria:
+                    # Victoria uses the X-Splunk-Authorization Header
+                    # It also uses --data-binary for the app content
+                    # https://docs.splunk.com/Documentation/SplunkCloud/9.1.2308/Config/ManageApps#Manage_private_apps_using_the_ACS_API_on_Victoria_Experience
+                    headers.update({'X-Splunk-Authorization':  appinspect_token})
+                    address = f"https://admin.splunk.com/{config.splunk_cloud_stack}/adminconfig/v2/apps/victoria"
+                    res = post(address, headers=headers, data=app_data.read())
+                else:
+                    raise Exception(f"Unsupported stack type: '{config.stack_type}'")
+        except Exception as e:
+            raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{str(e)}")
 
-
-        if input_dto.splunk_cloud_jwt_token is None or input_dto.splunk_cloud_stack is None:
-            if input_dto.splunk_cloud_jwt_token is None:
-                raise Exception("Cannot deploy app via ACS, --splunk_cloud_jwt_token was not defined on command line.")
-            else:
-                raise Exception("Cannot deploy app via ACS, --splunk_cloud_stack was not defined on command line.")
-
-        conf_output.deploy_via_acs(input_dto.splunk_cloud_jwt_token,
-                                input_dto.splunk_cloud_stack, 
-                                appinspect_token,
-                                input_dto.stack_type)
-
+        try:
+            # Request went through and completed, but may have returned a non-successful error code.
+            # This likely includes a more verbose response describing the error
+            res.raise_for_status()
+            print(res.json())
+        except Exception as e:
+            try:
+                error_text = res.json()
+            except Exception as e:
+                error_text = "No error text - request failed"
+            formatted_error_text = pprint.pformat(error_text)
+            print("While this may not be the cause of your error, ensure that the uid and appid of your Private App does not exist in Splunkbase\n"
+                  "ACS cannot deploy and app with the same uid or appid as one that exists in Splunkbase.")
+            raise Exception(f"Error installing to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS:\n{formatted_error_text}")
 
-
+        print(f"'{config.getPackageFilePath(include_version=False)}' successfully installed to stack '{config.splunk_cloud_stack}' (stack_type='{config.stack_type}') via ACS!")

contentctl/actions/detection_testing/GitService.py

@@ -13,6 +13,7 @@
 from contentctl.objects.macro import Macro
 from contentctl.objects.lookup import Lookup
 from contentctl.objects.detection import Detection
+from contentctl.objects.data_source import DataSource
 from contentctl.objects.security_content_object import SecurityContentObject
 from contentctl.objects.config import test_common, All, Changes, Selected
 
@@ -67,9 +68,12 @@ def getChanges(self, target_branch:str)->List[Detection]:
 
         #Make a filename to content map
         filepath_to_content_map = { obj.file_path:obj for (_,obj) in self.director.name_to_content_map.items()} 
-        updated_detections:List[Detection] = []
-        updated_macros:List[Macro] = []
-        updated_lookups:List[Lookup] =[]
+
+        updated_detections: set[Detection] = set()
+        updated_macros: set[Macro] = set()
+        updated_lookups: set[Lookup] = set()
+        updated_datasources: set[DataSource] = set()
+
 
         for diff in all_diffs:
             if type(diff) == pygit2.Patch:
@@ -80,16 +84,23 @@ def getChanges(self, target_branch:str)->List[Detection]:
                     if decoded_path.is_relative_to(self.config.path/"detections") and decoded_path.suffix == ".yml":
                         detectionObject = filepath_to_content_map.get(decoded_path, None)
                         if isinstance(detectionObject, Detection):
-                            updated_detections.append(detectionObject)
+                            updated_detections.add(detectionObject)
                         else:
                             raise Exception(f"Error getting detection object for file {str(decoded_path)}")
 
                     elif decoded_path.is_relative_to(self.config.path/"macros") and decoded_path.suffix == ".yml":
                         macroObject = filepath_to_content_map.get(decoded_path, None)
                         if isinstance(macroObject, Macro):
-                            updated_macros.append(macroObject)
+                            updated_macros.add(macroObject)
                         else:
                             raise Exception(f"Error getting macro object for file {str(decoded_path)}")
+
+                    elif decoded_path.is_relative_to(self.config.path/"data_sources") and decoded_path.suffix == ".yml":
+                        datasourceObject = filepath_to_content_map.get(decoded_path, None)
+                        if isinstance(datasourceObject, DataSource):
+                            updated_datasources.add(datasourceObject)
+                        else:
+                            raise Exception(f"Error getting data source object for file {str(decoded_path)}")
 
                     elif decoded_path.is_relative_to(self.config.path/"lookups"):
                         # We need to convert this to a yml. This means we will catch
@@ -98,7 +109,7 @@ def getChanges(self, target_branch:str)->List[Detection]:
                             updatedLookup = filepath_to_content_map.get(decoded_path, None)
                             if not isinstance(updatedLookup,Lookup):
                                 raise Exception(f"Expected {decoded_path} to be type {type(Lookup)}, but instead if was {(type(updatedLookup))}")
-                            updated_lookups.append(updatedLookup)
+                            updated_lookups.add(updatedLookup)
 
                         elif decoded_path.suffix == ".csv":
                             # If the CSV was updated, we want to make sure that we 
@@ -115,7 +126,6 @@ def getChanges(self, target_branch:str)->List[Detection]:
                             # Detected a changed .mlmodel file. However, since we do not have testing for these detections at 
                             # this time, we will ignore this change.
                             updatedLookup = None
-
 
                         else:
                             raise Exception(f"Detected a changed file in the lookups/ directory '{str(decoded_path)}'.\n"
@@ -125,7 +135,7 @@ def getChanges(self, target_branch:str)->List[Detection]:
                         if updatedLookup is not None and updatedLookup not in updated_lookups:
                             # It is possible that both the CSV and YML have been modified for the same lookup,
                             # and we do not want to add it twice. 
-                            updated_lookups.append(updatedLookup)
+                            updated_lookups.add(updatedLookup)
 
                     else:
                         pass
@@ -136,24 +146,25 @@ def getChanges(self, target_branch:str)->List[Detection]:
 
         # If a detection has at least one dependency on changed content,
         # then we must test it again
-        changed_macros_and_lookups = updated_macros + updated_lookups
+
+        changed_macros_and_lookups_and_datasources:set[SecurityContentObject] = updated_macros.union(updated_lookups, updated_datasources)
 
         for detection in self.director.detections:
             if detection in updated_detections:
                 # we are already planning to test it, don't need 
                 # to add it again
                 continue
 
-            for obj in changed_macros_and_lookups:
+            for obj in changed_macros_and_lookups_and_datasources:
                 if obj in detection.get_content_dependencies():
-                   updated_detections.append(detection)
+                   updated_detections.add(detection)
                    break
 
         #Print out the names of all modified/new content
         modifiedAndNewContentString = "\n - ".join(sorted([d.name for d in updated_detections]))
 
         print(f"[{len(updated_detections)}] Pieces of modifed and new content (this may include experimental/deprecated/manual_test content):\n - {modifiedAndNewContentString}")
-        return updated_detections
+        return sorted(list(updated_detections))
 
     def getSelected(self, detectionFilenames: List[FilePath]) -> List[Detection]:
         filepath_to_content_map: dict[FilePath, SecurityContentObject] = {