Exception on extra fields

contentctl/contentctl.py

@@ -154,7 +154,7 @@ def main():
 
         else:
             #The file exists, so load it up!
-            config_obj = YmlReader().load_file(configFile)
+            config_obj = YmlReader().load_file(configFile,add_fields=False)
             t = test.model_validate(config_obj)
     except Exception as e:
         print(f"Error validating 'contentctl.yml':\n{str(e)}")

contentctl/objects/abstract_security_content_objects/security_content_object_abstract.py

@@ -33,8 +33,7 @@
 
 # TODO (#266): disable the use_enum_values configuration
 class SecurityContentObject_Abstract(BaseModel, abc.ABC):
-    model_config = ConfigDict(use_enum_values=True,validate_default=True)
-
+    model_config = ConfigDict(use_enum_values=True,validate_default=True,extra="forbid")
     name: str = Field(...,max_length=99)
     author: str = Field(...,max_length=255)
     date: datetime.date = Field(...)

contentctl/objects/alert_action.py

@@ -1,5 +1,5 @@
 from __future__ import annotations
-from pydantic import BaseModel, model_serializer
+from pydantic import BaseModel, model_serializer, ConfigDict
 from typing import Optional
 
 from contentctl.objects.deployment_email import DeploymentEmail
@@ -9,6 +9,7 @@
 from contentctl.objects.deployment_phantom import DeploymentPhantom
 
 class AlertAction(BaseModel):
+    model_config = ConfigDict(extra="forbid")
     email: Optional[DeploymentEmail] = None
     notable: Optional[DeploymentNotable] = None
     rba: Optional[DeploymentRBA] = DeploymentRBA()

contentctl/objects/atomic.py

@@ -41,6 +41,7 @@ class InputArgumentType(StrEnum):
     Url = "Url"
 
 class AtomicExecutor(BaseModel):
+    model_config = ConfigDict(extra="forbid")
     name: str
     elevation_required: Optional[bool] = False #Appears to be optional
     command: Optional[str] = None

contentctl/objects/base_test.py

@@ -2,7 +2,7 @@
 from typing import Union
 from abc import ABC, abstractmethod
 
-from pydantic import BaseModel
+from pydantic import BaseModel,ConfigDict
 
 from contentctl.objects.base_test_result import BaseTestResult
 
@@ -21,6 +21,7 @@ def __str__(self) -> str:
 
 # TODO (#224): enforce distinct test names w/in detections
 class BaseTest(BaseModel, ABC):
+    model_config = ConfigDict(extra="forbid")
     """
     A test case for a detection
     """

contentctl/objects/baseline.py

@@ -1,7 +1,7 @@
 
 from __future__ import annotations
-from typing import Annotated, Optional, List,Any
-from pydantic import field_validator, ValidationInfo, Field, model_serializer
+from typing import Annotated, List,Any
+from pydantic import field_validator, ValidationInfo, Field, model_serializer, computed_field
 from contentctl.objects.deployment import Deployment
 from contentctl.objects.security_content_object import SecurityContentObject
 from contentctl.objects.enums import DataModel
@@ -15,7 +15,6 @@
 class Baseline(SecurityContentObject):
     name:str = Field(...,max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH)
     type: Annotated[str,Field(pattern="^Baseline$")] = Field(...)
-    datamodel: Optional[List[DataModel]] = None
     search: str = Field(..., min_length=4)
     how_to_implement: str = Field(..., min_length=4)
     known_false_positives: str = Field(..., min_length=4)
@@ -34,6 +33,10 @@ def get_conf_stanza_name(self, app:CustomApp)->str:
     def getDeployment(cls, v:Any, info:ValidationInfo)->Deployment:
         return Deployment.getDeployment(v,info)
 
+    @computed_field
+    @property
+    def datamodel(self) -> List[DataModel]:
+        return [dm for dm in DataModel if dm.value in self.search]
 
     @model_serializer
     def serialize_model(self):

contentctl/objects/baseline_tags.py

@@ -1,5 +1,5 @@
 from __future__ import annotations
-from pydantic import BaseModel, Field, field_validator, ValidationInfo, model_serializer
+from pydantic import BaseModel, Field, field_validator, ValidationInfo, model_serializer, ConfigDict
 from typing import List, Any, Union
 
 from contentctl.objects.story import Story
@@ -12,6 +12,7 @@
 
 
 class BaselineTags(BaseModel):
+    model_config = ConfigDict(extra="forbid")
     analytic_story: list[Story] = Field(...)
     #deployment: Deployment = Field('SET_IN_GET_DEPLOYMENT_FUNCTION')
     # TODO (#223): can we remove str from the possible types here?

contentctl/objects/config.py

@@ -35,7 +35,7 @@
 
 # TODO (#266): disable the use_enum_values configuration
 class App_Base(BaseModel,ABC):
-    model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True)
+    model_config = ConfigDict(use_enum_values=True,validate_default=True, arbitrary_types_allowed=True, extra='forbid')
     uid: Optional[int] = Field(default=None)
     title: str = Field(description="Human-readable name used by the app. This can have special characters.")
     appid: Optional[APPID_TYPE]= Field(default=None,description="Internal name used by your app. "

contentctl/objects/data_source.py

@@ -1,24 +1,24 @@
 from __future__ import annotations
 from typing import Optional, Any
-from pydantic import Field, HttpUrl, model_serializer, BaseModel
+from pydantic import Field, HttpUrl, model_serializer, BaseModel, ConfigDict
 from contentctl.objects.security_content_object import SecurityContentObject
-from contentctl.objects.event_source import EventSource
 
 
 class TA(BaseModel):
     name: str
     url: HttpUrl | None = None
     version: str
 class DataSource(SecurityContentObject):
+    model_config = ConfigDict(extra="forbid")
     source: str = Field(...)
     sourcetype: str = Field(...)
     separator: Optional[str] = None
     configuration: Optional[str] = None
     supported_TA: list[TA] = []
-    fields: Optional[list] = None
-    field_mappings: Optional[list] = None
-    convert_to_log_source: Optional[list] = None
-    example_log: Optional[str] = None
+    fields: None | list = None
+    field_mappings: None | list = None
+    convert_to_log_source: None | list = None
+    example_log: None | str = None
 
 
     @model_serializer

contentctl/objects/deployment.py

@@ -1,5 +1,5 @@
 from __future__ import annotations
-from pydantic import Field, computed_field,ValidationInfo, model_serializer, NonNegativeInt
+from pydantic import Field, computed_field,ValidationInfo, model_serializer, NonNegativeInt, ConfigDict
 from typing import Any
 import uuid
 import datetime
@@ -11,6 +11,7 @@
 
 
 class Deployment(SecurityContentObject):
+    model_config = ConfigDict(extra="forbid")
     #id: str = None
     #date: str = None
     #author: str = None
@@ -72,7 +73,6 @@ def serialize_model(self):
             "tags": self.tags
         }
 
-
         #Combine fields from this model with fields from parent
         model.update(super_fields)
 

contentctl/objects/deployment_email.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 
 class DeploymentEmail(BaseModel):
+    model_config = ConfigDict(extra="forbid")
     message: str
     subject: str
     to: str

contentctl/objects/deployment_notable.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 from typing import List
 
 class DeploymentNotable(BaseModel):
+    model_config = ConfigDict(extra="forbid")
     rule_description: str
     rule_title: str
     nes_fields: List[str]

contentctl/objects/deployment_phantom.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 
 class DeploymentPhantom(BaseModel):
+    model_config = ConfigDict(extra="forbid")
     cam_workers : str
     label : str
     phantom_server : str

contentctl/objects/deployment_rba.py

@@ -1,6 +1,7 @@
 from __future__ import annotations
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 
 class DeploymentRBA(BaseModel):
+    model_config = ConfigDict(extra="forbid")
     enabled: bool = False

contentctl/objects/deployment_scheduling.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 
 class DeploymentScheduling(BaseModel):
+    model_config = ConfigDict(extra="forbid")
     cron_schedule: str
     earliest_time: str
     latest_time: str

contentctl/objects/deployment_slack.py

@@ -1,7 +1,8 @@
 from __future__ import annotations
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 
 
 class DeploymentSlack(BaseModel):
+    model_config = ConfigDict(extra="forbid")
     channel: str
     message: str

contentctl/objects/detection_tags.py

@@ -38,13 +38,13 @@
 # TODO (#266): disable the use_enum_values configuration
 class DetectionTags(BaseModel):
     # detection spec
-    model_config = ConfigDict(use_enum_values=True, validate_default=False)
+    model_config = ConfigDict(use_enum_values=True,validate_default=False, extra='forbid')
     analytic_story: list[Story] = Field(...)
     asset_type: AssetType = Field(...)
-
-    confidence: NonNegativeInt = Field(..., le=100)
-    impact: NonNegativeInt = Field(..., le=100)
-
+    group: list[str] = []
+    context: list[str] = []
+    confidence: NonNegativeInt = Field(...,le=100)
+    impact: NonNegativeInt = Field(...,le=100)
     @computed_field
     @property
     def risk_score(self) -> int:

contentctl/objects/investigation.py

@@ -16,13 +16,10 @@
 class Investigation(SecurityContentObject):
     model_config = ConfigDict(use_enum_values=True,validate_default=False)
     type: str = Field(...,pattern="^Investigation$")
-    datamodel: list[DataModel] = Field(...)
     name:str = Field(...,max_length=CONTENTCTL_MAX_SEARCH_NAME_LENGTH)
     search: str = Field(...)
     how_to_implement: str = Field(...)
     known_false_positives: str = Field(...)
-
-
     tags: InvestigationTags
 
     # enrichment
@@ -38,6 +35,11 @@ def inputs(self)->List[str]:
 
         return inputs
 
+    @computed_field
+    @property
+    def datamodel(self) -> List[DataModel]:
+        return [dm for dm in DataModel if dm.value in self.search]
+
     @computed_field
     @property
     def lowercase_name(self)->str:

contentctl/objects/investigation_tags.py

@@ -1,10 +1,11 @@
 from __future__ import annotations
 from typing import List
-from pydantic import BaseModel, Field, field_validator, ValidationInfo, model_serializer
+from pydantic import BaseModel, Field, field_validator, ValidationInfo, model_serializer,ConfigDict
 from contentctl.objects.story import Story
 from contentctl.objects.enums import SecurityContentInvestigationProductName, SecurityDomain
 
 class InvestigationTags(BaseModel):
+    model_config = ConfigDict(extra="forbid")
     analytic_story: List[Story] = Field([],min_length=1)
     product: List[SecurityContentInvestigationProductName] = Field(...,min_length=1)
     required_fields: List[str] = Field(min_length=1)

contentctl/objects/lookup.py

@@ -33,6 +33,7 @@ class Lookup(SecurityContentObject):
     default_match: Optional[bool] = None
     match_type: Optional[str] = None
     min_matches: Optional[int] = None
+    max_matches: Optional[int] = None
     case_sensitive_match: Optional[bool] = None
     # TODO: Add id field to all lookup ymls
     id: uuid.UUID = Field(default_factory=uuid.uuid4) 
@@ -52,6 +53,7 @@ def serialize_model(self):
             "default_match": "true" if self.default_match is True else "false",
             "match_type": self.match_type,
             "min_matches": self.min_matches,
+            "max_matches": self.max_matches,
             "case_sensitive_match": "true" if self.case_sensitive_match is True else "false",
             "collection": self.collection,
             "fields_list": self.fields_list

contentctl/objects/mitre_attack_enrichment.py

@@ -85,7 +85,7 @@ def standardize_contributors(cls, contributors:list[str] | None) -> list[str]:
 
 # TODO (#266): disable the use_enum_values configuration
 class MitreAttackEnrichment(BaseModel):
-    ConfigDict(use_enum_values=True)
+    ConfigDict(use_enum_values=True,extra='forbid')
     mitre_attack_id: MITRE_ATTACK_ID_TYPE = Field(...)
     mitre_attack_technique: str = Field(...)
     mitre_attack_tactics: List[MitreTactics] = Field(...)

contentctl/objects/observable.py

@@ -1,8 +1,9 @@
-from pydantic import BaseModel, field_validator
+from pydantic import BaseModel, field_validator, ConfigDict
 from contentctl.objects.constants import SES_OBSERVABLE_TYPE_MAPPING, RBA_OBSERVABLE_ROLE_MAPPING
 
 
 class Observable(BaseModel):
+    model_config = ConfigDict(extra="forbid")
     name: str
     type: str
     role: list[str]

contentctl/objects/playbook_tags.py

@@ -1,6 +1,6 @@
 from __future__ import annotations
 from typing import TYPE_CHECKING, Optional, List
-from pydantic import BaseModel, Field
+from pydantic import BaseModel, Field,ConfigDict
 import enum
 from contentctl.objects.detection import Detection
 
@@ -36,6 +36,7 @@ class DefendTechnique(str,enum.Enum):
     D3_SRA = "D3-SRA"
     D3_RUAA = "D3-RUAA"
 class PlaybookTag(BaseModel):
+    model_config = ConfigDict(extra="forbid")
     analytic_story: Optional[list] = None
     detections: Optional[list] = None
     platform_tags: list[str] = Field(...,min_length=0)
@@ -46,5 +47,8 @@ class PlaybookTag(BaseModel):
     use_cases: list[PlaybookUseCase] = Field([],min_length=0)
     defend_technique_id: Optional[List[DefendTechnique]] = None
 
+    labels:list[str] = []
+    playbook_outputs:list[str] = []
+
     detection_objects: list[Detection] = []
 

contentctl/objects/test_attack_data.py

@@ -1,8 +1,9 @@
 from __future__ import annotations
-from pydantic import BaseModel, HttpUrl, FilePath, Field
+from pydantic import BaseModel, HttpUrl, FilePath, Field, ConfigDict
 
 
 class TestAttackData(BaseModel):
+    model_config = ConfigDict(extra="forbid")
     data: HttpUrl | FilePath = Field(...)
     # TODO - should source and sourcetype should be mapped to a list
     # of supported source and sourcetypes in a given environment?

contentctl/objects/unit_test_baseline.py

@@ -1,9 +1,10 @@
 
 
-from pydantic import BaseModel
+from pydantic import BaseModel,ConfigDict
 from typing import Union
 
 class UnitTestBaseline(BaseModel):
+    model_config = ConfigDict(extra="forbid")
     name: str
     file: str
     pass_condition: str

contentctl/output/data_source_writer.py

@@ -1,6 +1,5 @@
 import csv
 from contentctl.objects.data_source import DataSource
-from contentctl.objects.event_source import EventSource
 from typing import List
 import pathlib