refactor(user_actions): Refactor user actions to link to filing and s…

…ubmission directly.

Remove direct links from FilingDAO and SubmissionDAO.  Update DTOs to make existing links to user_action entities be derived.  Update API accordingly.

closes cfpb#551

db_revisions/versions/ef815604e2a9_user_action_add_filing_submission_fks.py

@@ -0,0 +1,116 @@
+"""user_action add filing submission fks
+
+Revision ID: ef815604e2a9
+Revises: 6ec12afa5b37
+Create Date: 2025-01-24 10:15:32.975738
+
+"""
+
+from typing import Sequence, Union
+
+from alembic import op
+import sqlalchemy as sa
+
+from sbl_filing_api.entities.models.model_enums import UserActionType
+
+# revision identifiers, used by Alembic.
+revision: str = "ef815604e2a9"
+down_revision: Union[str, None] = "6ec12afa5b37"
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = None
+
+
+def upgrade() -> None:
+    # Add links to Filing and Submission to user_action
+    with op.batch_alter_table("user_action", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("filing_id", sa.Integer, nullable=True))
+        batch_op.create_foreign_key("user_action_filing_fkey", "filing", ["filing_id"], ["id"])
+        batch_op.add_column(sa.Column("submission_id", sa.Integer, nullable=True))
+        batch_op.create_foreign_key("user_action_submission_fkey", "submission", ["submission_id"], ["id"])
+    # Update links with existing info
+    op.execute(
+        """
+        UPDATE user_action 
+        SET filing_id = COALESCE(
+            (SELECT id FROM filing WHERE creator_id = user_action.id), 
+            (SELECT filing FROM filing_signature WHERE user_action = user_action.id),
+            (SELECT filing FROM submission WHERE submitter_id = user_action.id OR accepter_id = user_action.id)
+        )
+    """
+    )
+    op.execute(
+        """
+        UPDATE user_action 
+        SET submission_id = (SELECT id FROM submission WHERE submitter_id = user_action.id OR accepter_id = user_action.id) 
+    """
+    )
+    # Index the foreign keys
+    op.create_index("user_action_filing_id", table_name="user_action", columns=["filing_id"])
+    op.create_index("user_action_submission_id", table_name="user_action", columns=["submission_id"])
+    with op.batch_alter_table("user_action", schema=None) as batch_op:
+        batch_op.alter_column("filing_id", nullable=False)
+        batch_op.alter_column("submission_id", nullable=True)
+    # Drop old table and columns no longer needed
+    op.drop_table("filing_signature")
+    with op.batch_alter_table("filing", schema=None) as batch_op:
+        batch_op.drop_column("creator_id")
+    with op.batch_alter_table("submission", schema=None) as batch_op:
+        batch_op.drop_column("submitter_id")
+        batch_op.drop_column("accepter_id")
+
+
+def downgrade() -> None:
+    # recreate the filing_signature table
+    op.create_table(
+        "filing_signature",
+        sa.Column("user_action", sa.INTEGER, primary_key=True, unique=True, nullable=False),
+        sa.Column("filing", sa.Integer, nullable=False),
+        sa.PrimaryKeyConstraint("user_action", name="filing_signatures_pkey"),
+        sa.ForeignKeyConstraint(["user_action"], ["user_action.id"], name="filing_signatures_user_action_fkey"),
+        sa.ForeignKeyConstraint(["filing"], ["filing.id"], name="filing_signatures_filing_fkey"),
+    )
+    # re-add link columns
+    with op.batch_alter_table("filing", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("creator_id", sa.Integer))
+        batch_op.create_foreign_key("filing_creator_fkey", "user_action", ["creator_id"], ["id"])
+
+    with op.batch_alter_table("submission", schema=None) as batch_op:
+        batch_op.add_column(sa.Column("submitter_id", sa.Integer))
+        batch_op.add_column(sa.Column("accepter_id", sa.Integer))
+        batch_op.create_foreign_key("submission_submitter_fkey", "user_action", ["submitter_id"], ["id"])
+        batch_op.create_foreign_key("submission_accepter_fkey", "user_action", ["accepter_id"], ["id"])
+
+    # re-populate data
+    op.execute(
+        f"""
+        INSERT INTO filing_signature (user_action, filing)
+        SELECT id, filing_id FROM user_action WHERE action_type = '{UserActionType.SIGN}' 
+    """
+    )
+    op.execute(
+        f"""
+        UPDATE submission
+        SET submitter_id = (SELECT id FROM user_action WHERE submission.id = user_action.submission_id AND action_type = '{UserActionType.SUBMIT}'),
+        accepter_id = (SELECT id FROM user_action WHERE submission.id = user_action.submission_id AND action_type = '{UserActionType.ACCEPT}')
+    """
+    )
+    op.execute(
+        f"""
+        UPDATE filing
+        SET creator_id = (SELECT id FROM user_action WHERE filing.id = user_action.filing_id AND action_type = '{UserActionType.CREATE}')
+    """
+    )
+
+    # Make columns non-nullable
+    with op.batch_alter_table("filing", schema=None) as batch_op:
+        batch_op.alter_column("creator_id", nullable=False)
+    with op.batch_alter_table("submission", schema=None) as batch_op:
+        batch_op.alter_column("submitter_id", nullable=False)
+
+    # Drop new columns
+    op.drop_index("user_action_filing_id", table_name="user_action")
+    op.drop_index("user_action_submission_id", table_name="user_action")
+    op.drop_constraint(constraint_name="user_action_filing_fkey", table_name="user_action")
+    op.drop_column("user_action", "filing_id")
+    op.drop_constraint(constraint_name="user_action_submission_fkey", table_name="user_action")
+    op.drop_column("user_action", "submission_id")

src/sbl_filing_api/entities/models/dao.py

@@ -15,6 +15,8 @@ class Base(AsyncAttrs, DeclarativeBase):
 class UserActionDAO(Base):
     __tablename__ = "user_action"
     id: Mapped[int] = mapped_column(index=True, primary_key=True, autoincrement=True)
+    filing_id: Mapped[int] = mapped_column(ForeignKey("filing.id"))
+    submission_id: Mapped[int | None] = mapped_column(ForeignKey("submission.id"), nullable=True)
     user_id: Mapped[str] = mapped_column(String(36))
     user_name: Mapped[str] = mapped_column(String(255))
     user_email: Mapped[str] = mapped_column(String(255))
@@ -27,16 +29,15 @@ class SubmissionDAO(Base):
     id: Mapped[int] = mapped_column(index=True, primary_key=True, autoincrement=True)
     filing: Mapped[int] = mapped_column(ForeignKey("filing.id"))
     counter: Mapped[int]
-    submitter_id: Mapped[int] = mapped_column(ForeignKey("user_action.id"))
-    submitter: Mapped[UserActionDAO] = relationship(lazy="selectin", foreign_keys=[submitter_id])
-    accepter_id: Mapped[int] = mapped_column(ForeignKey("user_action.id"), nullable=True)
-    accepter: Mapped[UserActionDAO] = relationship(lazy="selectin", foreign_keys=[accepter_id])
     state: Mapped[SubmissionState] = mapped_column(SAEnum(SubmissionState))
     validation_ruleset_version: Mapped[str] = mapped_column(nullable=True)
     validation_results: Mapped[dict[str, Any]] = mapped_column(JSON, nullable=True)
     submission_time: Mapped[datetime] = mapped_column(server_default=func.now())
     filename: Mapped[str]
     total_records: Mapped[int] = mapped_column(nullable=True)
+    user_actions: Mapped[List[UserActionDAO] | None] = relationship(
+        "UserActionDAO", lazy="selectin", cascade="all, delete-orphan"
+    )
 
     __table_args__ = (UniqueConstraint("filing", "counter", name="unique_filing_counter"),)
 
@@ -99,14 +100,6 @@ def __str__(self):
         return f"ContactInfo ID: {self.id}, First Name: {self.first_name}, Last Name: {self.last_name}, Address Street 1: {self.hq_address_street_1}, Address Street 2: {self.hq_address_street_2}, Address City: {self.hq_address_city}, Address State: {self.hq_address_state}, Address Zip: {self.hq_address_zip}"
 
 
-class FilingSignatureDAO(Base):
-    __tablename__ = "filing_signature"
-    user_action: Mapped[int] = mapped_column(
-        ForeignKey("user_action.id"), nullable=False, primary_key=True, unique=True
-    )
-    filing: Mapped[int] = mapped_column(ForeignKey("filing.id"), index=True, nullable=False)
-
-
 class FilingDAO(Base):
     __tablename__ = "filing"
     id: Mapped[int] = mapped_column(primary_key=True, autoincrement=True)
@@ -118,13 +111,11 @@ class FilingDAO(Base):
     submissions: Mapped[List[SubmissionDAO] | None] = relationship(
         "SubmissionDAO", lazy="select", order_by=desc(SubmissionDAO.submission_time)
     )
-    signatures: Mapped[List[UserActionDAO] | None] = relationship(
-        "UserActionDAO", secondary="filing_signature", lazy="selectin", order_by="desc(UserActionDAO.timestamp)"
-    )
     confirmation_id: Mapped[str] = mapped_column(nullable=True)
-    creator_id: Mapped[int] = mapped_column(ForeignKey("user_action.id"))
-    creator: Mapped[UserActionDAO] = relationship(lazy="selectin", foreign_keys=[creator_id])
     is_voluntary: Mapped[bool] = mapped_column(nullable=True)
+    user_actions: Mapped[List[UserActionDAO] | None] = relationship(
+        "UserActionDAO", lazy="selectin", cascade="all, delete-orphan"
+    )
 
     def __str__(self):
         return f"ID: {self.id}, Filing Period: {self.filing_period}, LEI: {self.lei}, Tasks: {self.tasks}, Institution Snapshot ID: {self.institution_snapshot_id}, Contact Info: {self.contact_info}"

src/sbl_filing_api/entities/models/dto.py

@@ -1,12 +1,14 @@
 from sbl_filing_api.config import regex_configs
 from datetime import datetime
-from typing import Dict, Any, List
-from pydantic import BaseModel, ConfigDict, Field, model_validator
+from typing import Dict, Any, List, Annotated
+from pydantic import BaseModel, ConfigDict, Field, model_validator, computed_field
 from sbl_filing_api.entities.models.model_enums import FilingType, FilingTaskState, SubmissionState, UserActionType
 
 
 class UserActionDTO(BaseModel):
     id: int | None = None
+    filing_id: int | None = None
+    submission_id: int | None = None
     user_id: str = Field(max_length=36)
     user_name: str = Field(max_length=255)
     user_email: str = Field(max_length=255)
@@ -25,8 +27,17 @@ class SubmissionDTO(BaseModel):
     submission_time: datetime | None = None
     filename: str
     total_records: int | None = None
-    submitter: UserActionDTO
-    accepter: UserActionDTO | None = None
+    user_actions: Annotated[List[UserActionDTO] | None, Field(exclude=True)] = None
+
+    @computed_field
+    @property
+    def submitter(self) -> UserActionDTO | None:
+        return next((action for action in self.user_actions if action.action_type == UserActionType.SUBMIT), None)
+
+    @computed_field
+    @property
+    def accepter(self) -> UserActionDTO | None:
+        return next((action for action in self.user_actions if action.action_type == UserActionType.ACCEPT), None)
 
 
 class FilingTaskDTO(BaseModel):
@@ -86,9 +97,18 @@ class FilingDTO(BaseModel):
     institution_snapshot_id: str | None = None
     contact_info: ContactInfoDTO | None = None
     confirmation_id: str | None = None
-    signatures: List[UserActionDTO] = []
-    creator: UserActionDTO
     is_voluntary: bool | None = None
+    user_actions: Annotated[List[UserActionDTO] | None, Field(exclude=True)] = None
+
+    @computed_field
+    @property
+    def creator(self) -> UserActionDTO | None:
+        return next((action for action in self.user_actions if action.action_type == UserActionType.CREATE), None)
+
+    @computed_field
+    @property
+    def signatures(self) -> List[UserActionDTO]:
+        return [action for action in self.user_actions if action.action_type == UserActionType.SIGN]
 
 
 class FilingPeriodDTO(BaseModel):
@@ -103,7 +123,7 @@ class FilingPeriodDTO(BaseModel):
 
 
 class SnapshotUpdateDTO(BaseModel):
-    model_config = ConfigDict(from_attribute=True)
+    model_config = ConfigDict(from_attributes=True)
 
     institution_snapshot_id: str
 
@@ -115,6 +135,6 @@ class StateUpdateDTO(BaseModel):
 
 
 class VoluntaryUpdateDTO(BaseModel):
-    model_config = ConfigDict(from_attribute=True)
+    model_config = ConfigDict(from_attributes=True)
 
     is_voluntary: bool

src/sbl_filing_api/entities/models/model_enums.py

@@ -1,32 +1,32 @@
-from enum import Enum
+from enum import StrEnum
 
 
-class UserActionType(str, Enum):
-    SUBMIT = "SUBMIT"
-    ACCEPT = "ACCEPT"
-    SIGN = "SIGN"
-    CREATE = "CREATE"
+class UserActionType(StrEnum):
+    SUBMIT = ("SUBMIT",)
+    ACCEPT = ("ACCEPT",)
+    SIGN = ("SIGN",)
+    CREATE = ("CREATE",)
 
 
-class SubmissionState(str, Enum):
-    SUBMISSION_ACCEPTED = "SUBMISSION_ACCEPTED"
-    SUBMISSION_STARTED = "SUBMISSION_STARTED"
-    SUBMISSION_UPLOAD_MALFORMED = "SUBMISSION_UPLOAD_MALFORMED"
-    SUBMISSION_UPLOADED = "SUBMISSION_UPLOADED"
-    UPLOAD_FAILED = "UPLOAD_FAILED"
-    VALIDATION_ERROR = "VALIDATION_ERROR"
-    VALIDATION_EXPIRED = "VALIDATION_EXPIRED"
-    VALIDATION_IN_PROGRESS = "VALIDATION_IN_PROGRESS"
-    VALIDATION_SUCCESSFUL = "VALIDATION_SUCCESSFUL"
-    VALIDATION_WITH_ERRORS = "VALIDATION_WITH_ERRORS"
-    VALIDATION_WITH_WARNINGS = "VALIDATION_WITH_WARNINGS"
+class SubmissionState(StrEnum):
+    SUBMISSION_ACCEPTED = ("SUBMISSION_ACCEPTED",)
+    SUBMISSION_STARTED = ("SUBMISSION_STARTED",)
+    SUBMISSION_UPLOAD_MALFORMED = ("SUBMISSION_UPLOAD_MALFORMED",)
+    SUBMISSION_UPLOADED = ("SUBMISSION_UPLOADED",)
+    UPLOAD_FAILED = ("UPLOAD_FAILED",)
+    VALIDATION_ERROR = ("VALIDATION_ERROR",)
+    VALIDATION_EXPIRED = ("VALIDATION_EXPIRED",)
+    VALIDATION_IN_PROGRESS = ("VALIDATION_IN_PROGRESS",)
+    VALIDATION_SUCCESSFUL = ("VALIDATION_SUCCESSFUL",)
+    VALIDATION_WITH_ERRORS = ("VALIDATION_WITH_ERRORS",)
+    VALIDATION_WITH_WARNINGS = ("VALIDATION_WITH_WARNINGS",)
 
 
-class FilingTaskState(str, Enum):
-    NOT_STARTED = "NOT_STARTED"
-    IN_PROGRESS = "IN_PROGRESS"
-    COMPLETED = "COMPLETED"
+class FilingTaskState(StrEnum):
+    NOT_STARTED = ("NOT_STARTED",)
+    IN_PROGRESS = ("IN_PROGRESS",)
+    COMPLETED = ("COMPLETED",)
 
 
-class FilingType(str, Enum):
-    ANNUAL = "ANNUAL"
+class FilingType(StrEnum):
+    ANNUAL = ("ANNUAL",)

src/sbl_filing_api/entities/repos/submission_repo.py

@@ -95,20 +95,23 @@ async def get_user_actions(session: AsyncSession) -> List[UserActionDAO]:
     return await query_helper(session, UserActionDAO)
 
 
-async def add_submission(session: AsyncSession, filing_id: int, filename: str, submitter_id: int) -> SubmissionDAO:
+async def add_submission(
+    session: AsyncSession, filing_id: int, filename: str, submitter: UserActionDAO
+) -> SubmissionDAO:
     stmt = select(SubmissionDAO).filter_by(filing=filing_id).order_by(desc(SubmissionDAO.counter)).limit(1)
     last_sub = await session.scalar(stmt)
     current_count = last_sub.counter if last_sub else 0
     new_sub = SubmissionDAO(
         filing=filing_id,
         state=SubmissionState.SUBMISSION_STARTED,
         filename=filename,
-        submitter_id=submitter_id,
         counter=(current_count + 1),
+        user_actions=[submitter],
     )
     # this returns the attached object, most importantly with the new submission id
     new_sub = await session.merge(new_sub)
     await session.commit()
+    await session.refresh(new_sub)
     return new_sub
 
 
@@ -138,8 +141,8 @@ async def upsert_filing(session: AsyncSession, filing: FilingDTO) -> FilingDAO:
     return await upsert_helper(session, filing, FilingDAO)
 
 
-async def create_new_filing(session: AsyncSession, lei: str, filing_period: str, creator_id: int) -> FilingDAO:
-    new_filing = FilingDAO(filing_period=filing_period, lei=lei, creator_id=creator_id)
+async def create_new_filing(session: AsyncSession, lei: str, filing_period: str, creator: UserActionDAO) -> FilingDAO:
+    new_filing = FilingDAO(filing_period=filing_period, lei=lei, user_actions=[creator])
     return await upsert_helper(session, new_filing, FilingDAO)