Move repository to shadow Owner at the start of cleanup task

services/cleanup/cleanup.py

@@ -1,9 +1,13 @@
+import logging
+
 from django.db.models.query import QuerySet
 
 from services.cleanup.models import MANUAL_CLEANUP
 from services.cleanup.relations import build_relation_graph
 from services.cleanup.utils import CleanupContext, CleanupResult, CleanupSummary
 
+log = logging.getLogger(__name__)
+
 
 def run_cleanup(
     query: QuerySet,
@@ -32,6 +36,15 @@ def run_cleanup(
 
         if result.cleaned_models > 0 or result.cleaned_files > 0:
             summary[model] = result
+
+            log.info(
+                f"Finished cleaning up {model.__name__}",
+                extra={
+                    "cleaned_models": result.cleaned_models,
+                    "cleaned_files": result.cleaned_files,
+                },
+            )
+
         cleaned_models += result.cleaned_models
         cleaned_files += result.cleaned_files
 

services/cleanup/models.py

@@ -15,7 +15,7 @@
 from services.archive import ArchiveService, MinioEndpoints
 from services.cleanup.utils import CleanupContext, CleanupResult
 
-MANUAL_QUERY_CHUNKSIZE = 2_500
+MANUAL_QUERY_CHUNKSIZE = 5_000
 DELETE_FILES_BATCHSIZE = 50
 
 

services/cleanup/owner.py

@@ -1,12 +1,30 @@
+import logging
+
 from django.db import transaction
 from django.db.models import Q
 from shared.django_apps.codecov_auth.models import Owner
 from shared.django_apps.core.models import Commit, Pull, Repository
 
+from services.cleanup.cleanup import run_cleanup
+from services.cleanup.utils import CleanupSummary
+
+log = logging.getLogger(__name__)
+
 CLEAR_ARRAY_FIELDS = ["plan_activated_users", "organizations", "admins"]
 
 
-def clear_owner(owner_id: int):
+def cleanup_owner(owner_id: int) -> CleanupSummary:
+    log.info("Started/Continuing Owner cleanup", extra={"owner_id": owner_id})
+
+    clear_owner_references(owner_id)
+    owner_query = Owner.objects.filter(ownerid=owner_id)
+    summary = run_cleanup(owner_query)
+
+    log.info("Owner cleanup finished", extra={"owner_id": owner_id, "summary": summary})
+    return summary
+
+
+def clear_owner_references(owner_id: int):
     """
     This clears the `ownerid` from various DB arrays where it is being referenced.
     """
@@ -16,6 +34,10 @@ def clear_owner(owner_id: int):
     Commit.objects.filter(author=owner_id).update(author=None)
     Pull.objects.filter(author=owner_id).update(author=None)
 
+    # This uses a transaction / `select_for_update` to ensure consistency when
+    # modifying these `ArrayField`s in python.
+    # I don’t think we have such consistency anyplace else in the codebase, so
+    # if this is causing lock contention issues, its also fair to avoid this.
     with transaction.atomic():
         filter = Q()
         for field in CLEAR_ARRAY_FIELDS:

services/cleanup/repository.py

@@ -0,0 +1,83 @@
+import logging
+from uuid import uuid4
+
+from django.db import transaction
+from shared.django_apps.codecov_auth.models import Owner
+from shared.django_apps.core.models import Repository
+
+from services.cleanup.cleanup import run_cleanup
+from services.cleanup.utils import CleanupSummary
+
+log = logging.getLogger(__name__)
+
+
+def cleanup_repo(repo_id: int) -> CleanupSummary:
+    cleanup_started, owner_id = start_repo_cleanup(repo_id)
+
+    if cleanup_started:
+        log.info("Started Repository cleanup", extra={"repo_id": repo_id})
+    else:
+        log.info("Continuing Repository cleanup", extra={"repo_id": repo_id})
+
+    repo_query = Repository.objects.filter(repoid=repo_id)
+    summary = run_cleanup(repo_query)
+    Owner.filter(ownerid=owner_id).delete()
+
+    log.info(
+        "Repository cleanup finished", extra={"repoid": repo_id, "summary": summary}
+    )
+    return summary
+
+
+def start_repo_cleanup(repo_id: int) -> tuple[bool, int]:
+    """
+    Starts Repository deletion by marking the repository as `deleted`, and moving
+    it to a newly created "shadow Owner".
+
+    This newly created `Owner` only has a valid `service` and `service_id`,
+    which are the only required non-NULL fields without defaults, and is otherwise
+    completely empty.
+
+    The `ownerid` of this newly created owner is being returned along with a flag
+    indicating whether the repo cleanup was just started, or whether it is already
+    marked for deletion, and this function is being retried.
+    It is expected that repo cleanup is a slow process and might be done in more steps.
+    """
+    # Runs in a transaction as we do not want to leave leftover shadow owners in
+    # case anything goes wrong here.
+    with transaction.atomic():
+        (
+            repo_deleted,
+            owner_id,
+            owner_name,
+            owner_username,
+            owner_service,
+            owner_service_id,
+        ) = Repository.objects.values_list(
+            "deleted",
+            "author__owneridauthor__name",
+            "author__username",
+            "author__service",
+            "author__service_id",
+        ).get(repoid=repo_id)
+
+        if repo_deleted and not owner_name and not owner_username:
+            return (False, owner_id)
+
+        # We mark the repository as "scheduled for deletion" by setting the `deleted`
+        # flag, moving it to a new shadow owner, and clearing some tokens.
+        shadow_owner = Owner.objects.create(
+            service=owner_service, service_id=owner_service_id
+        )
+        new_token = uuid4().hex
+        Repository.objects.filter(repoid=repo_id).update(
+            deleted=True,
+            author=shadow_owner,
+            upload_token=new_token,
+            image_token=new_token,
+        )
+
+        # The equivalent of `SET NULL`:
+        Repository.objects.filter(forkid=repo_id).update(forkid=None)
+
+        return (True, shadow_owner.ownerid)

tasks/delete_owner.py

@@ -1,34 +1,21 @@
 import logging
 
 from shared.celery_config import delete_owner_task_name
-from shared.django_apps.codecov_auth.models import Owner
 
 from app import celery_app
-from services.cleanup.cleanup import run_cleanup
-from services.cleanup.owner import clear_owner
+from services.cleanup.owner import cleanup_owner
 from services.cleanup.utils import CleanupSummary
 from tasks.base import BaseCodecovTask
 
 log = logging.getLogger(__name__)
 
 
 class DeleteOwnerTask(BaseCodecovTask, name=delete_owner_task_name):
-    """
-    Delete an owner and their data:
-    - Repo archive data for each of their owned repos
-    - Owner entry from db
-    - Cascading deletes of repos, pulls, and branches for the owner
-    """
+    acks_late = True  # retry the task when the worker dies for whatever reason
+    max_retries = None  # aka, no limit on retries
 
     def run_impl(self, _db_session, ownerid: int) -> CleanupSummary:
-        log.info("Delete owner", extra={"ownerid": ownerid})
-
-        clear_owner(ownerid)
-        owner_query = Owner.objects.filter(ownerid=ownerid)
-
-        summary = run_cleanup(owner_query)
-        log.info("Deletion finished", extra={"ownerid": ownerid, "summary": summary})
-        return summary
+        return cleanup_owner(ownerid)
 
 
 RegisteredDeleteOwnerTask = celery_app.register_task(DeleteOwnerTask())

tasks/flush_repo.py

@@ -1,28 +1,19 @@
 import logging
 
-import sentry_sdk
-from shared.django_apps.core.models import Repository
-
 from app import celery_app
-from database.engine import Session
-from services.cleanup.cleanup import run_cleanup
+from services.cleanup.repository import cleanup_repo
 from services.cleanup.utils import CleanupSummary
 from tasks.base import BaseCodecovTask
 
 log = logging.getLogger(__name__)
 
 
 class FlushRepoTask(BaseCodecovTask, name="app.tasks.flush_repo.FlushRepo"):
-    @sentry_sdk.trace
-    def run_impl(
-        self, _db_session: Session, *, repoid: int, **kwargs
-    ) -> CleanupSummary:
-        log.info("Deleting repo contents", extra={"repoid": repoid})
-        repo_query = Repository.objects.filter(repoid=repoid)
+    acks_late = True  # retry the task when the worker dies for whatever reason
+    max_retries = None  # aka, no limit on retries
 
-        summary = run_cleanup(repo_query)
-        log.info("Deletion finished", extra={"repoid": repoid, "summary": summary})
-        return summary
+    def run_impl(self, _db_session, repoid: int) -> CleanupSummary:
+        return cleanup_repo(repoid)
 
 
 FlushRepo = celery_app.register_task(FlushRepoTask())