Apply some limits and fixup

Author: john-dupuy

backend/ibutsu_server/controllers/artifact_controller.py

@@ -1,4 +1,5 @@
 import json
+from datetime import datetime
 
 import connexion
 import magic
@@ -119,6 +120,7 @@ def upload_artifact(body):
         filename=filename,
         result_id=data["resultId"],
         content=file_.read(),
+        upload_date=datetime.utcnow(),
         data=additional_metadata,
     )
     session.add(artifact)

backend/ibutsu_server/db/models.py

@@ -68,6 +68,7 @@ class Artifact(Model, FileMixin):
     result_id = Column(PortableUUID(), ForeignKey("results.id"), nullable=False)
     filename = Column(Text)
     data = Column(mutable_json_type(dbtype=PortableJSON(), nested=True))
+    upload_date = Column(DateTime)
 
 
 class Group(Model, ModelMixin):

backend/ibutsu_server/scripts/mongo2postgres.py

@@ -7,6 +7,7 @@
 """
 from argparse import ArgumentParser
 from datetime import datetime
+from datetime import timedelta
 from datetime import timezone
 from uuid import UUID
 
@@ -27,7 +28,10 @@
 UUID_1_EPOCH = datetime(1582, 10, 15, tzinfo=timezone.utc)
 UUID_TICKS = 10000000
 UUID_VARIANT_1 = 0b1000000000000000
-ROWS_TO_COMMIT_AT_ONCE = 100000
+ROWS_TO_COMMIT_AT_ONCE = 10000
+MONTHS_TO_KEEP = 3
+MIGRATION_TIME_FRAME = MONTHS_TO_KEEP * 30 * 24 * 60 * 60  # 3 months in seconds
+MIGRATION_LIMIT = 10000000000  # mostly for testing purposes
 
 Base = declarative_base()
 session = None
@@ -36,9 +40,9 @@
 TABLE_MAP = [
     ("groups", Group),
     ("projects", Project),
+    ("widgetConfigs", WidgetConfig),
     ("runs", Run),
     ("results", Result),
-    ("widgetConfigs", WidgetConfig),
 ]
 FILE_MAP = [
     # only convert artifacts, the reports in the existing DB aren't particularly useful
@@ -60,6 +64,28 @@
     "run",
 ]
 
+# indexes for the tables
+INDEXES = {
+    "results": [
+        "CREATE INDEX results_env ON results(env);",
+        "CREATE INDEX results_component ON results(component);",
+        "CREATE INDEX results_project ON results(project_id);",
+        "CREATE INDEX results_start_time ON results(start_time);",
+        "CREATE INDEX results_run ON results(run_id);",
+    ],
+    "runs": [
+        "CREATE INDEX runs_env ON runs(env);",
+        "CREATE INDEX runs_component ON runs(component);",
+        "CREATE INDEX runs_project ON runs(project_id);",
+        "CREATE INDEX runs_start_time ON runs(start_time);",
+    ],
+    "widget_configs": ["CREATE INDEX widget_configs_project ON widget_configs(project_id);"],
+    "artifacts": [
+        "CREATE INDEX artifact_result ON artifacts(result_id);",
+        "CREATE INDEX artifact_upload_date ON artifacts(upload_date);",
+    ],
+}
+
 
 def is_uuid(candidate):
     """Determine if this is a uuid"""
@@ -112,15 +138,25 @@ def setup_postgres(postgres_url):
 def migrate_table(collection, Model, vprint):
     """Migrate a collection from MongoDB into a table in PostgreSQL"""
     # TODO: update indexes once we know them
-    # indexes = collection.list_indexes()
-    # conn = Base.metadata.bind.connect()
-    # for idx in indexes:
-    #     if idx['name'] == '_id_':
-    #         continue
-    #     vprint('.', end='')
-    #     sql = convert_index(idx, Model.__tablename__)
-    #     conn.execute(sql)
-    for idx, row in enumerate(collection.find()):
+    conn = Base.metadata.bind.connect()
+    for sql_index in INDEXES.get(Model.__tablename__, []):
+        vprint(".", end="")
+        conn.execute(sql_index)
+
+    # for runs and results, sort by descending start_time
+    if Model.__tablename__ == "runs" or Model.__tablename__ == "results":
+        sort = [("start_time", -1)]
+        most_recent_record = collection.find_one(sort=sort)
+        most_recent_start_time = most_recent_record["start_time"]
+        # only include most recent runs and results
+        filter_ = {"start_time": {"$gt": most_recent_start_time - MIGRATION_TIME_FRAME}}
+    else:
+        sort = None
+        filter_ = None
+
+    for idx, row in enumerate(collection.find(filter_, sort=sort)):
+        if idx > MIGRATION_LIMIT:
+            break
         vprint(".", end="")
         mongo_id = row.pop("_id")
         # overwrite id with PSQL uuid
@@ -136,54 +172,71 @@ def migrate_table(collection, Model, vprint):
 
         # promote some metadata fields to the appropriate column
         for field in FIELDS_TO_PROMOTE:
-            if row.get("metadata") and field in row["metadata"]:
+            if row.get("metadata") and row["metadata"].get(field):
                 row[field] = row["metadata"][field]
         # convert some ObjectId's to UUID's
         for field in ID_FIELDS:
-            if field == "project":
-                if row.get("metadata") and field in row["metadata"]:
+            if row.get("metadata") and row["metadata"].get(field):
+                if field == "project":
                     row["project_id"] = convert_objectid_to_uuid(row["metadata"][field])
                     # also update the metadata field
                     row["metadata"][field] = row["project_id"]
-            elif field == "run":
-                if row.get("metadata") and field in row["metadata"]:
+                elif field == "run":
                     row["run_id"] = convert_objectid_to_uuid(row["metadata"][field])
                     # also update the metadata field
                     row["metadata"][field] = row["run_id"]
-            elif field in ["result_id", "resultId"]:
-                if row.get("metadata") and field in row["metadata"]:
+                elif field in ["result_id", "resultId"]:
                     row["result_id"] = convert_objectid_to_uuid(row["metadata"][field])
-            else:
-                if row.get("metadata") and field in row["metadata"]:
+                else:
                     row["metadata"][field] = convert_objectid_to_uuid(row["metadata"][field])
+
         obj = Model.from_dict(**row)
         session.add(obj)
         if idx % ROWS_TO_COMMIT_AT_ONCE == 0:
             session.commit()
     session.commit()
+    # at the end of the session do a little cleanup
+    if Model.__tablename__ == "runs" or Model.__tablename__ == "results":
+        conn = Base.metadata.bind.connect()
+        # delete any results or runs without start_time
+        sql_delete = f"DELETE FROM {Model.__tablename__} where start_time IS NULL;"
+        conn.execute(sql_delete)
     vprint(" done")
 
 
 def migrate_file(collection, Model, vprint):
     """Migrate a GridFS collection from MongoDB into a table in PostgreSQL"""
     # Access the underlying collection object
     # TODO: update indexes once we know them
-    # indexes = collection._collection.list_indexes()
-    # conn = Base.metadata.bind.connect()
-    # for idx in indexes:
-    #     if idx['name'] == '_id_':
-    #         continue
-    #     vprint('.', end='')
-    #     sql = convert_index(idx, Model.__tablename__)
-    #     conn.execute(sql)
-    for idx, row in enumerate(collection.find({})):
+    conn = Base.metadata.bind.connect()
+    for sql_index in INDEXES.get(Model.__tablename__, []):
+        vprint(".", end="")
+        conn.execute(sql_index)
+
+    # for runs and results, sort by descending start_time
+    if Model.__tablename__ == "artifacts":
+        sort = [("_id", -1)]  # in MongoDB sorting by id is like sorting by a 'created' field
+        most_recent_record = [x for x in collection.find(limit=1, sort=sort)][0]
+        most_recent_upload_date = most_recent_record.upload_date
+        # only include most recent runs and results
+        filter_ = {
+            "uploadDate": {"$gt": (most_recent_upload_date - timedelta(days=30 * MONTHS_TO_KEEP))}
+        }
+    else:
+        sort = None
+        filter_ = None
+
+    for idx, row in enumerate(collection.find(filter_, sort=sort)):
+        if idx > MIGRATION_LIMIT:
+            break
         vprint(".", end="")
         pg_id = convert_objectid_to_uuid(row._id)
-        data = {}
+        data = dict()
         data["metadata"] = row.metadata
         data["id"] = pg_id
         data["filename"] = row.filename
         data["content"] = row.read()
+        data["upload_date"] = row.upload_date
         for field in ID_FIELDS:
             if field == "resultId":
                 data["result_id"] = convert_objectid_to_uuid(row.metadata[field])
@@ -198,14 +251,15 @@ def migrate_file(collection, Model, vprint):
     vprint(" done")
 
 
-def migrate_tables(mongo, vprint):
+def migrate_tables(mongo, vprint, migrate_files=False):
     """Migrate all the tables"""
     for collection, model in TABLE_MAP:
         vprint("Migrating {} ".format(collection), end="")
         migrate_table(mongo[collection], model, vprint)
-    for collection, model in FILE_MAP:
-        vprint("Migrating {} ".format(collection), end="")
-        migrate_file(GridFSBucket(mongo, collection), model, vprint)
+    if migrate_files:
+        for collection, model in FILE_MAP:
+            vprint("Migrating {} ".format(collection), end="")
+            migrate_file(GridFSBucket(mongo, collection), model, vprint)
 
 
 def build_mongo_connection(url):
@@ -225,6 +279,7 @@ def parse_args():
     parser.add_argument("mongo_url", help="URL to MongoDB database")
     parser.add_argument("postgres_url", help="URL to PostgreSQL database")
     parser.add_argument("-v", "--verbose", action="store_true", help="Say what I'm doing")
+    parser.add_argument("-f", "--files", action="store_true", help="Migrate artifact files")
     return parser.parse_args()
 
 
@@ -234,7 +289,7 @@ def main():
     mongo_url, database = build_mongo_connection(args.mongo_url)
     mongo = get_mongo(mongo_url, database)
     setup_postgres(args.postgres_url)
-    migrate_tables(mongo, vprint)
+    migrate_tables(mongo, vprint, args.files)
 
 
 if __name__ == "__main__":

frontend/src/result-list.js

@@ -341,10 +341,10 @@ export class ResultList extends React.Component {
     let filters = this.state.filters;
     const project = getActiveProject();
     if (project) {
-      filters['metadata.project'] = {'val': project.id, 'op': 'eq'};
+      filters['project_id'] = {'val': project.id, 'op': 'eq'};
     }
-    else if (Object.prototype.hasOwnProperty.call(filters, 'metadata.project')) {
-      delete filters['metadata.project']
+    else if (Object.prototype.hasOwnProperty.call(filters, 'project_id')) {
+      delete filters['project_id']
     }
     if (filters) {
       params['apply_max'] = true;  // if filters are applied limit the documents returned

frontend/src/run-list.js

@@ -323,10 +323,10 @@ export class RunList extends React.Component {
     let filters = this.state.filters;
     const project = getActiveProject();
     if (project) {
-      filters['metadata.project'] = {'val': project.id, 'op': 'eq'};
+      filters['project_id'] = {'val': project.id, 'op': 'eq'};
     }
-    else if (Object.prototype.hasOwnProperty.call(filters, 'metadata.project')) {
-      delete filters['metadata.project']
+    else if (Object.prototype.hasOwnProperty.call(filters, 'project_id')) {
+      delete filters['project_id']
     }
     params['pageSize'] = this.state.pageSize;
     params['page'] = this.state.page;