Add some indexes for JSON fields, fix numerous bugs

Author: john-dupuy

backend/ibutsu_server/controllers/widget_config_controller.py

@@ -76,7 +76,7 @@ def get_widget_config_list(filter_=None, page=1, page_size=25):
     offset = (page * page_size) - page_size
     total_items = query.count()
     total_pages = (total_items // page_size) + (1 if total_items % page_size > 0 else 0)
-    widgets = query.order_by(WidgetConfig.weight.desc()).offset(offset).limit(page_size)
+    widgets = query.order_by(WidgetConfig.weight.asc()).offset(offset).limit(page_size)
     return {
         "widgets": [widget.to_dict() for widget in widgets],
         "pagination": {

backend/ibutsu_server/filters.py

@@ -26,6 +26,8 @@ def string_to_column(field, model):
             if idx == 0:
                 continue
             column = column[part]
+        if field not in ARRAY_FIELDS:
+            column = column.as_string()
     else:
         column = getattr(model, field)
     return column

backend/ibutsu_server/scripts/mongo2postgres.py

@@ -28,8 +28,8 @@
 UUID_1_EPOCH = datetime(1582, 10, 15, tzinfo=timezone.utc)
 UUID_TICKS = 10000000
 UUID_VARIANT_1 = 0b1000000000000000
-ROWS_TO_COMMIT_AT_ONCE = 10000
-MONTHS_TO_KEEP = 1
+ROWS_TO_COMMIT_AT_ONCE = 1000
+MONTHS_TO_KEEP = 0.1
 # To avoid foreign key constraints, just shift the range of artifacts to keep a bit
 ARTIFACT_MONTHS_TO_KEEP = 0.75 * MONTHS_TO_KEEP
 MIGRATION_LIMIT = 10000000000  # mostly for testing purposes
@@ -69,10 +69,18 @@
 
 # json indexes for the tables
 INDEXES = {
-    # "results": [
-    # ],
-    # "runs": [
-    # ],
+    "results": [
+        "CREATE INDEX ix_results_jenkins_job_name ON results((data->'jenkins'->>'job_name'));",
+        "CREATE INDEX ix_results_jenkins_build_number "
+        "ON results((data->'jenkins'->>'build_number'));",
+        "CREATE INDEX ix_results_classification ON results((data->>'classification'));",
+        "CREATE INDEX ix_results_assignee ON results((data->>'assignee'));",
+        "CREATE INDEX ix_results_exception_name ON results((data->>'exception_name'));",
+    ],
+    "runs": [
+        "CREATE INDEX ix_runs_jenkins_job_name ON runs((data->'jenkins'->>'job_name'));"
+        "CREATE INDEX ix_runs_jenkins_build_number ON runs((data->'jenkins'->>'build_number'));"
+    ],
 }
 
 
@@ -127,10 +135,6 @@ def setup_postgres(postgres_url):
 def migrate_table(collection, Model, vprint, filter_=None):
     """Migrate a collection from MongoDB into a table in PostgreSQL"""
     # TODO: update indexes once we know them
-    conn = Base.metadata.bind.connect()
-    for sql_index in INDEXES.get(Model.__tablename__, []):
-        vprint(".", end="")
-        conn.execute(sql_index)
 
     if Model.__tablename__ == "runs":
         run_ids = []
@@ -195,6 +199,8 @@ def migrate_table(collection, Model, vprint, filter_=None):
                 if row.get("params") and row["params"].get("sort_field"):
                     # we no longer use this field
                     row["params"].pop("sort_field")
+                if row.get("params") and row["params"].get("group_field") == "metadata.component":
+                    row["params"]["group_field"] = "component"
 
         obj = Model.from_dict(**row)
         session.add(obj)
@@ -288,6 +294,13 @@ def migrate_tables(mongo, vprint, migrate_files=False):
 
     # loop over collections and migrate
     for collection, model in TABLE_MAP:
+        # first create indexes for the table
+        conn = Base.metadata.bind.connect()
+        for sql_index in INDEXES.get(model.__tablename__, []):
+            vprint(".", end="")
+            conn.execute(sql_index)
+
+        # migrate the table over
         vprint("Migrating {} ".format(collection), end="")
         if collection == "runs":
             run_ids = migrate_table(mongo[collection], model, vprint, filter_=filter_)

backend/ibutsu_server/widgets/jenkins_heatmap.py

@@ -1,4 +1,5 @@
 from ibutsu_server.db.base import Float
+from ibutsu_server.db.base import Integer
 from ibutsu_server.db.base import session
 from ibutsu_server.db.models import Run
 from ibutsu_server.filters import apply_filters
@@ -42,7 +43,7 @@ def _get_builds(job_name, builds, project=None):
 
     # create the query
     query = (
-        session.query(group_field.label("build_number"))
+        session.query(group_field.cast(Integer).label("build_number"))
         .group_by("build_number")
         .order_by(desc("build_number"))
     )
@@ -51,7 +52,7 @@ def _get_builds(job_name, builds, project=None):
     query = apply_filters(query, filters, Run)
 
     # make the query
-    return [build_number[0] for build_number in query.limit(builds)]
+    return [str(build_number[0]) for build_number in query.limit(builds)]
 
 
 def _get_heatmap(job_name, builds, group_field, count_skips, project=None):
@@ -67,7 +68,7 @@ def _get_heatmap(job_name, builds, group_field, count_skips, project=None):
         f"{group_field}@y",
     ]
     if project:
-        filters.append(f"metadata.project={project}")
+        filters.append(f"project_id={project}")
 
     # generate the group_fields
     group_field = string_to_column(group_field, Run)

frontend/src/app.js

@@ -93,7 +93,7 @@ class App extends React.Component {
     let params = {'filter': ['type=view', 'navigable=true']};
     let project = getActiveProject();
     if (project) {
-      params['filter'].push('project=' + project.id);
+      params['filter'].push('project_id=' + project.id);
     }
     fetch(buildUrl(Settings.serverUrl + '/widget-config', params))
       .then(response => response.json())

frontend/src/dashboard.js

@@ -40,7 +40,7 @@ export class Dashboard extends React.Component {
     let params = {"type": "widget"};
     let project = getActiveProject();
     if (project) {
-      params['filter'] = 'project=' + project.id;
+      params['filter'] = 'project_id=' + project.id;
     }
     fetch(buildUrl(Settings.serverUrl + '/widget-config', params))
       .then(response => response.json())

frontend/src/run.js

@@ -343,7 +343,7 @@ export class Run extends React.Component {
   }
 
   getResultsForTree(page) {
-    let params = {filter: 'metadata.run=' + this.state.id};
+    let params = {filter: 'run_id=' + this.state.id};
     params['pageSize'] = 500;
     params['page'] = page;
     fetch(buildUrl(Settings.serverUrl + '/result', params))