Merge pull request #47 from john-dupuy/add-migration-script

Update the migration script to work with new psql db

Author: rsnyman

.pre-commit-config.yaml

@@ -27,7 +27,7 @@ repos:
   hooks:
   - id: flake8
     language_version: python3
-    args: [--max-line-length, "100"]
+    args: [--max-line-length, "100", --ignore, "E203"]
 - repo: https://github.com/asottile/pyupgrade
   rev: v2.6.1
   hooks:

backend/ibutsu_server/controllers/artifact_controller.py

@@ -1,4 +1,5 @@
 import json
+from datetime import datetime
 
 import connexion
 import magic
@@ -119,6 +120,7 @@ def upload_artifact(body):
         filename=filename,
         result_id=data["resultId"],
         content=file_.read(),
+        upload_date=datetime.utcnow(),
         data=additional_metadata,
     )
     session.add(artifact)

backend/ibutsu_server/controllers/result_controller.py

@@ -81,41 +81,53 @@ def get_result_list(filter_=None, page=1, page_size=25, apply_max=False):
     :rtype: List[Result]
     """
     query = Result.query
+    count_estimate = None
     if filter_:
         for filter_string in filter_:
             filter_clause = convert_filter(filter_string, Result)
             if filter_clause is not None:
                 query = query.filter(filter_clause)
+    else:
+        # use a count estimate when no filter is applied
+        count_estimate = int(
+            session.execute(
+                "SELECT reltuples as approx_count FROM pg_class WHERE relname='results'"
+            ).fetchall()[0][0]
+        )
+
     offset = (page * page_size) - page_size
-    try:
-        # if the count is fast, just use it! Even if apply_max is set to true
-        session.execute(f"SET statement_timeout TO {int(COUNT_TIMEOUT*1000)}; commit;")
-        total_items = query.count()
-    except OperationalError:
-        # reset the timeout if we hit an exception
-        session.execute("SET statement_timeout TO 0; commit;")
-        if apply_max:
-            print(
-                f"FunctionTimedOut: 'query.count' with filters: {filter_} timed out, "
-                f"using default items of {MAX_DOCUMENTS}"
-            )
-            if offset > MAX_DOCUMENTS:
-                raise ValueError(
-                    f"Offset: {offset} exceeds the "
-                    f"MAX_DOCUMENTS: {MAX_DOCUMENTS} able to be displayed in the UI. "
-                    f"Please use the API for this request."
+    if not count_estimate:
+        try:
+            # if the count is fast, just use it! Even if apply_max is set to true
+            session.execute(f"SET statement_timeout TO {int(COUNT_TIMEOUT*1000)}; commit;")
+            total_items = query.count()
+        except OperationalError:
+            # reset the timeout if we hit an exception
+            session.execute("SET statement_timeout TO 0; commit;")
+            if apply_max:
+                print(
+                    f"FunctionTimedOut: 'query.count' with filters: {filter_} timed out, "
+                    f"using default items of {MAX_DOCUMENTS}"
                 )
-            total_items = MAX_DOCUMENTS
+                if offset > MAX_DOCUMENTS:
+                    raise ValueError(
+                        f"Offset: {offset} exceeds the "
+                        f"MAX_DOCUMENTS: {MAX_DOCUMENTS} able to be displayed in the UI. "
+                        f"Please use the API for this request."
+                    )
+                total_items = MAX_DOCUMENTS
+            else:
+                print(
+                    f"FunctionTimedOut: 'query.count' with args: {filter_} timed out, "
+                    f"but limit_documents is set to False, proceeding"
+                )
+                # if we don't want to limit documents, just do the standard count
+                total_items = query.count()
         else:
-            print(
-                f"FunctionTimedOut: 'query.count' with args: {filter_} timed out, "
-                f"but limit_documents is set to False, proceeding"
-            )
-            # if we don't want to limit documents, just do the standard count
-            total_items = query.count()
+            # reset the timeout if we don't hit an exception
+            session.execute("SET statement_timeout TO 0; commit;")
     else:
-        # reset the timeout if we don't hit an exception
-        session.execute("SET statement_timeout TO 0; commit;")
+        total_items = count_estimate
 
     total_pages = (total_items // page_size) + (1 if total_items % page_size > 0 else 0)
     results = query.order_by(Result.start_time.desc()).offset(offset).limit(page_size).all()

backend/ibutsu_server/controllers/widget_config_controller.py

@@ -76,7 +76,7 @@ def get_widget_config_list(filter_=None, page=1, page_size=25):
     offset = (page * page_size) - page_size
     total_items = query.count()
     total_pages = (total_items // page_size) + (1 if total_items % page_size > 0 else 0)
-    widgets = query.order_by(WidgetConfig.weight.desc()).offset(offset).limit(page_size)
+    widgets = query.order_by(WidgetConfig.weight.asc()).offset(offset).limit(page_size)
     return {
         "widgets": [widget.to_dict() for widget in widgets],
         "pagination": {

backend/ibutsu_server/db/models.py

@@ -1,3 +1,4 @@
+from datetime import datetime
 from uuid import uuid4
 
 from ibutsu_server.db.base import Boolean
@@ -65,38 +66,39 @@ def to_dict(self):
 
 class Artifact(Model, FileMixin):
     __tablename__ = "artifacts"
-    result_id = Column(PortableUUID(), ForeignKey("results.id"), nullable=False)
-    filename = Column(Text)
+    result_id = Column(PortableUUID(), ForeignKey("results.id"), nullable=False, index=True)
+    filename = Column(Text, index=True)
     data = Column(mutable_json_type(dbtype=PortableJSON(), nested=True))
+    upload_date = Column(DateTime, default=datetime.utcnow, index=True)
 
 
 class Group(Model, ModelMixin):
     __tablename__ = "groups"
-    name = Column(Text)
+    name = Column(Text, index=True)
     projects = relationship("Project")
     data = Column(mutable_json_type(dbtype=PortableJSON(), nested=True))
 
 
 class Import(Model, ModelMixin):
     __tablename__ = "imports"
     file = relationship("ImportFile")
-    filename = Column(Text)
-    format = Column(Text)
+    filename = Column(Text, index=True)
+    format = Column(Text, index=True)
     data = Column(mutable_json_type(dbtype=PortableJSON(), nested=True))
-    status = Column(Text)
+    status = Column(Text, index=True)
 
 
 class ImportFile(Model, FileMixin):
     __tablename__ = "import_files"
-    import_id = Column(PortableUUID(), ForeignKey("imports.id"), nullable=False)
+    import_id = Column(PortableUUID(), ForeignKey("imports.id"), nullable=False, index=True)
 
 
 class Project(Model, ModelMixin):
     __tablename__ = "projects"
-    name = Column(Text)
-    title = Column(Text)
-    owner_id = Column(Text)
-    group_id = Column(PortableUUID(), ForeignKey("groups.id"))
+    name = Column(Text, index=True)
+    title = Column(Text, index=True)
+    owner_id = Column(Text, index=True)
+    group_id = Column(PortableUUID(), ForeignKey("groups.id"), index=True)
     reports = relationship("Report")
     results = relationship("Result")
     runs = relationship("Run")
@@ -105,64 +107,64 @@ class Project(Model, ModelMixin):
 
 class Report(Model, ModelMixin):
     __tablename__ = "reports"
-    created = Column(DateTime)
-    download_url = Column(Text)
-    filename = Column(Text)
-    mimetype = Column(Text)
-    name = Column(Text)
+    created = Column(DateTime, default=datetime.utcnow, index=True)
+    download_url = Column(Text, index=True)
+    filename = Column(Text, index=True)
+    mimetype = Column(Text, index=True)
+    name = Column(Text, index=True)
     params = Column(mutable_json_type(dbtype=PortableJSON()))
-    project_id = Column(PortableUUID(), ForeignKey("projects.id"))
+    project_id = Column(PortableUUID(), ForeignKey("projects.id"), index=True)
     file = relationship("ReportFile")
-    status = Column(Text)
-    url = Column(Text)
-    view_url = Column(Text)
+    status = Column(Text, index=True)
+    url = Column(Text, index=True)
+    view_url = Column(Text, index=True)
 
 
 class ReportFile(Model, FileMixin):
     __tablename__ = "report_files"
-    report_id = Column(PortableUUID(), ForeignKey("reports.id"), nullable=False)
-    filename = Column(Text)
+    report_id = Column(PortableUUID(), ForeignKey("reports.id"), nullable=False, index=True)
+    filename = Column(Text, index=True)
     data = Column(mutable_json_type(dbtype=PortableJSON(), nested=True))
 
 
 class Result(Model, ModelMixin):
     __tablename__ = "results"
     artifacts = relationship("Artifact")
-    component = Column(Text)
+    component = Column(Text, index=True)
     # this is metadata but it is a reserved attr
     data = Column(mutable_json_type(dbtype=PortableJSON(), nested=True))
-    duration = Column(Float)
-    env = Column(Text)
+    duration = Column(Float, index=True)
+    env = Column(Text, index=True)
     params = Column(mutable_json_type(dbtype=PortableJSON()))
-    project_id = Column(PortableUUID(), ForeignKey("projects.id"))
-    result = Column(Text)
-    run_id = Column(PortableUUID(), ForeignKey("runs.id"))
-    source = Column(Text)
-    start_time = Column(DateTime)
-    test_id = Column(Text)
+    project_id = Column(PortableUUID(), ForeignKey("projects.id"), index=True)
+    result = Column(Text, index=True)
+    run_id = Column(PortableUUID(), ForeignKey("runs.id"), index=True)
+    source = Column(Text, index=True)
+    start_time = Column(DateTime, default=datetime.utcnow, index=True)
+    test_id = Column(Text, index=True)
 
 
 class Run(Model, ModelMixin):
     __tablename__ = "runs"
-    component = Column(Text)
-    created = Column(DateTime)
+    component = Column(Text, index=True)
+    created = Column(DateTime, default=datetime.utcnow, index=True)
     # this is metadata but it is a reserved attr
     data = Column(mutable_json_type(dbtype=PortableJSON(), nested=True))
-    duration = Column(Float)
-    env = Column(Text)
-    project_id = Column(PortableUUID(), ForeignKey("projects.id"))
+    duration = Column(Float, index=True)
+    env = Column(Text, index=True)
+    project_id = Column(PortableUUID(), ForeignKey("projects.id"), index=True)
     results = relationship("Result")
-    source = Column(Text)
-    start_time = Column(DateTime)
+    source = Column(Text, index=True)
+    start_time = Column(DateTime, default=datetime.utcnow, index=True)
     summary = Column(mutable_json_type(dbtype=PortableJSON()))
 
 
 class WidgetConfig(Model, ModelMixin):
     __tablename__ = "widget_configs"
-    navigable = Column(Boolean)
+    navigable = Column(Boolean, index=True)
     params = Column(mutable_json_type(dbtype=PortableJSON()))
-    project_id = Column(PortableUUID(), ForeignKey("projects.id"))
-    title = Column(Text)
-    type = Column(Text)
-    weight = Column(Integer)
-    widget = Column(Text)
+    project_id = Column(PortableUUID(), ForeignKey("projects.id"), index=True)
+    title = Column(Text, index=True)
+    type = Column(Text, index=True)
+    weight = Column(Integer, index=True)
+    widget = Column(Text, index=True)

backend/ibutsu_server/filters.py

@@ -26,6 +26,8 @@ def string_to_column(field, model):
             if idx == 0:
                 continue
             column = column[part]
+        if field not in ARRAY_FIELDS:
+            column = column.as_string()
     else:
         column = getattr(model, field)
     return column