Fix error handling and update lint settings

Author: suecharo

.dockerignore

@@ -5,4 +5,5 @@
 !pyproject.toml
 !README.md
 !sapporo
+!sapporo-wes-spec-2.0.0.yml
 !tests

Dockerfile

@@ -9,6 +9,7 @@ LABEL org.opencontainers.image.licenses="Apache2.0"
 
 RUN apt update && \
     apt install -y --no-install-recommends \
+    curl \
     jq && \
     apt clean && \
     rm -rf /var/lib/apt/lists/*

compose.dev.yml

@@ -8,6 +8,8 @@ services:
     volumes:
       - ${PWD}/sapporo:/app/sapporo
       - ${PWD}/tests:/app/tests
+      - ${PWD}/pyproject.toml:/app/pyproject.toml
+      - ${PWD}/sapporo-wes-spec-2.0.0.yml:/app/sapporo-wes-spec-2.0.0.yml
       # The ones below are mounted for cwltool and DinD.
       - ${PWD}/runs:${PWD}/runs
       - /var/run/docker.sock:/var/run/docker.sock
@@ -20,11 +22,12 @@ services:
       - SAPPORO_RUN_DIR=${PWD}/runs
     ports:
       - 127.0.0.1:1122:1122
-    restart: on-failure
+    restart: always
     working_dir: /app
     command: [ "sleep", "infinity" ]
     networks:
       - sapporo-dev-network
+    init: true
 
 networks:
   sapporo-dev-network:

compose.keycloak.dev.yml

@@ -11,13 +11,12 @@ services:
     ports:
       - 127.0.0.1:8080:8080
     user: 0:0
-    restart: on-failure
-    command: ["start-dev"]
+    restart: always
+    command: [ "start-dev" ]
     networks:
       - sapporo-network
 
 networks:
   sapporo-network:
-    # name: sapporo-network
     name: sapporo-dev-network
     external: true

compose.yml

@@ -15,7 +15,7 @@ services:
       - SAPPORO_RUN_DIR=${PWD}/runs
     ports:
       - 127.0.0.1:1122:1122
-    restart: on-failure
+    restart: always
     working_dir: /app
     command: [ "sapporo" ]
     networks:

pyproject.toml

@@ -78,19 +78,9 @@ testpaths = ["tests/py_tests"]
 
 [tool.mypy]
 files = ["./sapporo/**/*.py"]
+follow_imports = "silent"
+strict = true
 ignore_missing_imports = true
-follow_imports = "skip"
-warn_redundant_casts = true
-warn_unused_ignores = true
-disallow_any_generics = true
-check_untyped_defs = true
-no_implicit_reexport = true
-disallow_untyped_defs = true
-
-[tool.pydantic-mypy]
-init_forbid_extra = true
-init_typed = true
-warn_required_dynamic_aliases = true
 
 [tool.pylint.messages_control]
 disable = [
@@ -105,5 +95,7 @@ disable = [
     "R0913", # Too many arguments
     "R0914", # Too many local variables
     "R0915", # Too many statements
+    "R0917", # Too many positional arguments
     "R1705", # Unnecessary "else" after "return"
+    "W0719", # Raising too general exception
 ]

sapporo/auth.py

@@ -89,7 +89,7 @@ def get_auth_config() -> AuthConfig:
 
 
 class HTTPBearerCustom(HTTPBearer):
-    async def __call__(self, request: Request) -> str:
+    async def __call__(self, request: Request) -> str:  # type: ignore
         authorization = request.headers.get("Authorization")
         scheme, credentials = get_authorization_scheme_param(authorization)
         if not (authorization and scheme and credentials):
@@ -262,7 +262,7 @@ async def external_create_access_token(username: str, password: str) -> str:
         async with httpx.AsyncClient() as client:
             res = await client.post(token_url, data=data, headers=headers, follow_redirects=True)
             res.raise_for_status()
-            return res.json()["access_token"]
+            return res.json()["access_token"]  # type: ignore
     except Exception as e:
         raise HTTPException(
             status_code=status.HTTP_401_UNAUTHORIZED,

sapporo/database.py

@@ -51,7 +51,7 @@ def get_session() -> Generator[Session, None, None]:
 # === Models ===
 
 
-class Run(SQLModel, table=True):  # type: ignore
+class Run(SQLModel, table=True):
     __tablename__ = "runs"
 
     run_id: str = Field(primary_key=True)
@@ -125,7 +125,7 @@ def add_run_db(
 
 
 def system_state_counts() -> Dict[str, int]:
-    statement = select(Run.state, func.count(Run.run_id)).group_by(Run.state)  # pylint: disable=E1102
+    statement = select(Run.state, func.count(Run.run_id)).group_by(Run.state)  # type: ignore # pylint: disable=E1102
     with get_session() as session:
         results = session.exec(statement).all()
 
@@ -146,7 +146,7 @@ def _encode_page_token(last_run: Run) -> str:
 
 def _decode_page_token(page_token: str) -> Dict[str, str]:
     token_data = base64.urlsafe_b64decode(page_token).decode("utf-8")
-    return json.loads(token_data)
+    return json.loads(token_data)  # type: ignore
 
 
 def list_runs_db(

sapporo/ro_crate.py

@@ -18,6 +18,7 @@
 import magic
 import multiqc
 from fastapi import UploadFile
+from multiqc.core.update_config import ClConfig
 from pydantic import BaseModel, TypeAdapter
 from rocrate.model.computationalworkflow import ComputationalWorkflow
 from rocrate.model.computerlanguage import ComputerLanguage
@@ -510,7 +511,15 @@ def add_multiqc_stats(crate: ROCrate, run_dir: Path, create_action_ins: ContextE
     stderr = io.StringIO()
     try:
         with contextlib.redirect_stdout(stdout), contextlib.redirect_stderr(stderr):
-            multiqc.run(str(run_dir), outdir=str(run_dir), data_format="json", no_report=True, quiet=True)
+            multiqc.run(
+                str(run_dir),
+                cfg=ClConfig(
+                    output_dir=str(run_dir),
+                    data_format="json",
+                    make_report=False,
+                    quiet=True,
+                ),
+            )
     except Exception:  # pylint: disable=broad-except
         print(stderr.getvalue(), file=sys.stderr)
         return
@@ -672,7 +681,7 @@ def extract_exe_dir_file_ids(crate: ROCrate) -> List[str]:
     for entity in crate.get_entities():
         if isinstance(entity, Dataset):
             if str(entity["@id"]) == f"{RUN_DIR_STRUCTURE['exe_dir']}/":
-                return get_norm_value(entity, "hasPart")
+                return get_norm_value(entity, "hasPart")  # type: ignore
     return []
 
 

sapporo/run.py

@@ -85,8 +85,8 @@ def wf_engine_params_to_str(run_request: RunRequestForm) -> str:
         service_info = create_service_info()
         default_wf_engine_params = service_info.default_workflow_engine_parameters.get(wf_engine or "", [])  # pylint: disable=E1101
         for param in default_wf_engine_params:
-            params.append(param.get("name", ""))
-            params.append(param.get("default_value", ""))
+            params.append(param.name or "")
+            params.append(param.default_value or "")
     else:
         for key, value in wf_engine_params.items():
             params.append(str(key))
@@ -120,9 +120,12 @@ def download_wf_attachment(run_id: str, run_request: RunRequestForm) -> None:
                     res.raise_for_status()
                     with file_path.open(mode="wb") as f:
                         f.write(res.content)
+            except httpx.HTTPStatusError as e:
+                # Because it is a background task, raise Exception instead of HTTPException
+                raise Exception(f"Failed to download workflow attachment {obj}: {res.status_code} {res.text}") from e
             except Exception as e:
                 # Because it is a background task, raise Exception instead of HTTPException
-                raise Exception(f"Failed to download workflow attachment {obj}: {res.status_code} {res.text}") from e  # pylint: disable=W0719
+                raise Exception(f"Failed to download workflow attachment {obj}: {e}") from e
 
 
 def fork_run(run_id: str) -> None:

sapporo/schemas.py

@@ -277,7 +277,7 @@ class RunSummary(RunStatus):
 
 
 class RunListResponse(BaseModel):
-    runs: Optional[List[Union[RunStatus, RunSummary]]] = Field(
+    runs: Optional[Union[List[RunStatus], List[RunSummary]]] = Field(
         None,
         description=GA4GH_WES_SCHEMAS["RunListResponse"]["properties"]["runs"]["description"],
     )