flesh out schema syntax

Author: bollwyvl

pyodide_lock/spec.py

@@ -2,42 +2,124 @@
 from pathlib import Path
 from typing import Literal
 
-from pydantic import BaseModel, Extra
+from pydantic import BaseModel, Extra, Field
 
 from .utils import (
+    _add_required,
     _generate_package_hash,
     _wheel_depends,
     parse_top_level_import_name,
 )
 
 
 class InfoSpec(BaseModel):
-    arch: Literal["wasm32", "wasm64"] = "wasm32"
-    platform: str
-    version: str
-    python: str
+    arch: Literal["wasm32", "wasm64"] = Field(
+        default="wasm32",
+        description=(
+            "the short name for the compiled architecture, available in "
+            "dependency markers as `platform_machine`"
+        ),
+    )
+    platform: str = Field(
+        description=(
+            "the emscripten virtual machine for which this distribution is "
+            " compiled, not available directly in a dependency marker: use e.g. "
+            """`plaform_system == "Emscripten" and platform_release == "3.1.45"`"""
+        ),
+        examples=["emscripten_3_1_32", "emscripten_3_1_45"],
+    )
+    version: str = Field(
+        description="the PEP 440 version of pyodide",
+        examples=["0.24.1", "0.23.3"],
+    )
+    python: str = Field(
+        description=(
+            "the version of python for which this lockfile is valid, available in "
+            "version markers as `platform_machine`"
+        ),
+        examples=["3.11.2", "3.11.3"],
+    )
 
     class Config:
         extra = Extra.forbid
 
+        schema_extra = _add_required(
+            "arch",
+            description=(
+                "the execution environment in which the packages in this lockfile "
+                "can be installed"
+            ),
+        )
+
 
 class PackageSpec(BaseModel):
-    name: str
-    version: str
-    file_name: str
-    install_dir: str
-    sha256: str = ""
+    name: str = Field(
+        description="the verbatim name as found in the package's metadata",
+        examples=["pyodide-lock", "PyYAML", "ruamel.yaml"],
+    )
+    version: str = Field(
+        description="the reported version of the package",
+        examples=["0.1.0", "1.0.0a0", "1.0.0a0.post1"],
+    )
+    file_name: str = Field(
+        format="uri-reference",
+        description="the URL of the file",
+        examples=[
+            "pyodide_lock-0.1.0-py3-none-any.whl",
+            "https://files.pythonhosted.org/packages/py3/m/micropip/micropip-0.5.0-py3-none-any.whl",
+        ],
+    )
+    install_dir: str = Field(
+        default="site",
+        description="the file system destination for a package's data",
+        examples=["dynlib", "stdlib"],
+    )
+    sha256: str = Field(description="the SHA256 cryptographic hash of the file")
     package_type: Literal[
         "package", "cpython_module", "shared_library", "static_library"
-    ] = "package"
-    imports: list[str] = []
-    depends: list[str] = []
-    unvendored_tests: bool = False
+    ] = Field(
+        default="package",
+        description="the top-level kind of content provided by this package",
+    )
+    imports: list[str] = Field(
+        default=[],
+        description=(
+            "the importable names provided by this package."
+            "note that PEP 420 namespace packages will likely not be correctly found."
+        ),
+    )
+    depends: list[str] = Field(
+        default=[],
+        unique_items=True,
+        description=(
+            "package names that must be installed when this package in installed"
+        ),
+    )
+    unvendored_tests: bool = Field(
+        default=False,
+        description=(
+            "whether the package's tests folder have been repackaged "
+            "as a separate archive"
+        ),
+    )
     # This field is deprecated
-    shared_library: bool = False
+    shared_library: bool = Field(
+        default=False,
+        deprecated=True,
+        description=(
+            "(deprecated) whether this package is a shared library. "
+            "replaced with `package_type: shared_library`"
+        ),
+    )
 
     class Config:
         extra = Extra.forbid
+        schema_extra = _add_required(
+            "depends",
+            "imports",
+            "install_dir",
+            description="a single pyodide-compatible file",
+        )
 
     @classmethod
     def from_wheel(
@@ -78,11 +160,27 @@ def update_sha256(self, path: Path) -> "PackageSpec":
 class PyodideLockSpec(BaseModel):
     """A specification for the pyodide-lock.json file."""
 
-    info: InfoSpec
-    packages: dict[str, PackageSpec]
+    info: InfoSpec = Field(
+        description=(
+            "the execution environment in which the packages in this lockfile "
+            "can be installable"
+        )
+    )
+    packages: dict[str, PackageSpec] = Field(
+        default={},
+        description="a set of packages keyed by name",
+    )
 
     class Config:
         extra = Extra.forbid
+        schema_extra = {
+            "$schema": "https://json-schema.org/draft/2019-09/schema#",
+            "$id": ("https://pyodide.org/schema/pyodide-lock/v0-lockfile.schema.json"),
+            "description": (
+                "a description of a viable pyodide runtime environment, "
+                "as defined by pyodide-lock"
+            ),
+        }
 
     @classmethod
     def from_json(cls, path: Path) -> "PyodideLockSpec":

pyodide_lock/utils.py

@@ -4,8 +4,9 @@
 import sys
 import zipfile
 from collections import deque
+from collections.abc import Callable
 from pathlib import Path
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
 
 if TYPE_CHECKING:
     from pkginfo import Distribution
@@ -130,3 +131,13 @@ def _wheel_depends(
             depends += [canonicalize_name(req.name)]
 
     return sorted(set(depends))
+
+
+def _add_required(
+    *field_names: str, **extra: Any
+) -> Callable[[dict[str, Any], Any], None]:
+    def add_required(schema: dict[str, Any], *args: Any) -> None:
+        schema["required"] = sorted([*field_names, *schema.get("required", [])])
+        schema.update(extra)
+
+    return add_required

pyproject.toml

@@ -26,13 +26,22 @@ wheel = [
     "pkginfo",
     "packaging",
 ]
+schema = [
+    "jsonschema >=4",
+    "rfc3986-validator",
+]
 dev = [
     "pytest",
     "pytest-cov",
     "build",
     # from wheel
     "pkginfo",
     "packaging",
+    # from schema
+    "jsonschema >=4",
+    "rfc3986-validator",
+    # stubs
+    "types-jsonschema",
 ]
 
 [project.urls]

tests/conftest.py

@@ -0,0 +1,25 @@
+import gzip
+import shutil
+from pathlib import Path
+
+import pytest
+
+HERE = Path(__file__).parent
+DATA_DIR = Path(__file__).parent / "data"
+SPEC_JSON_GZ = sorted(DATA_DIR.glob("*.json.gz"))
+
+
+@pytest.fixture(params=SPEC_JSON_GZ)
+def an_historic_spec_gz(request) -> Path:
+    return request.param
+
+
+@pytest.fixture
+def an_historic_spec_json(tmp_path: Path, an_historic_spec_gz: Path) -> Path:
+    target_path = tmp_path / "pyodide-lock.json"
+
+    with gzip.open(an_historic_spec_gz) as fh_in:
+        with target_path.open("wb") as fh_out:
+            shutil.copyfileobj(fh_in, fh_out)
+
+    return target_path

tests/test_schema.py

@@ -0,0 +1,60 @@
+import json
+from pathlib import Path
+from typing import Any
+
+import pytest
+from jsonschema import ValidationError
+from jsonschema.validators import Draft201909Validator
+
+from pyodide_lock import PyodideLockSpec
+
+#: a schema that constrains the schema itself for schema syntax
+META_SCHEMA = {
+    "type": "object",
+    "required": ["description", "$id", "$schema"],
+    "properties": {
+        "description": {"type": "string"},
+        "$id": {"type": "string", "format": "uri"},
+        "$schema": {"type": "string", "format": "uri"},
+        "definitions": {"patternProperties": {".*": {"required": ["description"]}}},
+    },
+}
+
+Validator = Draft201909Validator
+FORMAT_CHECKER = Draft201909Validator.FORMAT_CHECKER
+
+
+@pytest.fixture
+def schema() -> dict[str, Any]:
+    return PyodideLockSpec.schema()
+
+
+@pytest.fixture
+def spec_validator(schema: dict[str, Any]) -> Validator:
+    return Validator(schema, format_checker=FORMAT_CHECKER)
+
+
+def test_documentation(schema: dict[str, Any]) -> None:
+    meta_validator = Validator(META_SCHEMA, format_checker=FORMAT_CHECKER)
+    _assert_validation_errors(meta_validator, schema)
+
+
+def test_validate(an_historic_spec_json: Path, spec_validator: Validator) -> None:
+    spec_json = json.loads(an_historic_spec_json.read_text(encoding="utf-8"))
+    _assert_validation_errors(spec_validator, spec_json)
+
+
+def _assert_validation_errors(
+    validator: Draft201909Validator,
+    instance: dict[str, Any],
+    expect_errors: list[str] | None = None,
+) -> None:
+    expect_errors = expect_errors or []
+    expect_error_count = len(expect_errors)
+
+    errors: list[ValidationError] = list(validator.iter_errors(instance))
+    error_count = len(errors)
+
+    print("\n".join([f"""{err.json_path}: {err.message}""" for err in errors]))
+
+    assert error_count == expect_error_count

tests/test_spec.py

@@ -1,5 +1,3 @@
-import gzip
-import shutil
 from copy import deepcopy
 from pathlib import Path
 
@@ -33,17 +31,10 @@
 }
 
 
-@pytest.mark.parametrize("pyodide_version", ["0.22.1", "0.23.3"])
-def test_lock_spec_parsing(pyodide_version, tmp_path):
-    source_path = DATA_DIR / f"pyodide-lock-{pyodide_version}.json.gz"
-    target_path = tmp_path / "pyodide-lock.json"
+def test_lock_spec_parsing(an_historic_spec_json: Path, tmp_path):
     target2_path = tmp_path / "pyodide-lock2.json"
 
-    with gzip.open(source_path) as fh_in:
-        with target_path.open("wb") as fh_out:
-            shutil.copyfileobj(fh_in, fh_out)
-
-    spec = PyodideLockSpec.from_json(target_path)
+    spec = PyodideLockSpec.from_json(an_historic_spec_json)
     spec.to_json(target2_path, indent=2)
 
     spec2 = PyodideLockSpec.from_json(target2_path)
@@ -53,6 +44,9 @@ def test_lock_spec_parsing(pyodide_version, tmp_path):
     for key in spec.packages:
         assert spec.packages[key] == spec2.packages[key]
 
+    with pytest.raises(ValueError, match="does not match package version"):
+        spec.check_wheel_filenames()
+
 
 def test_check_wheel_filenames():
     lock_data = deepcopy(LOCK_EXAMPLE)