Swift: add unit tests to code generation

Tests can be run with
```
bazel test //swift/codegen:tests
```

Coverage can be checked installing `pytest-cov` and running
```
pytest --cov=swift/codegen swift/codegen/test
```

Author: redsun82

.github/workflows/swift-codegen.yml

@@ -19,9 +19,14 @@ jobs:
           cache: 'pip'
       - uses: ./.github/actions/fetch-codeql
       - uses: bazelbuild/setup-bazelisk@v2
-      - name: Check code generation
+      - name: Install dependencies
         run: |
           pip install -r swift/codegen/requirements.txt
+      - name: Run unit tests
+        run: |
+          bazel test //swift/codegen:tests --test_output=errors
+      - name: Check that code was generated
+        run: |
           bazel run //swift/codegen
           git add swift
           git diff --exit-code --stat HEAD

.pre-commit-config.yaml

@@ -40,3 +40,10 @@ repos:
         language: system
         entry: bazel run //swift/codegen
         pass_filenames: false
+
+      - id: swift-codegen-unit-tests
+        name: Run Swift code generation unit tests
+        files: ^swift/codegen
+        language: system
+        entry: bazel test //swift/codegen:tests
+        pass_filenames: false

conftest.py

@@ -0,0 +1 @@
+# this empty file adds the repo root to PYTHON_PATH when running pytest

swift/codegen/.coverage

@@ -1,4 +1,31 @@
 py_binary(
     name = "codegen",
-    srcs = glob(["**/*.py"]),
+    srcs = glob([
+        "lib/*.py",
+        "*.py",
+    ]),
+)
+
+py_library(
+    name = "test_utils",
+    testonly = True,
+    srcs = ["test/utils.py"],
+    deps = [":codegen"],
+)
+
+[
+    py_test(
+        name = src[len("test/"):-len(".py")],
+        size = "small",
+        srcs = [src],
+        deps = [
+            ":codegen",
+            ":test_utils",
+        ],
+    )
+    for src in glob(["test/test_*.py"])
+]
+
+test_suite(
+    name = "tests",
 )

swift/codegen/BUILD.bazel

@@ -3,8 +3,8 @@
 
 import inflection
 
-from lib import paths, schema, generator
-from lib.dbscheme import *
+from swift.codegen.lib import paths, schema, generator
+from swift.codegen.lib.dbscheme import *
 
 log = logging.getLogger(__name__)
 
@@ -60,7 +60,7 @@ def cls_to_dbscheme(cls: schema.Class):
 
 
 def get_declarations(data: schema.Schema):
-    return [d for cls in data.classes.values() for d in cls_to_dbscheme(cls)]
+    return [d for cls in data.classes for d in cls_to_dbscheme(cls)]
 
 
 def get_includes(data: schema.Schema, include_dir: pathlib.Path):
@@ -73,11 +73,10 @@ def get_includes(data: schema.Schema, include_dir: pathlib.Path):
 
 
 def generate(opts, renderer):
-    input = opts.schema.resolve()
-    out = opts.dbscheme.resolve()
+    input = opts.schema
+    out = opts.dbscheme
 
-    with open(input) as src:
-        data = schema.load(src)
+    data = schema.load(input)
 
     dbscheme = DbScheme(src=input.relative_to(paths.swift_dir),
                         includes=get_includes(data, include_dir=input.parent),

swift/codegen/dbschemegen.py

@@ -10,13 +10,17 @@
 
 def _init_options():
     Option("--verbose", "-v", action="store_true")
-    Option("--schema", tags=["schema"], type=pathlib.Path, default=paths.swift_dir / "codegen/schema.yml")
-    Option("--dbscheme", tags=["dbscheme"], type=pathlib.Path, default=paths.swift_dir / "ql/lib/swift.dbscheme")
-    Option("--ql-output", tags=["ql"], type=pathlib.Path, default=paths.swift_dir / "ql/lib/codeql/swift/generated")
-    Option("--ql-stub-output", tags=["ql"], type=pathlib.Path, default=paths.swift_dir / "ql/lib/codeql/swift/elements")
+    Option("--schema", tags=["schema"], type=_abspath, default=paths.swift_dir / "codegen/schema.yml")
+    Option("--dbscheme", tags=["dbscheme"], type=_abspath, default=paths.swift_dir / "ql/lib/swift.dbscheme")
+    Option("--ql-output", tags=["ql"], type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/generated")
+    Option("--ql-stub-output", tags=["ql"], type=_abspath, default=paths.swift_dir / "ql/lib/codeql/swift/elements")
     Option("--codeql-binary", tags=["ql"], default="codeql")
 
 
+def _abspath(x):
+    return pathlib.Path(x).resolve()
+
+
 _options = collections.defaultdict(list)
 
 

swift/codegen/lib/options.py

@@ -5,13 +5,16 @@
 import os
 
 try:
-    _workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY'])  # <- means we are using bazel run
+    _workspace_dir = pathlib.Path(os.environ['BUILD_WORKSPACE_DIRECTORY']).resolve()  # <- means we are using bazel run
     swift_dir = _workspace_dir / 'swift'
-    lib_dir = swift_dir / 'codegen' / 'lib'
 except KeyError:
     _this_file = pathlib.Path(__file__).resolve()
     swift_dir = _this_file.parents[2]
-    lib_dir = _this_file.parent
 
+lib_dir = swift_dir / 'codegen' / 'lib'
+templates_dir = lib_dir / 'templates'
 
-exe_file = pathlib.Path(sys.argv[0]).resolve()
+try:
+    exe_file = pathlib.Path(sys.argv[0]).resolve().relative_to(swift_dir)
+except ValueError:
+    exe_file = pathlib.Path(sys.argv[0]).name

swift/codegen/lib/paths.py

@@ -0,0 +1,88 @@
+import pathlib
+from dataclasses import dataclass, field
+from typing import List, ClassVar
+
+import inflection
+
+
+@dataclass
+class QlParam:
+    param: str
+    type: str = None
+    first: bool = False
+
+
+@dataclass
+class QlProperty:
+    singular: str
+    type: str
+    tablename: str
+    tableparams: List[QlParam]
+    plural: str = None
+    params: List[QlParam] = field(default_factory=list)
+    first: bool = False
+    local_var: str = "x"
+
+    def __post_init__(self):
+        if self.params:
+            self.params[0].first = True
+        while self.local_var in (p.param for p in self.params):
+            self.local_var += "_"
+        assert self.tableparams
+        if self.type_is_class:
+            self.tableparams = [x if x != "result" else self.local_var for x in self.tableparams]
+        self.tableparams = [QlParam(x) for x in self.tableparams]
+        self.tableparams[0].first = True
+
+    @property
+    def indefinite_article(self):
+        if self.plural:
+            return "An" if self.singular[0] in "AEIO" else "A"
+
+    @property
+    def type_is_class(self):
+        return self.type[0].isupper()
+
+
+@dataclass
+class QlClass:
+    template: ClassVar = 'ql_class'
+
+    name: str
+    bases: List[str] = field(default_factory=list)
+    final: bool = False
+    properties: List[QlProperty] = field(default_factory=list)
+    dir: pathlib.Path = pathlib.Path()
+    imports: List[str] = field(default_factory=list)
+
+    def __post_init__(self):
+        self.bases = sorted(self.bases)
+        if self.properties:
+            self.properties[0].first = True
+
+    @property
+    def db_id(self):
+        return "@" + inflection.underscore(self.name)
+
+    @property
+    def root(self):
+        return not self.bases
+
+    @property
+    def path(self):
+        return self.dir / self.name
+
+
+@dataclass
+class QlStub:
+    template: ClassVar = 'ql_stub'
+
+    name: str
+    base_import: str
+
+
+@dataclass
+class QlImportList:
+    template: ClassVar = 'ql_imports'
+
+    imports: List[str] = field(default_factory=list)

swift/codegen/lib/ql.py

@@ -19,8 +19,7 @@ class Renderer:
     """ Template renderer using mustache templates in the `templates` directory """
 
     def __init__(self):
-        self.r = pystache.Renderer(search_dirs=str(paths.lib_dir / "templates"), escape=lambda u: u)
-        self.generator = paths.exe_file.relative_to(paths.swift_dir)
+        self._r = pystache.Renderer(search_dirs=str(paths.lib_dir / "templates"), escape=lambda u: u)
         self.written = set()
 
     def render(self, data, output: pathlib.Path):
@@ -32,7 +31,7 @@ def render(self, data, output: pathlib.Path):
         """
         mnemonic = type(data).__name__
         output.parent.mkdir(parents=True, exist_ok=True)
-        data = self.r.render_name(data.template, data, generator=self.generator)
+        data = self._r.render_name(data.template, data, generator=paths.exe_file)
         with open(output, "w") as out:
             out.write(data)
         log.debug(f"generated {mnemonic} {output.name}")
@@ -41,6 +40,5 @@ def render(self, data, output: pathlib.Path):
     def cleanup(self, existing):
         """ Remove files in `existing` for which no `render` has been called """
         for f in existing - self.written:
-            if f.is_file():
-                f.unlink()
-                log.info(f"removed {f.name}")
+            f.unlink(missing_ok=True)
+            log.info(f"removed {f.name}")

swift/codegen/lib/render.py

@@ -3,7 +3,6 @@
 import pathlib
 import re
 from dataclasses import dataclass, field
-from enum import Enum, auto
 from typing import List, Set, Dict, ClassVar
 
 import yaml
@@ -47,7 +46,7 @@ class Class:
 
 @dataclass
 class Schema:
-    classes: Dict[str, Class]
+    classes: List[Class]
     includes: Set[str] = field(default_factory=set)
 
 
@@ -65,6 +64,7 @@ def _parse_property(name, type):
 
 class _DirSelector:
     """ Default output subdirectory selector for generated QL files, based on the `_directories` global field"""
+
     def __init__(self, dir_to_patterns):
         self.selector = [(re.compile(p), pathlib.Path(d)) for d, p in dir_to_patterns]
         self.selector.append((re.compile(""), pathlib.Path()))
@@ -73,19 +73,19 @@ def get(self, name):
         return next(d for p, d in self.selector if p.search(name))
 
 
-def load(file):
-    """ Parse the schema from `file` """
-    data = yaml.load(file, Loader=yaml.SafeLoader)
+def load(path):
+    """ Parse the schema from the file at `path` """
+    with open(path) as input:
+        data = yaml.load(input, Loader=yaml.SafeLoader)
     grouper = _DirSelector(data.get("_directories", {}).items())
-    ret = Schema(classes={cls: Class(cls, dir=grouper.get(cls)) for cls in data if not cls.startswith("_")},
-                 includes=set(data.get("_includes", [])))
-    assert root_class_name not in ret.classes
-    ret.classes[root_class_name] = Class(root_class_name)
+    classes = {root_class_name: Class(root_class_name)}
+    assert root_class_name not in data
+    classes.update((cls, Class(cls, dir=grouper.get(cls))) for cls in data if not cls.startswith("_"))
     for name, info in data.items():
         if name.startswith("_"):
             continue
         assert name[0].isupper()
-        cls = ret.classes[name]
+        cls = classes[name]
         for k, v in info.items():
             if not k.startswith("_"):
                 cls.properties.append(_parse_property(k, v))
@@ -94,11 +94,11 @@ def load(file):
                     v = [v]
                 for base in v:
                     cls.bases.add(base)
-                    ret.classes[base].derived.add(name)
+                    classes[base].derived.add(name)
             elif k == "_dir":
                 cls.dir = pathlib.Path(v)
         if not cls.bases:
             cls.bases.add(root_class_name)
-            ret.classes[root_class_name].derived.add(name)
+            classes[root_class_name].derived.add(name)
 
-    return ret
+    return Schema(classes=list(classes.values()), includes=set(data.get("_includes", [])))