[.github] Add helper to return in-memory "model" of spec folder

.github/src/array.js

@@ -0,0 +1,11 @@
+// @ts-check
+
+/**
+ * @template T, U
+ * @param {T[]} array
+ * @param {(value: T, index: number, array: T[]) => Promise<U>} callbackfn
+ * @returns {Promise<U[]>}
+ */
+export async function mapAsync(array, callbackfn) {
+  return Promise.all(array.map(callbackfn));
+}

.github/src/spec-model.js

@@ -0,0 +1,152 @@
+// @ts-check
+import $RefParser from "@apidevtools/json-schema-ref-parser";
+import { readFile, readdir } from "fs/promises";
+import yaml from "js-yaml";
+import { marked } from "marked";
+import { dirname, join, resolve } from "path";
+import { fileURLToPath } from "url";
+import { readme } from "../src/changed-files.js";
+import { mapAsync } from "./array.js";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+/**
+ * @typedef {Object} SpecModel
+ * @prop {Map<string, Readme>} readmes
+ */
+
+/**
+ * @typedef {Object} Readme
+ * @prop {string} path
+ * @prop {string} content
+ * @prop {Object} globalConfig
+ * @prop {Map<string, Swagger[]>} tags
+ */
+
+/**
+ * @typedef {Object} Swagger
+ * @prop {string} path
+ * @prop {string} content
+ * @prop {Set<string>} refs
+ */
+
+/**
+ * @param {string} folder
+ * @param {Object} [options]
+ * @param {import('./types.js').ILogger} [options.logger]
+ * @returns {Promise<SpecModel>} All input files for all tags
+ */
+export async function getSpecModel(folder, options = {}) {
+  console.log(options);
+
+  const files = await readdir(join(__dirname, "..", "..", folder), {
+    recursive: true,
+  });
+  const readmes = files.filter(readme);
+
+  return {
+    readmes: new Map(
+      await mapAsync(readmes, async (r) => {
+        return [join(folder, r), await getReadme(join(folder, r))];
+      }),
+    ),
+  };
+}
+
+/**
+ * @param {string} path
+ * @returns {Promise<Readme>}
+ */
+async function getReadme(path) {
+  // TODO: Do not assume location is with respect to repo root, could be reading
+  // files from a different root location (e.g. "before" state of repo in
+  // another folder).
+  const content = await readFile(join(__dirname, "..", "..", path), {
+    encoding: "utf8",
+  });
+
+  const tokens = marked.lexer(content);
+
+  /** @type import("marked").Tokens.Code[] */
+  const yamlBlocks = tokens
+    .filter((token) => token.type === "code")
+    .map((token) => /** @type import("marked").Tokens.Code */ (token))
+    // Include default block and tagged blocks (```yaml $(tag) == 'package-2021-11-01')
+    .filter((token) => token.lang?.toLowerCase().startsWith("yaml"));
+
+  const globalConfigYamlBlocks = yamlBlocks.filter(
+    (token) => token.lang === "yaml",
+  );
+
+  const globalConfig = globalConfigYamlBlocks.reduce(
+    (obj, token) => Object.assign(obj, yaml.load(token.text)),
+    {},
+  );
+
+  /** @prop {Map<string, Swagger[]>} */
+  const tags = new Map();
+  for (const block of yamlBlocks) {
+    const tagName =
+      block.lang?.match(/yaml \$\(tag\) == '([^']*)'/)?.[1] || "default";
+
+    if (tagName === "default") {
+      // Skip yaml blocks where this is no tag
+      continue;
+    }
+    const obj = /** @type {any} */ (yaml.load(block.text));
+
+    /** @type Swagger[] */
+    const swaggers = [];
+    for (const swaggerPath of obj["input-file"]) {
+      const swagger = await getSwagger(join(dirname(path), swaggerPath));
+      swaggers.push(swagger);
+    }
+
+    tags.set(tagName, swaggers);
+  }
+
+  const readme = {
+    path,
+    content,
+    globalConfig,
+    tags,
+  };
+
+  return readme;
+}
+
+/**
+ * @param {string} path
+ * @returns {Promise<Swagger>}
+ */
+async function getSwagger(path) {
+  // TODO: Do not assume location is with respect to repo root, could be reading
+  // files from a different root location (e.g. "before" state of repo in
+  // another folder).
+
+  const fullPath = join(__dirname, "..", "..", path);
+
+  const content = await readFile(fullPath, { encoding: "utf8" });
+
+  return {
+    path,
+    content,
+    refs: await getExternalFileRefs(fullPath),
+  };
+}
+
+/**
+ * @param {string} path
+ * @returns {Promise<Set<string>>}
+ */
+async function getExternalFileRefs(path) {
+  const schema = await $RefParser.resolve(path);
+
+  const refs = schema
+    .paths()
+    .filter((p) => p !== path && !p.startsWith("#"))
+    .map((p) => resolve(path, p));
+
+  return new Set(refs);
+}

.github/test/spec-model.test.js

@@ -0,0 +1,100 @@
+import { readFileSync } from "fs";
+import { dirname, join } from "path";
+import { fileURLToPath } from "url";
+import { describe, it } from "vitest";
+import { getSpecModel } from "../src/spec-model.js";
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+const repoRoot = join(__dirname, "..", "..");
+
+describe("spec-model", () => {
+  it("getSpecModel", async ({ expect }) => {
+    const readmePath =
+      "specification/contosowidgetmanager/resource-manager/readme.md";
+    const readmeContent = readFileSync(join(repoRoot, readmePath), {
+      encoding: "utf8",
+    });
+
+    const swaggerPathPreview =
+      "specification/contosowidgetmanager/resource-manager/Microsoft.Contoso/preview/2021-10-01-preview/contoso.json";
+    const swaggerContentPreview = readFileSync(
+      join(repoRoot, swaggerPathPreview),
+      {
+        encoding: "utf8",
+      },
+    );
+
+    const swaggerPathStable =
+      "specification/contosowidgetmanager/resource-manager/Microsoft.Contoso/stable/2021-11-01/contoso.json";
+    const swaggerContentStable = readFileSync(
+      join(repoRoot, swaggerPathStable),
+      {
+        encoding: "utf8",
+      },
+    );
+
+    const expected = {
+      readmes: new Map([
+        [
+          readmePath,
+          {
+            path: readmePath,
+            content: readmeContent,
+            globalConfig: {
+              "openapi-type": "arm",
+              "openapi-subtype": "rpaas",
+              tag: "package-2021-11-01",
+            },
+            tags: new Map([
+              [
+                "package-2021-11-01",
+                [
+                  {
+                    path: swaggerPathStable,
+                    content: swaggerContentStable,
+                    refs: new Set([
+                      "/home/mharder/specs-mh/specification/common-types/resource-management/v5/types.json",
+                      "/home/mharder/specs-mh/specification/contosowidgetmanager/resource-manager/Microsoft.Contoso/stable/2021-11-01/examples/Operations_List.json",
+                      "/home/mharder/specs-mh/specification/contosowidgetmanager/resource-manager/Microsoft.Contoso/stable/2021-11-01/examples/Employees_ListBySubscription.json",
+                      "/home/mharder/specs-mh/specification/contosowidgetmanager/resource-manager/Microsoft.Contoso/stable/2021-11-01/examples/Employees_ListByResourceGroup.json",
+                      "/home/mharder/specs-mh/specification/contosowidgetmanager/resource-manager/Microsoft.Contoso/stable/2021-11-01/examples/Employees_Get.json",
+                      "/home/mharder/specs-mh/specification/contosowidgetmanager/resource-manager/Microsoft.Contoso/stable/2021-11-01/examples/Employees_CreateOrUpdate.json",
+                      "/home/mharder/specs-mh/specification/contosowidgetmanager/resource-manager/Microsoft.Contoso/stable/2021-11-01/examples/Employees_Update.json",
+                      "/home/mharder/specs-mh/specification/contosowidgetmanager/resource-manager/Microsoft.Contoso/stable/2021-11-01/examples/Employees_Delete.json",
+                    ]),
+                  },
+                ],
+              ],
+              [
+                "package-2021-10-01-preview",
+                [
+                  {
+                    path: swaggerPathPreview,
+                    content: swaggerContentPreview,
+                    refs: new Set([
+                      "/home/mharder/specs-mh/specification/common-types/resource-management/v5/types.json",
+                      "/home/mharder/specs-mh/specification/contosowidgetmanager/resource-manager/Microsoft.Contoso/preview/2021-10-01-preview/examples/Operations_List.json",
+                      "/home/mharder/specs-mh/specification/contosowidgetmanager/resource-manager/Microsoft.Contoso/preview/2021-10-01-preview/examples/Employees_ListBySubscription.json",
+                      "/home/mharder/specs-mh/specification/contosowidgetmanager/resource-manager/Microsoft.Contoso/preview/2021-10-01-preview/examples/Employees_ListByResourceGroup.json",
+                      "/home/mharder/specs-mh/specification/contosowidgetmanager/resource-manager/Microsoft.Contoso/preview/2021-10-01-preview/examples/Employees_Get.json",
+                      "/home/mharder/specs-mh/specification/contosowidgetmanager/resource-manager/Microsoft.Contoso/preview/2021-10-01-preview/examples/Employees_CreateOrUpdate.json",
+                      "/home/mharder/specs-mh/specification/contosowidgetmanager/resource-manager/Microsoft.Contoso/preview/2021-10-01-preview/examples/Employees_Update.json",
+                      "/home/mharder/specs-mh/specification/contosowidgetmanager/resource-manager/Microsoft.Contoso/preview/2021-10-01-preview/examples/Employees_Delete.json",
+                    ]),
+                  },
+                ],
+              ],
+            ]),
+          },
+        ],
+      ]),
+    };
+
+    const specModel = await getSpecModel(
+      "specification/contosowidgetmanager/resource-manager",
+    );
+
+    expect(specModel).toEqual(expected);
+  });
+});