refactor: move config validation to backend (#214)

* chore: rebase on main

* refactor: remove legacy config type

* refactor: move bundler files into own directory

* chore: rebase on main

* fix: default to not replacing a variable when not declared

* fix: tweak type of BundleSuccess config

* chore: rebase from main

* chore: bump deps

* chore: bump codehike deps

* chore: bump dependencies

* chore: lint and format

Author: cabljac

.nvmrc

@@ -0,0 +1 @@
+v18.0.0

api/package.json

@@ -1,47 +1,46 @@
 {
   "type": "module",
-  "name": "ts-express",
+  "name": "docs-page-api",
   "version": "1.0.0",
   "description": "template node and express app with typescript using esm ",
   "main": "src/app.ts",
   "author": "<[email protected]>",
   "license": "MIT",
   "dependencies": {
-    "@code-hike/mdx": "^v0.3.0-next.20",
+    "@code-hike/mdx": "^0.7.2",
     "@ltd/j-toml": "^1.30.0",
-    "@octokit/graphql": "^4.8.0",
+    "@octokit/graphql": "^5.0.0",
     "@types/cors": "^2.8.12",
     "@types/express": "^4.17.13",
-    "@types/lodash.get": "^4.4.6",
+    "@types/lodash.get": "^4.4.7",
     "@types/morgan": "^1.9.3",
-    "@types/node": "^16.7.1",
+    "@types/node": "^18.0.0",
     "a2a": "^0.2.0",
-    "camelcase": "^6.2.1",
+    "camelcase": "^7.0.0",
     "cors": "^2.8.5",
-    "dotenv": "^10.0.0",
-    "esbuild": "^0.14.2",
-    "express": "^4.17.1",
-    "express-basic-auth": "^1.1.1",
+    "dotenv": "^16.0.1",
+    "esbuild": "^0.14.47",
+    "express": "^4.18.1",
+    "express-basic-auth": "^1.2.1",
     "hast-util-parse-selector": "^3.1.0",
-    "http-status": "^1.5.0",
+    "http-status": "^1.5.2",
     "is-badge": "^2.1.0",
     "js-yaml": "^4.1.0",
     "lodash.get": "^4.4.2",
-    "mdx-bundler": "^8.0.0",
+    "mdx-bundler": "^9.0.1",
     "morgan": "^1.10.0",
-    "node-fetch": "^3.2.2",
-    "probot": "^12.2.1",
-    "react": "^17.0.2",
+    "node-fetch": "^3.2.6",
+    "probot": "^12.2.4",
+    "react": "^18.2.0",
     "rehype-accessible-emojis": "^0.3.2",
     "rehype-katex": "^6.0.2",
-    "rehype-slug": "^5.0.0",
+    "rehype-slug": "^5.0.1",
     "remark-comment": "^1.0.0",
     "remark-gfm": "^3.0.1",
     "remark-math": "^5.1.1",
     "remark-parse": "^10.0.1",
-    "shiki": "^0.9.15",
+    "shiki": "^0.10.1",
     "tiny-invariant": "^1.2.0",
-    "typescript": "4.4.4",
     "unist-util-visit": "^4.1.0"
   },
   "scripts": {
@@ -51,10 +50,10 @@
     "postinstall": "tsc"
   },
   "devDependencies": {
-    "@docs.page/server": "^1.0.0",
-    "@octokit/webhooks-types": "^5.3.0",
-    "nodemon": "^2.0.12",
-    "ts-node": "^10.4.0",
-    "typescript": "4.4.3"
+    "@docs.page/server": "^1.0.1",
+    "@octokit/webhooks-types": "^6.2.4",
+    "nodemon": "^2.0.18",
+    "ts-node": "^10.8.1",
+    "typescript": "4.7.4"
   }
 }

api/src/utils/bundler.ts api/src/bundler/bundler.ts

@@ -46,7 +46,7 @@ export async function bundle(
 
   const { code, frontmatter, errors } = await bundleMDX({
     source: rawText,
-    xdmOptions(options) {
+    mdxOptions(options) {
       // @ts-ignore TODO fix types
       options.remarkPlugins = [...(options.remarkPlugins ?? []), ...bundleOptions.remarkPlugins];
       // @ts-ignore TODO fix types

api/src/utils/getPlugins.ts api/src/bundler/getPlugins.ts

@@ -1,9 +1,9 @@
-import { theme } from '../utils/plugins/codeHikeTheme.js';
+import { theme } from './plugins/codeHikeTheme.js';
 import { remarkCodeHike } from '@code-hike/mdx';
 import remarkGfm from 'remark-gfm';
-import rehypeCodeBlocks from '../utils/plugins/rehype-code-blocks.js';
+import rehypeCodeBlocks from './plugins/rehype-code-blocks.js';
 import { rehypeAccessibleEmojis } from 'rehype-accessible-emojis';
-import rehypeInlineBadges from '../utils/plugins/rehype-inline-badges.js';
+import rehypeInlineBadges from './plugins/rehype-inline-badges.js';
 import rehypeSlug from 'rehype-slug';
 import remarkMath from 'remark-math';
 import rehypeKatex from 'rehype-katex';

api/src/utils/plugins/codeHikeTheme.ts api/src/bundler/plugins/codeHikeTheme.ts

@@ -1,6 +1,7 @@
 import { Request, Response } from 'express';
 import { BundleResponseData } from '@docs.page/server';
 import { Bundle, BundleError } from '../utils/bundle.js';
+import { formatConfigLocales } from '../utils/config.js';
 /**
  * Gets the API information.
  *
@@ -23,7 +24,7 @@ export const bundleRaw = async (
     markdown,
   });
   if (sourceConfig) {
-    bundleInstance.formatConfigLocales(sourceConfig);
+    bundleInstance.config = formatConfigLocales(sourceConfig, path);
   }
   try {
     const data = await bundleInstance.build();

api/src/utils/plugins/rehype-code-blocks.ts api/src/bundler/plugins/rehype-code-blocks.ts

@@ -1,18 +1,11 @@
-import { bundle } from './bundler.js';
-import { getPlugins } from './getPlugins.js';
+import { bundle } from '../bundler/bundler.js';
+import { getPlugins } from '../bundler/getPlugins.js';
 import { Contents, getConfigs, getGitHubContents } from './github.js';
-import { HeadingNode } from './plugins/rehype-headings.js';
+import { HeadingNode } from '../bundler/plugins/rehype-headings.js';
 import { formatSourceAndRef } from './ref.js';
 import { getRepositorySymLinks } from './symlinks.js';
-import {
-  hasLocales,
-  InputConfig,
-  OutputConfig,
-  defaultConfig,
-  ErrorReason,
-} from '@docs.page/server';
-import yaml from 'js-yaml';
-import toml from '@ltd/j-toml';
+import { OutputConfig, defaultConfig, ErrorReason } from '@docs.page/server';
+import { formatConfigLocales } from './config.js';
 
 type Frontmatter = Record<string, string>;
 
@@ -115,7 +108,8 @@ export class Bundle {
 
     this.repositoryFound = githubContents.repositoryFound;
 
-    this.formatConfigLocales(githubContents.config);
+    this.config = formatConfigLocales(githubContents.config, this.path);
+
     await this.matchSymLinks();
     if (githubContents.md === null) {
       throw new BundleError(404, "Couldn't find file", 'FILE_NOT_FOUND');
@@ -133,7 +127,8 @@ export class Bundle {
     if (!repositoryFound || !config) {
       throw new BundleError(404, 'Unable to fetch config file.');
     }
-    this.formatConfigLocales(config);
+
+    this.config = formatConfigLocales(config, this.path);
 
     return this.config;
   }
@@ -190,43 +185,6 @@ export class Bundle {
       }
     }
   }
-
-  formatConfigLocales(config?: { configJson?: string; configYaml?: string; configToml?: string }) {
-    if (!config?.configJson && !config?.configYaml && !config?.configToml) {
-      throw new BundleError(404, 'Not found: Config file missing', 'MISSING_CONFIG');
-    }
-    const { configJson, configYaml, configToml } = config;
-
-    let inputConfig: InputConfig = defaultConfig;
-    // TODO: validation of config?
-
-    try {
-      if (configJson) {
-        inputConfig = JSON.parse(configJson) as InputConfig;
-      } else if (configYaml) {
-        inputConfig = yaml.load(configYaml) as InputConfig;
-      } else if (configToml) {
-        //@ts-ignore
-        inputConfig = Object.assign({}, toml.parse(configToml) as InputConfig);
-      }
-    } catch (e) {
-      console.error(e);
-      throw new BundleError(500, 'Error parsing config', 'BAD_CONFIG');
-    }
-
-    if (hasLocales(inputConfig)) {
-      const defaulLocale = inputConfig.locales[0];
-
-      const currentLocale = this.path.split('/')[0] || defaulLocale;
-
-      this.config = {
-        ...inputConfig,
-        sidebar: inputConfig?.sidebar[currentLocale],
-      };
-    } else {
-      this.config = inputConfig;
-    }
-  }
 }
 
 export class BundleError extends Error {

api/src/utils/plugins/rehype-headings.ts api/src/bundler/plugins/rehype-headings.ts

@@ -1,11 +1,15 @@
+import { getBoolean, getNumber, getString, getValue } from './get.js';
 import get from 'lodash.get';
-import { getBoolean, getNumber, getString, getValue } from './get';
+
+import yaml from 'js-yaml';
+import toml from '@ltd/j-toml';
+import { OutputConfig } from '@docs.page/server';
 
 // Represents how the sidebar should look in the config file.
 export type SidebarItem = [string, Array<[string, string]>] | [string, string];
 
 // Merges in a user sidebar config and ensures all items are valid.
-function mergeSidebarConfig(json: Partial<ProjectConfig> | null): SidebarItem[] {
+function mergeSidebarConfig(json: Partial<OutputConfig> | null): SidebarItem[] {
   const sidebar = get(json, 'sidebar', defaultConfig.sidebar);
 
   if (!Array.isArray(sidebar)) {
@@ -57,54 +61,8 @@ function mergeSidebarConfig(json: Partial<ProjectConfig> | null): SidebarItem[]
  * This can be provided by creating a `docs.json` file at the root of your
  * repository.
  */
-export interface ProjectConfig {
-  // Project name.
-  name: string;
-  // URL to project logo.
-  logo: string;
-  // URL to project logo for dark mode
-  logoDark: string;
-  // URL to the favicon
-  favicon: string;
-  // Image to display as the social preview on shared URLs
-  socialPreview: string;
-  // Twitter tag for use in the header.
-  twitter: string;
-  // Whether the website should be indexable by search bots.
-  noindex: boolean;
-  // A color theme used for this project. Defaults to "#00bcd4".
-  theme: string;
-  // Docsearch Application ID. If populated, a search box with autocomplete will be rendered.
-  docsearch?: {
-    appId?: string;
-    apiKey: string;
-    indexName: string;
-  };
-  // Header navigation
-  // navigation: NavigationItem[];
-  // Sidebar
-  sidebar: SidebarItem[];
-  // Locales:
-  locales?: Record<string, string>;
-  // The depth to heading tags are linked. Set to 0 to remove any linking.
-  headerDepth: number;
-  // Variables which can be injected into the pages content.
-  variables: Record<string, string>;
-  // Adds Google Tag Manager to your documentation pages.
-  googleTagManager: string;
-  // Adds Google Analytics to your documentation pages.
-  googleAnalytics: string;
-  // Whether zoomable images are enabled by default
-  zoomImages: boolean;
-  // Whether CodeHike is enabled
-  experimentalCodehike: boolean;
-  // Whether Math is enabled
-  experimentalMath: boolean;
-  // Whether Next/Previous buttons are showing automatically based on the sidebar
-  automaticallyInferNextPrevious: boolean;
-}
 
-export const defaultConfig: ProjectConfig = {
+export const defaultConfig: OutputConfig = {
   name: '',
   logo: '',
   logoDark: '',
@@ -126,7 +84,7 @@ export const defaultConfig: ProjectConfig = {
 };
 
 // Merges any user config with default values.
-export function mergeConfig(json: Record<string, unknown>): ProjectConfig {
+export function mergeConfig(json: Record<string, unknown>): OutputConfig {
   return {
     name: getString(json, 'name', defaultConfig.name),
     logo: getString(json, 'logo', defaultConfig.logo),
@@ -166,25 +124,49 @@ export function mergeConfig(json: Record<string, unknown>): ProjectConfig {
   };
 }
 
-export async function getConfiguration({
-  owner,
-  repo,
-  ref,
-}: Record<string, string>): Promise<ProjectConfig> {
-  let config: ProjectConfig = defaultConfig;
+type Configs = {
+  configJson?: string;
+  configYaml?: string;
+  configToml?: string;
+};
+
+export function formatConfigLocales(configFiles: Configs, path: string): OutputConfig {
+  const { configJson, configYaml, configToml } = configFiles;
 
-  const host =
-    process.env.NODE_ENV === 'production' ? 'https://api.docs.page' : 'http://localhost:8000';
+  let config: Record<string, unknown> = {};
 
-  const endpoint = `${host}/config?owner=${owner}&repository=${repo}${
-    ref ? `&ref=${encodeURIComponent(ref)}` : ''
-  }`;
   try {
-    const res = await (await fetch(endpoint)).json();
-    config = res.config;
+    if (configJson) {
+      config = JSON.parse(configJson);
+    } else if (configYaml) {
+      config = yaml.load(configYaml) as Record<string, unknown>;
+    } else if (configToml) {
+      config = Object.assign({}, toml.parse(configToml));
+    }
   } catch (e) {
-    console.error(e);
+    console.error('Error parsing config, using default.');
+    return defaultConfig;
   }
 
-  return config;
+  if (
+    config.hasOwnProperty('locales') &&
+    Array.isArray(config.locales) &&
+    config.hasOwnProperty('sidebar')
+  ) {
+    // TODO: edge cases of bad configs, e.g what if locales is not an array?
+
+    const defaulLocale = config.locales[0];
+
+    const currentLocale = path.split('/')[0] || defaulLocale;
+
+    const sidebar = (getValue(config, 'sidebar') as Record<string, unknown>)[currentLocale];
+
+    config = {
+      ...config,
+      sidebar,
+    };
+
+    return mergeConfig(config);
+  }
+  return mergeConfig(config);
 }

api/src/utils/plugins/rehype-inline-badges.ts api/src/bundler/plugins/rehype-inline-badges.ts

@@ -26,7 +26,7 @@
     "eslint-config-prettier": "^8.3.0",
     "eslint-plugin-prettier": "^4.0.0",
     "eslint-plugin-react": "^7.28.0",
-    "prettier": "2.5.1",
+    "prettier": "2.7.1",
     "prettier-plugin-tailwindcss": "^0.1.3"
   }
 }