Merge pull request #237 from TorstenDittmann/229-feature-request-custom-fencecodeblock-highlighting-function-in-preprocessor-config

feat: add highlighter function

Author: TorstenDittmann

.github/workflows/test.yml

@@ -18,5 +18,4 @@ jobs:
             - run: corepack enable
             - run: pnpm install --frozen-lockfile
             - run: pnpm run build
-            - run: pnpx playwright install --with-deps chromium
             - run: pnpm test

apps/demo/package.json

@@ -6,7 +6,7 @@
         "dev": "vite dev",
         "build": "vite build",
         "preview": "vite preview",
-        "test": "playwright test",
+        "test": "playwright install --with-deps && playwright test",
         "check": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json",
         "check:watch": "svelte-kit sync && svelte-check --tsconfig ./tsconfig.json --watch",
         "lint": "prettier --check . && eslint .",

packages/process/package.json

@@ -1,6 +1,6 @@
 {
     "name": "svelte-markdoc-preprocess",
-    "version": "2.0.2",
+    "version": "2.1.0",
     "description": "A Svelte preprocessor that allows you to use Markdoc.",
     "type": "commonjs",
     "keywords": [

packages/process/src/config.ts

@@ -40,4 +40,8 @@ export type Config = {
      * Configuration for the markdoc compiler.
      */
     config: ConfigType | null;
+    /**
+     * A function that will provide a custom highlighter for code blocks.
+     */
+    highlighter: ((code: string, language: string) => Promise<string>) | null;
 };

packages/process/src/processor.ts

@@ -12,6 +12,7 @@ const default_config: Config = {
     config: null,
     validationThreshold: 'error',
     allowComments: false,
+    highlighter: null,
 };
 
 const processor = ({
@@ -24,6 +25,7 @@ const processor = ({
     config = default_config.config,
     validationThreshold = default_config.validationThreshold,
     allowComments = default_config.allowComments,
+    highlighter: highlighter = default_config.highlighter,
 }: Partial<Config> = default_config): PreprocessorGroup => {
     return {
         name: 'svelte-markdoc-preprocess',
@@ -40,7 +42,7 @@ const processor = ({
             /**
              * Add svelte components to be used with markdoc tags
              */
-            const code = transformer({
+            const code = await transformer({
                 filename,
                 config,
                 content,
@@ -51,6 +53,7 @@ const processor = ({
                 partials_dir: partials,
                 validation_threshold: validationThreshold,
                 allow_comments: allowComments,
+                highlighter,
             });
 
             return {

packages/process/src/renderer.ts

@@ -3,23 +3,40 @@ import { sanitize_for_svelte } from './transformer';
 import { escape } from 'html-escaper';
 import { IMAGE_PREFIX, IMPORT_PREFIX, NODES_IMPORT } from './constants';
 import { is_relative_path } from './utils';
+import { Config } from './config';
 
-export function render_html(
+export async function render_html(
     node: RenderableTreeNodes,
     dependencies: Map<string, string>,
-): string {
+    highlighter: Config['highlighter'],
+    escape_html = true,
+): Promise<string> {
     /**
      * if the node is a string or number, it's a text node.
      */
     if (typeof node === 'string' || typeof node === 'number') {
-        return sanitize_for_svelte(escape(String(node)));
+        if (escape_html) {
+            return sanitize_for_svelte(escape(String(node)));
+        } else {
+            return sanitize_for_svelte(String(node));
+        }
     }
 
     /**
      * if the node is an array, render its items.
      */
     if (Array.isArray(node)) {
-        return node.map((item) => render_html(item, dependencies)).join('');
+        return Promise.all(
+            node.map(
+                async (item) =>
+                    await render_html(
+                        item,
+                        dependencies,
+                        highlighter,
+                        escape_html,
+                    ),
+            ),
+        ).then((items) => items.join(''));
     }
 
     /**
@@ -29,10 +46,10 @@ export function render_html(
         return '';
     }
 
-    const { name, attributes, children = [] } = node;
+    let { name, attributes, children = [] } = node;
 
     if (!name) {
-        return render_html(children, dependencies);
+        return await render_html(children, dependencies, highlighter);
     }
 
     const is_svelte = is_svelte_component(node);
@@ -41,9 +58,10 @@ export function render_html(
      * add attributes to the tag.
      */
     let output = `<${name}`;
-    for (const [key, value] of Object.entries(attributes ?? {})) {
+    for (let [key, value] of Object.entries(attributes ?? {})) {
         const is_src_key = key === 'src';
         const is_imported_image = is_src_key && is_relative_path(value);
+
         if (is_svelte) {
             switch (name.toLowerCase()) {
                 case `${NODES_IMPORT}.image`.toLowerCase():
@@ -97,11 +115,39 @@ export function render_html(
         return output;
     }
 
+    let escape_next = true;
+
+    if (highlighter) {
+        const run_highlighter =
+            name.toLowerCase() === `${NODES_IMPORT}.fence`.toLowerCase() ||
+            name.toLowerCase() === 'pre'.toLowerCase();
+        if (run_highlighter) {
+            escape_next = false;
+            children = await Promise.all(
+                children.map(async (child) =>
+                    typeof child === 'string'
+                        ? await highlighter(
+                              child,
+                              (is_svelte
+                                  ? attributes?.language
+                                  : attributes['data-language']) ?? '',
+                          )
+                        : child,
+                ),
+            );
+        }
+    }
+
     /**
      * render the children if present.
      */
     if (children.length) {
-        output += render_html(children, dependencies);
+        output += await render_html(
+            children,
+            dependencies,
+            highlighter,
+            escape_next,
+        );
     }
 
     /**

packages/process/src/transformer.ts

@@ -39,7 +39,7 @@ type Var = {
     type: StringConstructor | NumberConstructor | BooleanConstructor;
 };
 
-export function transformer({
+export async function transformer({
     content,
     filename,
     nodes_file,
@@ -50,6 +50,7 @@ export function transformer({
     config,
     validation_threshold,
     allow_comments,
+    highlighter,
 }: {
     content: string;
     filename: string;
@@ -61,7 +62,8 @@ export function transformer({
     config: Config['config'];
     validation_threshold: Config['validationThreshold'];
     allow_comments: Config['allowComments'];
-}): string {
+    highlighter: Config['highlighter'];
+}): Promise<string> {
     /**
      * create tokenizer
      */
@@ -189,7 +191,7 @@ export function transformer({
     /**
      * render to html
      */
-    const code = render_html(nast, dependencies);
+    const code = await render_html(nast, dependencies, highlighter);
 
     let transformed = '';
 
@@ -406,14 +408,9 @@ function prepare_nodes(
     if (nodes_file) {
         for (const [name] of each_exported_var(nodes_file)) {
             const type = name.toLowerCase() as NodeType;
-            if (type === 'image') {
-            }
             nodes[name.toLowerCase()] = {
                 ...get_node_defaults(type),
                 transform(node, config) {
-                    if (type === 'image') {
-                        node.attributes.src;
-                    }
                     return new Tag(
                         `${NODES_IMPORT}.${name}`,
                         node.transformAttributes(config),
@@ -470,7 +467,8 @@ function each_exported_var(filepath: string): Array<[string, string]> {
             if (node.type === 'ExportSpecifier') {
                 if (
                     parent?.type === 'ExportNamedDeclaration' &&
-                    parent?.source
+                    parent?.source &&
+                    node.exported.type === 'Identifier'
                 ) {
                     tup.push([node.exported.name, String(parent.source.value)]);
                 }

packages/process/tests/nodes/module.svelte

@@ -1,4 +1,5 @@
 <script context="module">
     export { default as Heading } from './mock.svelte';
     export { default as Image } from './mock.svelte';
+    export { default as Fence } from './mock.svelte';
 </script>

packages/process/tests/processor.test.mjs

@@ -69,7 +69,10 @@ test('preprocessor', async (context) => {
         files.map(async (entry) => {
             return context.test('tests ' + basename(entry), async () => {
                 const before = read_file(join(entry, 'source.markdoc'));
-                const after = read_file(join(entry, 'compiled.txt'));
+                const after = read_file(join(entry, 'compiled.txt')).replace(
+                    /\\n/g,
+                    '\n',
+                );
                 const config = await import('../' + join(entry, 'config.mjs'));
                 const preprocess = config.default;
                 const exception = config.exception ?? false;

packages/process/tests/processor/highlighter - fence node/compiled.txt

@@ -0,0 +1 @@
+<script>import * as INTERNAL__NODES from 'tests/nodes/module.svelte';</script><article><INTERNAL__NODES.Fence content="console.log(&#39;hello world&#39;);\n" language="javascript" process={true}>javascript:console.log('hello world');\n</INTERNAL__NODES.Fence><INTERNAL__NODES.Fence content="&lt;b&gt;bold&lt;/b&gt;\n" language="html" process={true}>html:<b>bold</b>\n</INTERNAL__NODES.Fence></article>

packages/process/tests/processor/highlighter - fence node/config.mjs

@@ -0,0 +1,7 @@
+import { markdoc } from '../../../dist/module.js';
+import { absoulute } from '../../utils.mjs';
+
+export default markdoc({
+    nodes: absoulute(import.meta.url, '../../nodes/module.svelte'),
+    highlighter: (code, lang) => `${lang}:${code}`,
+});

packages/process/tests/processor/highlighter - fence node/source.markdoc

@@ -0,0 +1,7 @@
+```javascript
+console.log('hello world');
+```
+
+```html
+<b>bold</b>
+```

packages/process/tests/processor/highlighter/compiled.txt

@@ -0,0 +1 @@
+<article><pre data-language="javascript">javascript:console.log('hello world');\n</pre><pre data-language="html">html:<b>bold</b>\n</pre></article>

packages/process/tests/processor/highlighter/config.mjs

@@ -0,0 +1,5 @@
+import { markdoc } from '../../../dist/module.js';
+
+export default markdoc({
+    highlighter: (code, lang) => `${lang}:${code}`,
+});

packages/process/tests/processor/highlighter/source.markdoc

@@ -0,0 +1,7 @@
+```javascript
+console.log('hello world');
+```
+
+```html
+<b>bold</b>
+```