parsing system files first

packages/core/bundleprompts.js

@@ -105,12 +105,6 @@ async function main() {
     console.log(`found ${functions.length} tools`)
 
     const text = `// autogenerated - node bundleprompts.mjs
-export const defaultPrompts = Object.freeze<Record<string, string>>(${JSON.stringify(
-        promptMap,
-        null,
-        4
-    )});
-
 export const promptDefinitions = Object.freeze<Record<string, string>>(${JSON.stringify(
         promptDefs,
         null,

packages/core/src/ast.ts

@@ -1,6 +1,7 @@
 /// <reference path="./types/prompt_template.d.ts" />
 // Import necessary regular expressions for file type detection and host utilities
 import {
+    BUILTIN_SCRIPT_PREFIX,
     GENAI_ANYJS_REGEX,
     GENAI_ANYTS_REGEX,
     PROMPTY_REGEX,
@@ -69,6 +70,7 @@ export function collectFolders(prj: Project) {
         (t) => t.filename && !PROMPTY_REGEX.test(t.filename)
     )) {
         const dirname = host.path.dirname(t.filename) // Get directory name from the filename
+        if ((dirname + "/").startsWith(BUILTIN_SCRIPT_PREFIX)) continue
         const folder = folders[dirname] || (folders[dirname] = { dirname })
         folder.js = folder.js || GENAI_ANYJS_REGEX.test(t.filename) // Check for presence of JS files
         folder.ts = folder.ts || GENAI_ANYTS_REGEX.test(t.filename) // Check for presence of TS files

packages/core/src/constants.ts

@@ -71,7 +71,7 @@ export const TRANSCRIPTION_MODEL_ID = "transcription"
 export const SPEECH_MODEL_ID = "speech"
 export const DEFAULT_FENCE_FORMAT: FenceFormat = "xml"
 export const DEFAULT_TEMPERATURE = 0.8
-export const BUILTIN_PREFIX = "_builtin/"
+export const BUILTIN_SCRIPT_PREFIX = ".genaiscript/__system/"
 export const CACHE_LLMREQUEST_PREFIX = "genaiscript/cache/llm/"
 export const CACHE_AIREQUEST_TRACE_PREFIX = "genaiscript/cache/ai/trace/"
 export const CACHE_AIREQUEST_TEXT_PREFIX = "genaiscript/cache/ai/text/"

packages/core/src/parser.ts

@@ -1,14 +1,15 @@
 // Importing utility functions and constants from other files
-import { logWarn, strcmp } from "./util" // String comparison function
+import { logVerbose, logWarn, strcmp } from "./util" // String comparison function
 import { parsePromptScript } from "./template" // Function to parse scripts
 import { readText } from "./fs" // Function to read text from a file
-import { BUILTIN_PREFIX, GENAI_ANYTS_REGEX } from "./constants" // Constants for MIME types and prefixes
+import { BUILTIN_SCRIPT_PREFIX, GENAI_ANYTS_REGEX } from "./constants" // Constants for MIME types and prefixes
 import { Project } from "./server/messages"
 import { resolveSystems } from "./systems"
 import { resolveScriptParametersSchema } from "./vars"
-import { dirname, join } from "node:path"
+import { dirname, join, resolve } from "node:path"
 import { fileURLToPath } from "node:url"
 import { readdir } from "node:fs/promises"
+import { uniq } from "es-toolkit"
 
 /**
  * Converts a string to a character position represented as [row, column].
@@ -33,38 +34,46 @@ export async function parseProject(options: { scriptFiles: string[] }) {
         scripts: [],
         diagnostics: [],
     }
-    const genaisrcDir = join(
-        dirname(dirname(__filename ?? fileURLToPath(import.meta.url))),
-        "genaisrc"
+    const genaisrcDir = resolve(
+        join(
+            dirname(dirname(__filename ?? fileURLToPath(import.meta.url))),
+            "genaisrc"
+        )
     ) // ignore esbuild warning
     const systemPrompts = await (
         await readdir(genaisrcDir)
     ).filter((f) => GENAI_ANYTS_REGEX.test(f))
     // Process each script file, parsing its content and updating the project
-    const ids = new Set<string>()
-    for (const f of scriptFiles) {
-        const tmpl = await parsePromptScript(f, await readText(f))
+    const scripts: Record<string, PromptScript> = {}
+    for (const fn of systemPrompts) {
+        const f = join(genaisrcDir, fn)
+        const tmpl = await parsePromptScript(
+            BUILTIN_SCRIPT_PREFIX + fn,
+            await readText(f)
+        )
         if (!tmpl) {
-            logWarn(`skipping invalid script ${f}`)
+            logWarn(`skipping invalid system scruipt: ${fn}`)
             continue
         } // Skip if no template is parsed
         prj.scripts.push(tmpl) // Add to project templates
-        ids.add(tmpl.id)
+        scripts[tmpl.id] = tmpl
     }
-    for (const f of systemPrompts) {
-        const tmpl = await parsePromptScript(
-            join(genaisrcDir, f),
-            await readText(f)
-        )
+
+    for (const f of uniq(scriptFiles).filter(
+        (f) => resolve(dirname(f)) !== genaisrcDir
+    )) {
+        const tmpl = await parsePromptScript(f, await readText(f))
         if (!tmpl) {
-            logWarn(`skipping invalid system scruipt: ${f}`)
+            logWarn(`skipping invalid script ${f}`)
             continue
         } // Skip if no template is parsed
-        if (!ids.has(tmpl.id)) {
-            tmpl.filename = BUILTIN_PREFIX + f
-            prj.scripts.push(tmpl) // Add to project templates
-            ids.add(tmpl.id)
+        if (scripts[tmpl.id]) {
+            logWarn(`duplicate script ${tmpl.id} (${f})`)
+            logVerbose(`  already defined in ${scripts[tmpl.id]}`)
+            continue
         }
+        prj.scripts.push(tmpl) // Add t
+        scripts[tmpl.id] = tmpl
     }
 
     /**

packages/core/src/template.ts

@@ -4,7 +4,7 @@
  * data types and formats.
  */
 
-import { BUILTIN_PREFIX, GENAI_ANY_REGEX, PROMPTY_REGEX } from "./constants"
+import { BUILTIN_SCRIPT_PREFIX, GENAI_ANY_REGEX, PROMPTY_REGEX } from "./constants"
 import { host } from "./host"
 import { JSON5TryParse } from "./json5"
 import { humanize } from "inflection"
@@ -83,12 +83,12 @@ async function parsePromptTemplateCore(filename: string, content: string) {
         ),
         jsSource: content,
     } as PromptScript
-    if (filename.startsWith(BUILTIN_PREFIX)) {
+    if (filename.startsWith(BUILTIN_SCRIPT_PREFIX)) {
         const genaisrcDir = join(
             dirname(dirname(__filename ?? fileURLToPath(import.meta.url))),
             "genaisrc"
         ) // ignore esbuild warning
-        const id = filename.slice(BUILTIN_PREFIX.length)
+        const id = filename.slice(BUILTIN_SCRIPT_PREFIX.length)
         r.filename = host.path.join(genaisrcDir, id)
         console.log(r)
     } else {

packages/vscode/src/markdowndocumentprovider.ts

@@ -13,12 +13,8 @@ import {
     TRACE_NODE_PREFIX,
     CACHE_LLMREQUEST_PREFIX,
     CACHE_AIREQUEST_TRACE_PREFIX,
-    BUILTIN_PREFIX,
-    GENAI_ANY_REGEX,
     CACHE_AIREQUEST_TEXT_PREFIX,
-    GENAI_MJS_EXT,
 } from "../../core/src/constants"
-import { defaultPrompts } from "../../core/src/default_prompts"
 import { extractFenced, renderFencedVariables } from "../../core/src/fence"
 import { prettifyMarkdown } from "../../core/src/markdown"
 import {
@@ -131,12 +127,6 @@ ${prettifyMarkdown(md)}
                 .replace(/\.md$/, "")
             return this.previewAIRequest(sha, "text")
         }
-        if (uri.path.startsWith(BUILTIN_PREFIX)) {
-            const id = uri.path
-                .slice(BUILTIN_PREFIX.length)
-                .replace(GENAI_ANY_REGEX, "")
-            return defaultPrompts[id] ?? `No such builtin prompt: ${id}`
-        }
         return ""
     }
 
@@ -207,13 +197,6 @@ export function infoUri(path: string) {
     return vscode.Uri.from({ scheme: SCHEME, path })
 }
 
-export function builtinPromptUri(id: string) {
-    return vscode.Uri.from({
-        scheme: SCHEME,
-        path: BUILTIN_PREFIX + id + GENAI_MJS_EXT,
-    })
-}
-
 export function activateMarkdownTextDocumentContentProvider(
     state: ExtensionState
 ) {

packages/vscode/src/promptcommands.ts

@@ -1,6 +1,5 @@
 import * as vscode from "vscode"
 import { ExtensionState } from "./state"
-import { builtinPromptUri } from "./markdowndocumentprovider"
 import { templatesToQuickPickItems } from "./fragmentcommands"
 import { registerCommand } from "./commands"
 import { createScript } from "../../core/src/scripts"
@@ -67,9 +66,7 @@ export function activatePromptCommands(state: ExtensionState) {
         registerCommand(
             "genaiscript.prompt.navigate",
             async (prompt: PromptScript) => {
-                const uri = prompt.filename
-                    ? host.toUri(prompt.filename)
-                    : builtinPromptUri(prompt.id)
+                const uri = host.toUri(prompt.filename)
                 await vscode.window.showTextDocument(uri)
             }
         )