Move prompt files to CLI directory (#1172)

* move prompt files to cli

* move to mts files

* log prompt files

* update gitignore, don't destroy

* parse/update

* fix typing errors in system scripts

* updated test:system

* parsing system files first

* updated note

* clean duplicates

* removed builtin prefix

* use resolvedTools info

docs/src/content/docs/reference/scripts/system.mdx

@@ -1090,7 +1090,7 @@ defTool(
         if (frontmatter) {
             const files = []
             for (const { filename } of res) {
-                const file = {
+                const file: WorkspaceFile & { frontmatter?: string } = {
                     filename,
                 }
                 files.push(file)
@@ -2108,7 +2108,11 @@ defTool(
         if (!matches?.length) return "No files found."
         const q = await host.promiseQueue(5)
         const files = await q.mapAll(matches, async ({ filename, content }) => {
-            const file = {
+            const file: WorkspaceFile & {
+                title?: string
+                description?: string
+                summary?: string
+            } = {
                 filename,
             }
             try {

package.json

@@ -48,7 +48,7 @@
         "retrieval:search": "node packages/cli/built/genaiscript.cjs retrieval search lorem \"packages/sample/src/rag/*\"",
         "retrieval:codequery": "node packages/cli/built/genaiscript.cjs code query packages/core/src/progress.ts \"(interface_declaration) @i\"",
         "test:tokens": "node packages/cli/built/genaiscript.cjs retrieval tokens packages/sample/src/rag/*",
-        "test:system": "cd packages/core && node ../cli/built/genaiscript.cjs scripts compile",
+        "test:system": "cd packages/cli && node ./built/genaiscript.cjs scripts compile",
         "test:compile": "node packages/cli/built/genaiscript.cjs scripts compile",
         "test:fix": "node packages/cli/built/genaiscript.cjs scripts fix",
         "test:infomodel": "node packages/cli/built/genaiscript.cjs scripts model",

packages/core/src/genaisrc/system.fs_find_files.genai.js → packages/cli/genaisrc/system.fs_find_files.genai.mts

@@ -60,7 +60,7 @@ defTool(
         if (frontmatter) {
             const files = []
             for (const { filename } of res) {
-                const file = {
+                const file: WorkspaceFile & { frontmatter?: string } = {
                     filename,
                 }
                 files.push(file)

packages/core/src/genaisrc/system.md_find_files.genai.js → packages/cli/genaisrc/system.md_find_files.genai.mts

@@ -36,7 +36,11 @@ defTool(
         if (!matches?.length) return "No files found."
         const q = await host.promiseQueue(5)
         const files = await q.mapAll(matches, async ({ filename, content }) => {
-            const file = {
+            const file: WorkspaceFile & {
+                title?: string
+                description?: string
+                summary?: string
+            } = {
                 filename,
             }
             try {

packages/cli/package.json

@@ -25,7 +25,8 @@
     "built/markdown.css",
     "built/codicon.css",
     "built/codicon.ttf",
-    "built/favicon.svg"
+    "built/favicon.svg",
+    "genaisrc/*.genai.mts"
   ],
   "publisher": "Microsoft",
   "repository": {

packages/core/bundleprompts.js

@@ -2,7 +2,7 @@ const { readdirSync, readFileSync, writeFileSync } = require("fs")
 const { parse } = require("json5")
 
 async function main() {
-    const dir = "./src/genaisrc"
+    const dir = "../cli/genaisrc"
     const fp = "./src/default_prompts.ts"
     const fmp = "../../docs/src/content/docs/reference/scripts/system.mdx"
     const fnp = "../../docs/src/components/BuiltinTools.mdx"
@@ -11,10 +11,10 @@ async function main() {
     const promptMap = {}
     const prompts = readdirSync(dir)
     for (const prompt of prompts) {
-        if (!/\.genai\.js$/.test(prompt)) continue
+        if (!/\.genai\.m?ts$/.test(prompt)) continue
         const text = readFileSync(`${dir}/${prompt}`, "utf-8")
         if (/^system\./.test(prompt)) {
-            const id = prompt.replace(/\.genai\.m?js$/i, "")
+            const id = prompt.replace(/\.genai\.m?ts$/i, "")
             if (promptMap[id]) throw new Error(`duplicate prompt ${id}`)
             promptMap[id] = text
         }
@@ -105,12 +105,6 @@ async function main() {
     console.log(`found ${functions.length} tools`)
 
     const text = `// autogenerated - node bundleprompts.mjs
-export const defaultPrompts = Object.freeze<Record<string, string>>(${JSON.stringify(
-        promptMap,
-        null,
-        4
-    )});
-
 export const promptDefinitions = Object.freeze<Record<string, string>>(${JSON.stringify(
         promptDefs,
         null,

packages/core/src/constants.ts

@@ -71,7 +71,6 @@ export const TRANSCRIPTION_MODEL_ID = "transcription"
 export const SPEECH_MODEL_ID = "speech"
 export const DEFAULT_FENCE_FORMAT: FenceFormat = "xml"
 export const DEFAULT_TEMPERATURE = 0.8
-export const BUILTIN_PREFIX = "_builtin/"
 export const CACHE_LLMREQUEST_PREFIX = "genaiscript/cache/llm/"
 export const CACHE_AIREQUEST_TRACE_PREFIX = "genaiscript/cache/ai/trace/"
 export const CACHE_AIREQUEST_TEXT_PREFIX = "genaiscript/cache/ai/text/"

packages/core/src/parser.ts

@@ -1,12 +1,15 @@
 // Importing utility functions and constants from other files
-import { logVerbose, strcmp } from "./util" // String comparison function
-import { defaultPrompts } from "./default_prompts" // Default prompt data
+import { logVerbose, logWarn, strcmp } from "./util" // String comparison function
 import { parsePromptScript } from "./template" // Function to parse scripts
 import { readText } from "./fs" // Function to read text from a file
-import { BUILTIN_PREFIX } from "./constants" // Constants for MIME types and prefixes
+import { GENAI_ANYTS_REGEX } from "./constants" // Constants for MIME types and prefixes
 import { Project } from "./server/messages"
 import { resolveSystems } from "./systems"
 import { resolveScriptParametersSchema } from "./vars"
+import { dirname, join, resolve } from "node:path"
+import { fileURLToPath } from "node:url"
+import { readdir } from "node:fs/promises"
+import { uniq } from "es-toolkit"
 
 /**
  * Converts a string to a character position represented as [row, column].
@@ -30,25 +33,44 @@ export async function parseProject(options: { scriptFiles: string[] }) {
     const prj: Project = {
         scripts: [],
         diagnostics: [],
-    } // Initialize a new project instance
-
-    // Clone the default prompts
-    const deflPr: Record<string, string> = Object.assign({}, defaultPrompts)
-
+    }
+    const genaisrcDir = resolve(
+        join(
+            dirname(dirname(__filename ?? fileURLToPath(import.meta.url))),
+            "genaisrc"
+        )
+    ) // ignore esbuild warning
+    const systemPrompts = await (
+        await readdir(genaisrcDir)
+    ).filter((f) => GENAI_ANYTS_REGEX.test(f))
     // Process each script file, parsing its content and updating the project
-    for (const f of scriptFiles) {
-        const tmpl = await parsePromptScript(f, await readText(f), prj)
+    const scripts: Record<string, PromptScript> = {}
+    for (const fn of systemPrompts) {
+        const f = join(genaisrcDir, fn)
+        const tmpl = await parsePromptScript(f, await readText(f))
         if (!tmpl) {
-            logVerbose(`skipping invalid script file: ${f}`)
+            logWarn(`skipping invalid system script: ${fn}`)
             continue
         } // Skip if no template is parsed
-        delete deflPr[tmpl.id] // Remove the parsed template from defaults
         prj.scripts.push(tmpl) // Add to project templates
+        scripts[tmpl.id] = tmpl
     }
 
-    // Add remaining default prompts to the project
-    for (const [id, v] of Object.entries(deflPr)) {
-        prj.scripts.push(await parsePromptScript(BUILTIN_PREFIX + id, v, prj))
+    for (const f of uniq(scriptFiles).filter(
+        (f) => resolve(dirname(f)) !== genaisrcDir
+    )) {
+        const tmpl = await parsePromptScript(f, await readText(f))
+        if (!tmpl) {
+            logWarn(`skipping invalid script ${f}`)
+            continue
+        } // Skip if no template is parsed
+        if (scripts[tmpl.id]) {
+            logWarn(`duplicate script '${tmpl.id}' (${f})`)
+            logVerbose(`  already defined in ${scripts[tmpl.id].filename}`)
+            continue
+        }
+        prj.scripts.push(tmpl) // Add t
+        scripts[tmpl.id] = tmpl
     }
 
     /**

packages/core/src/scripts.ts

@@ -33,14 +33,13 @@ export async function fixPromptDefinitions(project: Project) {
         const { dirname, ts, js } = folder
         {
             const fn = host.path.join(dirname, ".gitignore")
-            const current = await tryReadText(fn)
-            const content = dedent`# auto-generated
-            genaiscript.d.ts
+            const current = (await tryReadText(fn)) || ""
+            const content = dedent`genaiscript.d.ts
             tsconfig.json
             jsconfig.json`
-            if (current !== content) {
+            if (!current.includes(content)) {
                 logVerbose(`updating ${fn}`)
-                await writeText(fn, content)
+                await writeText(fn, current + "\n#GenAIScript\n" + content)
             }
         }
         for (let [defName, defContent] of Object.entries(promptDefinitions)) {

packages/core/src/systems.ts

@@ -101,7 +101,7 @@ export function resolveSystems(
     if (responseSchema && !responseType) systems.push("system.output_json")
 
     // Include tools-related systems if specified in the script
-    if (tools.length) {
+    if (tools.length || resolvedTools?.length) {
         systems.push("system.tools")
         // Resolve and add each tool's systems based on its definition in the project
         tools.forEach((tool) =>

packages/core/src/template.ts

@@ -4,13 +4,10 @@
  * data types and formats.
  */
 
-import { Project } from "./server/messages"
-import { BUILTIN_PREFIX, GENAI_ANY_REGEX, PROMPTY_REGEX } from "./constants"
-import { errorMessage } from "./error"
+import { GENAI_ANY_REGEX, PROMPTY_REGEX } from "./constants"
 import { host } from "./host"
 import { JSON5TryParse } from "./json5"
 import { humanize } from "inflection"
-import { validateSchema } from "./schema"
 import { promptyParse, promptyToGenAIScript } from "./prompty"
 
 /**
@@ -76,20 +73,15 @@ function parsePromptScriptTools(jsSource: string) {
  * @param finalizer - Finalizer function to perform additional validation.
  * @returns The parsed PromptScript or undefined in case of errors.
  */
-async function parsePromptTemplateCore(
-    filename: string,
-    content: string,
-    prj: Project
-) {
+async function parsePromptTemplateCore(filename: string, content: string) {
     const r = {
         id: templateIdFromFileName(filename),
         title: humanize(
             host.path.basename(filename).replace(GENAI_ANY_REGEX, "")
         ),
         jsSource: content,
     } as PromptScript
-    if (!filename.startsWith(BUILTIN_PREFIX))
-        r.filename = host.path.resolve(filename)
+    r.filename = host.path.resolve(filename)
     const meta = parsePromptScriptMeta(r.jsSource)
     Object.assign(r, meta)
     return r
@@ -103,19 +95,15 @@ async function parsePromptTemplateCore(
  * @param prj - The Project instance containing diagnostics.
  * @returns The parsed PromptScript or undefined in case of errors.
  */
-export async function parsePromptScript(
-    filename: string,
-    content: string,
-    prj: Project
-) {
+export async function parsePromptScript(filename: string, content: string) {
     let text: string = undefined
     if (PROMPTY_REGEX.test(filename)) {
         text = content
         const doc = await promptyParse(filename, content)
         content = await promptyToGenAIScript(doc)
     }
 
-    const script = await parsePromptTemplateCore(filename, content, prj)
+    const script = await parsePromptTemplateCore(filename, content)
     if (text) script.text = text
     return script
 }

packages/vscode/src/markdowndocumentprovider.ts

@@ -13,12 +13,8 @@ import {
     TRACE_NODE_PREFIX,
     CACHE_LLMREQUEST_PREFIX,
     CACHE_AIREQUEST_TRACE_PREFIX,
-    BUILTIN_PREFIX,
-    GENAI_ANY_REGEX,
     CACHE_AIREQUEST_TEXT_PREFIX,
-    GENAI_MJS_EXT,
 } from "../../core/src/constants"
-import { defaultPrompts } from "../../core/src/default_prompts"
 import { extractFenced, renderFencedVariables } from "../../core/src/fence"
 import { prettifyMarkdown } from "../../core/src/markdown"
 import {
@@ -131,12 +127,6 @@ ${prettifyMarkdown(md)}
                 .replace(/\.md$/, "")
             return this.previewAIRequest(sha, "text")
         }
-        if (uri.path.startsWith(BUILTIN_PREFIX)) {
-            const id = uri.path
-                .slice(BUILTIN_PREFIX.length)
-                .replace(GENAI_ANY_REGEX, "")
-            return defaultPrompts[id] ?? `No such builtin prompt: ${id}`
-        }
         return ""
     }
 
@@ -207,13 +197,6 @@ export function infoUri(path: string) {
     return vscode.Uri.from({ scheme: SCHEME, path })
 }
 
-export function builtinPromptUri(id: string) {
-    return vscode.Uri.from({
-        scheme: SCHEME,
-        path: BUILTIN_PREFIX + id + GENAI_MJS_EXT,
-    })
-}
-
 export function activateMarkdownTextDocumentContentProvider(
     state: ExtensionState
 ) {

packages/vscode/src/promptcommands.ts

@@ -1,6 +1,5 @@
 import * as vscode from "vscode"
 import { ExtensionState } from "./state"
-import { builtinPromptUri } from "./markdowndocumentprovider"
 import { templatesToQuickPickItems } from "./fragmentcommands"
 import { registerCommand } from "./commands"
 import { createScript } from "../../core/src/scripts"
@@ -67,9 +66,7 @@ export function activatePromptCommands(state: ExtensionState) {
         registerCommand(
             "genaiscript.prompt.navigate",
             async (prompt: PromptScript) => {
-                const uri = prompt.filename
-                    ? host.toUri(prompt.filename)
-                    : builtinPromptUri(prompt.id)
+                const uri = host.toUri(prompt.filename)
                 await vscode.window.showTextDocument(uri)
             }
         )