Move prompt files to CLI directory

docs/src/content/docs/reference/scripts/system.mdx

@@ -1089,8 +1089,8 @@
 
         if (frontmatter) {
             const files = []
             for (const { filename } of res) {
-                const file = {
+                const file: WorkspaceFile & { frontmatter?: string } = {
                     filename,
                 }
                 files.push(file)
@@ -2107,8 +2107,12 @@
               })
         if (!matches?.length) return "No files found."
         const q = await host.promiseQueue(5)
         const files = await q.mapAll(matches, async ({ filename, content }) => {
-            const file = {
+            const file: WorkspaceFile & {
+                title?: string
+                description?: string
+                summary?: string
+            } = {
                 filename,
             }
             try {

package.json

@@ -48,7 +48,7 @@
         "retrieval:search": "node packages/cli/built/genaiscript.cjs retrieval search lorem \"packages/sample/src/rag/*\"",
         "retrieval:codequery": "node packages/cli/built/genaiscript.cjs code query packages/core/src/progress.ts \"(interface_declaration) @i\"",
         "test:tokens": "node packages/cli/built/genaiscript.cjs retrieval tokens packages/sample/src/rag/*",
-        "test:system": "cd packages/core && node ../cli/built/genaiscript.cjs scripts compile",
+        "test:system": "cd packages/cli && node ./built/genaiscript.cjs scripts compile",
         "test:compile": "node packages/cli/built/genaiscript.cjs scripts compile",
         "test:fix": "node packages/cli/built/genaiscript.cjs scripts fix",
         "test:infomodel": "node packages/cli/built/genaiscript.cjs scripts model",

packages/core/src/genaisrc/system.fs_find_files.genai.js → packages/cli/genaisrc/system.fs_find_files.genai.mts

@@ -60,7 +60,7 @@ defTool(
         if (frontmatter) {
             const files = []
             for (const { filename } of res) {
-                const file = {
+                const file: WorkspaceFile & { frontmatter?: string } = {
                     filename,
                 }
                 files.push(file)

packages/core/src/genaisrc/system.md_find_files.genai.js → packages/cli/genaisrc/system.md_find_files.genai.mts

@@ -36,7 +36,11 @@ defTool(
         if (!matches?.length) return "No files found."
         const q = await host.promiseQueue(5)
         const files = await q.mapAll(matches, async ({ filename, content }) => {
-            const file = {
+            const file: WorkspaceFile & {
+                title?: string
+                description?: string
+                summary?: string
+            } = {
                 filename,
             }
             try {

packages/cli/package.json

@@ -25,7 +25,8 @@
     "built/markdown.css",
     "built/codicon.css",
     "built/codicon.ttf",
-    "built/favicon.svg"
+    "built/favicon.svg",
+    "genaisrc/*.genai.mts"
   ],
   "publisher": "Microsoft",
   "repository": {

packages/core/bundleprompts.js

@@ -2,7 +2,7 @@ const { readdirSync, readFileSync, writeFileSync } = require("fs")
 const { parse } = require("json5")
 
 async function main() {
-    const dir = "./src/genaisrc"
+    const dir = "../cli/genaisrc"
     const fp = "./src/default_prompts.ts"
     const fmp = "../../docs/src/content/docs/reference/scripts/system.mdx"
     const fnp = "../../docs/src/components/BuiltinTools.mdx"
@@ -11,10 +11,10 @@ async function main() {
     const promptMap = {}
     const prompts = readdirSync(dir)
     for (const prompt of prompts) {
-        if (!/\.genai\.js$/.test(prompt)) continue
+        if (!/\.genai\.m?ts$/.test(prompt)) continue
         const text = readFileSync(`${dir}/${prompt}`, "utf-8")
         if (/^system\./.test(prompt)) {
-            const id = prompt.replace(/\.genai\.m?js$/i, "")
+            const id = prompt.replace(/\.genai\.m?ts$/i, "")
             if (promptMap[id]) throw new Error(`duplicate prompt ${id}`)
             promptMap[id] = text
         }

packages/core/src/parser.ts

@@ -1,12 +1,14 @@
 // Importing utility functions and constants from other files
-import { logVerbose, strcmp } from "./util" // String comparison function
-import { defaultPrompts } from "./default_prompts" // Default prompt data
+import { logWarn, strcmp } from "./util" // String comparison function
 import { parsePromptScript } from "./template" // Function to parse scripts
 import { readText } from "./fs" // Function to read text from a file
-import { BUILTIN_PREFIX } from "./constants" // Constants for MIME types and prefixes
+import { BUILTIN_PREFIX, GENAI_ANYTS_REGEX } from "./constants" // Constants for MIME types and prefixes
 import { Project } from "./server/messages"
 import { resolveSystems } from "./systems"
 import { resolveScriptParametersSchema } from "./vars"
+import { dirname, join } from "node:path"
+import { fileURLToPath } from "node:url"
+import { readdir } from "node:fs/promises"
 
 /**
  * Converts a string to a character position represented as [row, column].
@@ -30,25 +32,39 @@ export async function parseProject(options: { scriptFiles: string[] }) {
     const prj: Project = {
         scripts: [],
         diagnostics: [],
-    } // Initialize a new project instance
-
-    // Clone the default prompts
-    const deflPr: Record<string, string> = Object.assign({}, defaultPrompts)
-
+    }
+    const genaisrcDir = join(
+        dirname(dirname(__filename ?? fileURLToPath(import.meta.url))),
+        "genaisrc"
+    ) // ignore esbuild warning
+    const systemPrompts = await (
+        await readdir(genaisrcDir)
+    ).filter((f) => GENAI_ANYTS_REGEX.test(f))
     // Process each script file, parsing its content and updating the project
+    const ids = new Set<string>()
     for (const f of scriptFiles) {
-        const tmpl = await parsePromptScript(f, await readText(f), prj)
+        const tmpl = await parsePromptScript(f, await readText(f))
         if (!tmpl) {
-            logVerbose(`skipping invalid script file: ${f}`)
+            logWarn(`skipping invalid script ${f}`)
             continue
         } // Skip if no template is parsed
-        delete deflPr[tmpl.id] // Remove the parsed template from defaults
         prj.scripts.push(tmpl) // Add to project templates
+        ids.add(tmpl.id)
     }
-
-    // Add remaining default prompts to the project
-    for (const [id, v] of Object.entries(deflPr)) {
-        prj.scripts.push(await parsePromptScript(BUILTIN_PREFIX + id, v, prj))
+    for (const f of systemPrompts) {
+        const tmpl = await parsePromptScript(
+            join(genaisrcDir, f),
+            await readText(f)
+        )
+        if (!tmpl) {
+            logWarn(`skipping invalid system scruipt: ${f}`)
+            continue
+        } // Skip if no template is parsed
+        if (!ids.has(tmpl.id)) {
+            tmpl.filename = BUILTIN_PREFIX + f
+            prj.scripts.push(tmpl) // Add to project templates
+            ids.add(tmpl.id)
+        }
     }
 
     /**

packages/core/src/scripts.ts

@@ -33,14 +33,13 @@ export async function fixPromptDefinitions(project: Project) {
         const { dirname, ts, js } = folder
         {
             const fn = host.path.join(dirname, ".gitignore")
-            const current = await tryReadText(fn)
-            const content = dedent`# auto-generated
-            genaiscript.d.ts
+            const current = (await tryReadText(fn)) || ""
+            const content = dedent`genaiscript.d.ts
             tsconfig.json
             jsconfig.json`
-            if (current !== content) {
+            if (!current.includes(content)) {
                 logVerbose(`updating ${fn}`)
-                await writeText(fn, content)
+                await writeText(fn, current + "\n#GenAIScript\n" + content)
             }
         }
         for (let [defName, defContent] of Object.entries(promptDefinitions)) {

packages/core/src/template.ts

@@ -4,14 +4,13 @@
  * data types and formats.
  */
 
-import { Project } from "./server/messages"
 import { BUILTIN_PREFIX, GENAI_ANY_REGEX, PROMPTY_REGEX } from "./constants"
-import { errorMessage } from "./error"
 import { host } from "./host"
 import { JSON5TryParse } from "./json5"
 import { humanize } from "inflection"
-import { validateSchema } from "./schema"
 import { promptyParse, promptyToGenAIScript } from "./prompty"
+import { fileURLToPath } from "node:url"
+import { dirname, join } from "node:path"
 
 /**
  * Extracts a template ID from the given filename by removing specific extensions
@@ -76,20 +75,25 @@ function parsePromptScriptTools(jsSource: string) {
  * @param finalizer - Finalizer function to perform additional validation.
  * @returns The parsed PromptScript or undefined in case of errors.
  */
-async function parsePromptTemplateCore(
-    filename: string,
-    content: string,
-    prj: Project
-) {
+async function parsePromptTemplateCore(filename: string, content: string) {
     const r = {
         id: templateIdFromFileName(filename),
         title: humanize(
             host.path.basename(filename).replace(GENAI_ANY_REGEX, "")
         ),
         jsSource: content,
     } as PromptScript
-    if (!filename.startsWith(BUILTIN_PREFIX))
+    if (filename.startsWith(BUILTIN_PREFIX)) {
+        const genaisrcDir = join(
+            dirname(dirname(__filename ?? fileURLToPath(import.meta.url))),
+            "genaisrc"
+        ) // ignore esbuild warning
+        const id = filename.slice(BUILTIN_PREFIX.length)
+        r.filename = host.path.join(genaisrcDir, id)
+        console.log(r)
+    } else {
         r.filename = host.path.resolve(filename)
+    }
     const meta = parsePromptScriptMeta(r.jsSource)
     Object.assign(r, meta)
     return r
@@ -103,19 +107,15 @@ async function parsePromptTemplateCore(
  * @param prj - The Project instance containing diagnostics.
  * @returns The parsed PromptScript or undefined in case of errors.
  */
-export async function parsePromptScript(
-    filename: string,
-    content: string,
-    prj: Project
-) {
+export async function parsePromptScript(filename: string, content: string) {
     let text: string = undefined
     if (PROMPTY_REGEX.test(filename)) {
         text = content
         const doc = await promptyParse(filename, content)
         content = await promptyToGenAIScript(doc)
     }
 
-    const script = await parsePromptTemplateCore(filename, content, prj)
+    const script = await parsePromptTemplateCore(filename, content)
     if (text) script.text = text
     return script
 }