Add "echo" model for dry run debugging

docs/public/schemas/llms.json

@@ -23,6 +23,10 @@
                         "type": "string",
                         "description": "Description of the LLM provider"
                     },
+                    "hidden": {
+                        "type": "boolean",
+                        "description": "Indicates if the provider is hidden"
+                    },
                     "limitations": {
                         "type": "string",
                         "description": "General note about limitations"

docs/src/content/docs/getting-started/configuration.mdx

@@ -1588,6 +1588,17 @@ docker run -d -p 9000:9000 -e ASR_MODEL=base -e ASR_ENGINE=openai_whisper onerah
 
 You can also override the `transcription` model alias to change the default model used by `transcribe`.
 
+## Echo
+
+This is a dry run LLM provider that returns the messages without calling any LLM.
+It is most useful for debugging when you want to see the result LLM request without sending it.
+
+```js 'model: "echo"'
+script({
+    model: "echo",
+})
+```
+
 ## Model specific environment variables
 
 You can provide different environment variables

docs/src/content/docs/reference/cli/commands.md

@@ -22,7 +22,7 @@ Options:
                            "anthropic", "anthropic_bedrock", "google",
                            "huggingface", "mistral", "alibaba", "deepseek",
                            "transformers", "lmstudio", "jan", "llamafile",
-                           "litellm", "whisperasr")
+                           "litellm", "whisperasr", "echo")
   -h, --help               display help for command
 ```
 
@@ -34,7 +34,7 @@ Usage: genaiscript run [options] <script> [files...]
 Runs a GenAIScript against files.
 
 Options:
-  -p, --provider <string>                    Preferred LLM provider aliases (choices: "openai", "azure", "azure_serverless", "azure_serverless_models", "github", "ollama", "anthropic", "anthropic_bedrock", "google", "huggingface", "mistral", "alibaba", "deepseek", "transformers", "lmstudio", "jan", "llamafile", "litellm", "whisperasr")
+  -p, --provider <string>                    Preferred LLM provider aliases (choices: "openai", "azure", "azure_serverless", "azure_serverless_models", "github", "ollama", "anthropic", "anthropic_bedrock", "google", "huggingface", "mistral", "alibaba", "deepseek", "transformers", "lmstudio", "jan", "llamafile", "litellm", "whisperasr", "echo")
   -m, --model <string>                       'large' model alias (default)
   -sm, --small-model <string>                'small' alias model
   -vm, --vision-model <string>               'vision' alias model
@@ -118,7 +118,7 @@ Options:
                                       "huggingface", "mistral", "alibaba",
                                       "deepseek", "transformers", "lmstudio",
                                       "jan", "llamafile", "litellm",
-                                      "whisperasr")
+                                      "whisperasr", "echo")
   -m, --model <string>                'large' model alias (default)
   -sm, --small-model <string>         'small' alias model
   -vm, --vision-model <string>        'vision' alias model

docs/src/content/docs/reference/cli/run.mdx

@@ -54,7 +54,7 @@ npx genaiscript run <script> <files> --exclude-git-ignore
 
 ### --model ...
 
-Configure the default or `large` model alias
+Configure the default or `large` model alias. Use `echo` to do a dry run and return the messages instead of calling a LLM provider.
 
 ## --provider ...
 

packages/cli/src/cli.ts

@@ -111,9 +111,9 @@ export async function cli() {
                 "-p, --provider <string>",
                 "Preferred LLM provider aliases"
             ).choices(
-                MODEL_PROVIDERS.filter(
-                    ({ id }) => id !== MODEL_PROVIDER_GITHUB_COPILOT_CHAT
-                ).map(({ id }) => id)
+                MODEL_PROVIDERS.filter(({ hidden }) => !hidden).map(
+                    ({ id }) => id
+                )
             )
         )
         .action(configure)
@@ -554,9 +554,9 @@ export async function cli() {
                     "-p, --provider <string>",
                     "Preferred LLM provider aliases"
                 ).choices(
-                    MODEL_PROVIDERS.filter(
-                        ({ id }) => id !== MODEL_PROVIDER_GITHUB_COPILOT_CHAT
-                    ).map(({ id }) => id)
+                    MODEL_PROVIDERS.filter(({ hidden }) => !hidden).map(
+                        ({ id }) => id
+                    )
                 )
             )
             .option("-m, --model <string>", "'large' model alias (default)")

packages/cli/src/configure.ts

@@ -18,13 +18,13 @@ export async function configure(options: { provider?: string }) {
             ? MODEL_PROVIDERS.find(({ id }) => options.provider === id)
             : await select({
                   message: "Select a LLM provider to configure",
-                  choices: MODEL_PROVIDERS.filter(
-                      (p) => p.id !== MODEL_PROVIDER_GITHUB_COPILOT_CHAT
-                  ).map((provider) => ({
-                      name: provider.detail,
-                      value: provider,
-                      description: `'${provider.id}': https://microsoft.github.io/genaiscript/getting-started/configuration#${provider.id}`,
-                  })),
+                  choices: MODEL_PROVIDERS.filter(({ hidden }) => !hidden).map(
+                      (provider) => ({
+                          name: provider.detail,
+                          value: provider,
+                          description: `'${provider.id}': https://microsoft.github.io/genaiscript/getting-started/configuration#${provider.id}`,
+                      })
+                  ),
               })
         if (!provider) break
 

packages/core/src/chatrender.ts

@@ -6,6 +6,7 @@ import type {
     ChatCompletionToolMessageParam,
     ChatCompletionUserMessageParam,
 } from "./chattypes"
+import { collapseNewlines } from "./cleaners"
 
 // Import utility functions for JSON5 parsing, markdown formatting, and YAML stringification.
 import { JSONLLMTryParse } from "./json5"
@@ -194,7 +195,7 @@ export function renderMessagesToMarkdown(
             }
         })
     // Join the result array into a single markdown string.
-    return res.filter((s) => s !== undefined).join("\n")
+    return collapseNewlines(res.filter((s) => s !== undefined).join("\n"))
 }
 
 /**

packages/core/src/cleaners.ts

@@ -60,3 +60,10 @@ export function unmarkdown(text: string) {
         ?.replace(/\[([^\]]+)\]\([^)]+\)/g, (m, n) => n)
         ?.replace(/<\/?([^>]+)>/g, "")
 }
+
+/**
+ * Collapse 3+ lines to 1
+ */
+export function collapseNewlines(res: string): string {
+    return res?.replace(/(\r?\n){3,}/g, "\n\n")
+}

packages/core/src/config.ts

@@ -2,12 +2,7 @@ import { homedir } from "os"
 import { existsSync, readFileSync } from "fs"
 import { YAMLTryParse } from "./yaml"
 import { JSON5TryParse } from "./json5"
-import {
-    DOT_ENV_FILENAME,
-    MODEL_PROVIDER_GITHUB_COPILOT_CHAT,
-    MODEL_PROVIDERS,
-    TOOL_ID,
-} from "./constants"
+import { DOT_ENV_FILENAME, MODEL_PROVIDERS, TOOL_ID } from "./constants"
 import { resolve } from "path"
 import { validateJSONWithSchema } from "./schema"
 import { HostConfiguration } from "./hostconfiguration"
@@ -24,7 +19,6 @@ import { errorMessage } from "./error"
 import schema from "../../../docs/public/schemas/config.json"
 import defaultConfig from "./config.json"
 import { CancellationOptions } from "./cancellation"
-import { runtimeHost } from "./host"
 
 export async function resolveGlobalConfiguration(
     dotEnvPath?: string

packages/core/src/connection.ts

@@ -38,8 +38,9 @@ import {
     MODEL_PROVIDER_ANTHROPIC_BEDROCK,
     MODEL_PROVIDER_DEEPSEEK,
     DEEPSEEK_API_BASE,
-    MODEL_WHISPERASR_PROVIDER,
+    MODEL_PROVIDER_WHISPERASR,
     WHISPERASR_API_BASE,
+    MODEL_PROVIDER_ECHO,
 } from "./constants"
 import { host, runtimeHost } from "./host"
 import { parseModelIdentifier } from "./models"
@@ -431,7 +432,7 @@ export async function parseTokenFromEnv(
         }
     }
 
-    if (provider === MODEL_WHISPERASR_PROVIDER) {
+    if (provider === MODEL_PROVIDER_WHISPERASR) {
         const base =
             findEnvVar(env, "WHISPERASR", BASE_SUFFIX)?.value ||
             WHISPERASR_API_BASE
@@ -554,6 +555,15 @@ export async function parseTokenFromEnv(
         }
     }
 
+    if (provider === MODEL_PROVIDER_ECHO) {
+        return {
+            provider,
+            model,
+            base: undefined,
+            token: "echo",
+        }
+    }
+
     return undefined
 
     function cleanAzureBase(b: string) {

packages/core/src/constants.ts

@@ -175,7 +175,8 @@ export const MODEL_PROVIDER_MISTRAL = "mistral"
 export const MODEL_PROVIDER_LMSTUDIO = "lmstudio"
 export const MODEL_PROVIDER_JAN = "jan"
 export const MODEL_PROVIDER_DEEPSEEK = "deepseek"
-export const MODEL_WHISPERASR_PROVIDER = "whisperasr"
+export const MODEL_PROVIDER_WHISPERASR = "whisperasr"
+export const MODEL_PROVIDER_ECHO = "echo"
 
 export const MODEL_PROVIDER_OPENAI_HOSTS = Object.freeze([
     MODEL_PROVIDER_OPENAI,
@@ -223,6 +224,7 @@ export const MODEL_PROVIDERS = Object.freeze<
         transcribe?: boolean
         speech?: boolean
         tokenless?: boolean
+        hidden?: boolean
         aliases?: Record<string, string>
         env?: Record<
             string,

packages/core/src/echomodel.ts

@@ -0,0 +1,39 @@
+import { LanguageModel } from "./chat"
+import { renderMessagesToMarkdown } from "./chatrender"
+import { deleteEmptyValues } from "./cleaners"
+import { MODEL_PROVIDER_ECHO } from "./constants"
+import { logVerbose } from "./util"
+
+export const EchoModel = Object.freeze<LanguageModel>({
+    id: MODEL_PROVIDER_ECHO,
+    completer: async (req, connection, options) => {
+        const { messages, model, ...rest } = req
+        const { partialCb, inner } = options
+        const text = `## Messages
+
+${renderMessagesToMarkdown(messages, {
+    textLang: "text",
+    assistant: true,
+    system: true,
+    user: true,
+})}
+
+## Request
+
+\`\`\`json
+${JSON.stringify(deleteEmptyValues({ messages, ...rest }), null, 2)}
+\`\`\`
+`
+        partialCb?.({
+            responseChunk: text,
+            tokensSoFar: 0,
+            responseSoFar: text,
+            inner,
+        })
+
+        return {
+            finishReason: "stop",
+            text,
+        }
+    },
+})

packages/core/src/encoders.test.ts

@@ -5,27 +5,27 @@ import { dedent } from "./indent"
 
 describe("resolveTokenEncoder", () => {
     test("gpt-3.5-turbo", async () => {
-        const encoder = await resolveTokenEncoder("gpt-3.5-turbo")
+        const encoder = await resolveTokenEncoder("openai:gpt-3.5-turbo")
         const result = encoder.encode("test line")
         assert.deepEqual(result, [1985, 1584])
     })
     test("gpt-4", async () => {
-        const encoder = await resolveTokenEncoder("gpt-4")
+        const encoder = await resolveTokenEncoder("openai:gpt-4")
         const result = encoder.encode("test line")
         assert.deepEqual(result, [1985, 1584])
     })
     test("gpt-4o", async () => {
-        const encoder = await resolveTokenEncoder("gpt-4o")
+        const encoder = await resolveTokenEncoder("openai:gpt-4o")
         const result = encoder.encode("test line")
         assert.deepEqual(result, [3190, 2543])
     })
     test("gpt-4o-mini", async () => {
-        const encoder = await resolveTokenEncoder("gpt-4o-mini")
+        const encoder = await resolveTokenEncoder("openai:gpt-4o-mini")
         const result = encoder.encode("test line")
         assert.deepEqual(result, [3190, 2543])
     })
     test("gpt-4o forbidden", async () => {
-        const encoder = await resolveTokenEncoder("gpt-4o")
+        const encoder = await resolveTokenEncoder("openai:gpt-4o")
         const result = encoder.encode("<|im_end|>")
         assert.deepEqual(result, [27, 91, 321, 13707, 91, 29])
     })
@@ -57,7 +57,7 @@ For example, to denote a heading, you add a number sign before it (e.g., # Headi
             {
                 chunkSize: 128,
                 chunkOverlap: 16,
-                model: "gpt-4o",
+                model: "openai:gpt-4o",
                 lineNumbers: true,
             }
         )

packages/core/src/llms.json

@@ -423,6 +423,7 @@
         {
             "id": "github_copilot_chat",
             "detail": "GitHub Copilot Chat Models",
+            "hidden": true,
             "tools": false,
             "prediction": false,
             "tokenless": true,
@@ -433,6 +434,12 @@
                 "reasoning_small": "o1-mini"
             },
             "env": {}
+        },
+        {
+            "id": "echo",
+            "detail": "A fake LLM provider that responds with the input messages.",
+            "tools": true,
+            "tokenless": true
         }
     ],
     "aliases": {

packages/core/src/lm.ts

@@ -11,8 +11,9 @@ import {
     MODEL_PROVIDER_OLLAMA,
     MODEL_PROVIDER_TRANSFORMERS,
     MODEL_PROVIDERS,
-    MODEL_WHISPERASR_PROVIDER,
+    MODEL_PROVIDER_WHISPERASR,
     MODEL_PROVIDER_AZURE_OPENAI,
+    MODEL_PROVIDER_ECHO,
 } from "./constants"
 import { runtimeHost } from "./host"
 import { OllamaModel } from "./ollama"
@@ -22,6 +23,7 @@ import { GitHubModel } from "./github"
 import { LMStudioModel } from "./lmstudio"
 import { WhiserAsrModel } from "./whisperasr"
 import { AzureOpenAIModel } from "./azureopenai"
+import { EchoModel } from "./echomodel"
 
 export function resolveLanguageModel(provider: string): LanguageModel {
     if (provider === MODEL_PROVIDER_GITHUB_COPILOT_CHAT) {
@@ -38,7 +40,8 @@ export function resolveLanguageModel(provider: string): LanguageModel {
         return AnthropicBedrockModel
     if (provider === MODEL_PROVIDER_TRANSFORMERS) return TransformersModel
     if (provider === MODEL_PROVIDER_LMSTUDIO) return LMStudioModel
-    if (provider === MODEL_WHISPERASR_PROVIDER) return WhiserAsrModel
+    if (provider === MODEL_PROVIDER_WHISPERASR) return WhiserAsrModel
+    if (provider === MODEL_PROVIDER_ECHO) return EchoModel
 
     const features = MODEL_PROVIDERS.find((p) => p.id === provider)
     return LocalOpenAICompatibleModel(provider, {

packages/core/src/markdown.ts

@@ -4,6 +4,7 @@
 // and working with trace trees.
 
 import { convertAnnotationsToMarkdown } from "./annotations"
+import { collapseNewlines } from "./cleaners"
 import { fenceMD } from "./mkmd"
 import { convertThinkToMarkdown } from "./think"
 
@@ -16,19 +17,10 @@ export function prettifyMarkdown(md: string) {
     let res = md
     res = convertAnnotationsToMarkdown(res) // Convert annotations to markdown format
     res = convertThinkToMarkdown(res)
-    res = cleanMarkdown(res) // Clean up excessive newlines
+    res = collapseNewlines(res) // Clean up excessive newlines
     return res
 }
 
-/**
- * Cleans markdown by reducing multiple consecutive newlines to two.
- * @param res - The string to be cleaned.
- * @returns The cleaned string.
- */
-function cleanMarkdown(res: string): string {
-    return res?.replace(/(\r?\n){3,}/g, "\n\n")
-}
-
 /**
  * Renders an object to a markdown string.
  * @param obj - The object to render.

packages/core/src/models.test.ts

@@ -37,16 +37,16 @@ describe("parseModelIdentifier", () => {
         const { provider, model, family } = parseModelIdentifier("llamafile")
         assert(provider === MODEL_PROVIDER_LLAMAFILE)
         assert(family === "*")
-        assert(model === "llamafile")
+        assert(model === "*")
     })
     test("github:gpt4", () => {
         const { provider, model, family } = parseModelIdentifier("github:gpt4")
         assert(provider === MODEL_PROVIDER_GITHUB)
         assert(model === "gpt4")
         assert(family === "gpt4")
     })
-    test("gpt4", () => {
-        const { provider, model, family } = parseModelIdentifier("gpt4")
+    test("openai:gpt4", () => {
+        const { provider, model, family } = parseModelIdentifier("openai:gpt4")
         assert(provider === MODEL_PROVIDER_OPENAI)
         assert(model === "gpt4")
         assert(family === "gpt4")