Merge branch 'main' of https://github.com/microsoft/genaiscript

.github/workflows/npm-check.yml

@@ -4,10 +4,24 @@ on:
     schedule:
         - cron: "0 4 * * *"
 jobs:
-    build:
+    ubuntu:
         runs-on: ubuntu-latest
         steps:
             - uses: actions/setup-node@v4
               with:
                   node-version: "20"
             - run: npx --yes genaiscript --help
+    windows:
+        runs-on: windows-latest
+        steps:
+            - uses: actions/setup-node@v4
+              with:
+                  node-version: "20"
+            - run: npx --yes genaiscript --help
+    macos:
+        runs-on: macos-latest
+        steps:
+            - uses: actions/setup-node@v4
+              with:
+                  node-version: "20"
+            - run: npx --yes genaiscript --help

README.md

@@ -291,6 +291,19 @@ Write me a poem
 
 ---
 
+### Pluggable Secret Scanning
+
+Scan your chats for secrets using [secret scanning](/genaiscript/reference/scripts/secret-scanning).
+
+```json
+{
+    "secretPatterns": {
+        ...,
+        "OpenAI API Key": "sk-[A-Za-z0-9]{32,48}"
+    }
+}
+```
+
 ### ⚙ Automate with CLI or API
 
 Automate using the [CLI](https://microsoft.github.io/genaiscript/reference/cli) or [API](https://microsoft.github.io/genaiscript/reference/cli/api).

docs/package.json

@@ -2,7 +2,7 @@
   "name": "docs",
   "type": "module",
   "private": true,
-  "version": "1.97.1",
+  "version": "1.97.2",
   "license": "MIT",
   "scripts": {
     "install:force": "rm yarn.lock && yarn install",

docs/public/schemas/config.json

@@ -40,7 +40,7 @@
         "modelAliases": {
             "type": "object",
             "patternProperties": {
-                "^[a-zA-Z0-9_:]+$": {
+                "^[a-zA-Z0-9_]+$": {
                     "oneOf": [
                         {
                             "type": "string",
@@ -64,7 +64,18 @@
                 }
             },
             "additionalProperties": true,
-            "description": "Aliases for model identifiers (provider:model:tag)"
+            "description": "Aliases for model identifiers (name)"
+        },
+        "secretPatterns": {
+            "type": "object",
+            "patternProperties": {
+                "^[a-zA-Z0-9_:\\-\\. ]+$": {
+                    "type": ["string", "null"],
+                    "description": "Secret regex"
+                }
+            },
+            "additionalProperties": true,
+            "description": "Secret scanners to use for scanning chat messages"
         }
     }
 }

docs/src/content/docs/index.mdx

@@ -360,7 +360,7 @@ to validate [content safety](/genaiscript/reference/scripts/content-safety).
 
 ```js wrap
 script({ ...,
-    system: ["system.safety_harmful_content", ...],
+    systemSafety: "default",
     contentSafety: "azure" // use azure content safety
 })
 
@@ -486,6 +486,21 @@ importTemplate("poem.prompty", { something: "code " })
 
 </Card>
 
+<Card title="Pluggable Secret Scanning" icon="seti:license">
+
+Scan your chats for secrets using [secret scanning](/genaiscript/reference/scripts/secret-scanning).
+
+```json title="genaiscript.config.json"
+{
+    "secretPatterns": {
+        ...,
+        "OpenAI API Key": "sk-[A-Za-z0-9]{32,48}"
+    }
+}
+```
+
+</Card>
+
 <Card title="Automate with CLI" icon="github">
 
 Automate using the [CLI](/genaiscript/reference/cli),

docs/src/content/docs/reference/cli/commands.md

@@ -452,6 +452,7 @@ Commands:
   jsonl2json                   Converts JSONL files to a JSON file
   prompty [options] <file...>  Converts .prompty files to genaiscript
   jinja2 [options] <file>      Renders Jinj2 or prompty template
+  secrets <file...>            Applies secret scanning and redaction to files
 ```
 
 ### `parse data`
@@ -578,6 +579,20 @@ Options:
   -h, --help             display help for command
 ```
 
+### `parse secrets`
+
+```
+Usage: genaiscript parse secrets [options] <file...>
+
+Applies secret scanning and redaction to files
+
+Arguments:
+  file        input files
+
+Options:
+  -h, --help  display help for command
+```
+
 ## `info`
 
 ```

docs/src/content/docs/reference/scripts/content-safety.mdx

@@ -16,6 +16,14 @@ The following safety prompts are included by default when running a prompt, unle
 - [system.safety_jailbreak](/genaiscript/reference/scripts/system#systemsafety_jailbreak), safety script to ignore prompting instructions in code sections, which are created by the `def` function.
 - [system.safety_protected_material](/genaiscript/reference/scripts/system#systemsafety_protected_material) safety prompt against Protected material. See https://learn.microsoft.com/en-us/azure/ai-services/openai/concepts/safety-system-message-templates
 
+You can ensure those safety are always used by setting the `systemSafety` option to `default`.
+
+```js
+script({
+    systemSafety: "default",
+})
+```
+
 Other system scripts can be added to the prompt by using the `system` option.
 
 - [system.safety_ungrounded_content_summarization](/genaiscript/reference/scripts/system#systemsafety_ungrounded_content_summarization) safety prompt against ungrounded content in summarization

docs/src/content/docs/reference/scripts/images.md

@@ -30,7 +30,7 @@ Local files are loaded and encoded as a data uri.
 
 ## Buffer, Blob, ReadableStream
 
-The `defImages` function also supports [Buffer](https://nodejs.org/api/buffer.html), 
+The `defImages` function also supports [Buffer](https://nodejs.org/api/buffer.html),
 [Blob](https://developer.mozilla.org/en-US/docs/Web/API/Blob), [ReadableStream](https://nodejs.org/api/stream.html).
 
 This example takes a screenshot of bing.com and adds it to the images.
@@ -108,3 +108,14 @@ defImages(img, { maxWidth: 800 })
 // and / or
 defImages(img, { maxHeight: 800 })
 ```
+
+## Tiling
+
+When you specify the `tiled: true` option, all the images will
+be tiled in a single image, after all the transformations are applied.
+
+The resulting image will be further resized to fit into the maximum image size constraints.
+
+```js "tiled: true"
+defImages(env.files, { details: "low", tiled: true })
+```

docs/src/content/docs/reference/scripts/secret-scanning.mdx

@@ -0,0 +1,77 @@
+---
+title: Secret Scanning
+sidebar:
+    order: 10
+---
+
+One should not have secrets lying around in their codebase, but sometimes it happens.
+To help you avoid this, we have a secret scanning feature that will scan your codebase for secrets
+and warn you if any are found.
+
+:::note
+
+The secret scanning feature is by no means exhaustive and should not be relied upon as the sole
+method of securing your codebase. It is a best-effort feature that will help you avoid common mistakes.
+
+:::
+
+## Supported patterns
+
+By default set of secret patterns
+are defined at https://github.com/microsoft/genaiscript/tree/main/packages/core/src/config.json.
+
+:::cautio
+
+\is is not a complete list by design, and needs to be updated to match your needs.
+
+:::
+
+You can find examples of patterns at https://github.com/mazen160/secrets-patterns-db/.
+
+## Scanning messages
+
+By default, all messages sent to LLMs are scanned and redacted if they contain secrets.
+
+You can disable secret scanning alltogher by setting the `secretScanning` option to `false` in your script.
+
+```js
+script({
+    secretScanning: false,
+})
+```
+
+## Configuring patterns
+
+If you have a specific pattern that you want to scan for, you can configure it in your
+[configuration file](/genaiscript/reference/configuration-files).
+
+```json title="genaiscript.config.json"
+{
+    "secretPatterns": {
+        ...,
+        "my secret pattern": "my-secret-pattern-regex"
+    }
+}
+```
+
+- do not use `^` or `$` in your regex pattern
+
+### Disabling patterns
+
+Set the pattern key to `null` or `false` to disable it.
+
+```json title="genaiscript.config.json"
+{
+    "secretPatterns": {
+        "OpenAI API Key": null
+    }
+}
+```
+
+## CLI
+
+You can test your patterns against files using the CLI.
+
+```sh
+npx --yes genaiscript parse secrets *
+```

docs/src/content/docs/reference/scripts/system.mdx

@@ -3463,7 +3463,8 @@ system({
     title: "Tools support",
 })
 
-$`Use tools if possible. 
+$`## Tools
+Use tools if possible. 
 - **Do NOT invent function names**. 
 - **Do NOT use function names starting with 'functions.'.
 - **Do NOT respond with multi_tool_use**.`

package.json

@@ -1,6 +1,6 @@
 {
     "name": "genaiscript-workspace",
-    "version": "1.97.1",
+    "version": "1.97.2",
     "license": "MIT",
     "private": true,
     "workspaces": {
@@ -99,7 +99,7 @@
         }
     },
     "devDependencies": {
-        "@inquirer/prompts": "^7.2.3",
+        "@inquirer/prompts": "^7.2.4",
         "glob": "^11.0.1",
         "npm-check-updates": "^17.1.14",
         "npm-run-all": "^4.1.5",

packages/cli/package.json

@@ -1,6 +1,6 @@
 {
   "name": "genaiscript",
-  "version": "1.97.1",
+  "version": "1.97.2",
   "main": "built/genaiscript.cjs",
   "type": "commonjs",
   "bin": {
@@ -52,7 +52,7 @@
     "@anthropic-ai/bedrock-sdk": "^0.12.4",
     "@anthropic-ai/sdk": "^0.36.3",
     "@azure/identity": "^4.6.0",
-    "@inquirer/prompts": "^7.2.3",
+    "@inquirer/prompts": "^7.2.4",
     "@modelcontextprotocol/sdk": "^1.4.1",
     "@octokit/plugin-paginate-rest": "^11.3.6",
     "@octokit/plugin-retry": "^7.1.2",
@@ -124,9 +124,9 @@
     "compile:runtime:code": "tsc src/runtime.ts --skipLibCheck --outDir built --target es2020 --moduleResolution node --module esnext --removeComments true",
     "compile:runtime": "yarn compile:runtime:declarations && yarn compile:runtime:code && mv built/runtime.js built/runtime.mjs",
     "compile:api": "esbuild src/api.ts --outfile=built/api.mjs",
-    "compile:cli": "esbuild src/main.ts --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity --external:gpt-tokenizer --external:playwright --external:@inquirer/prompts --external:jimp --external:turndown --external:turndown-plugin-gfm --external:vectra --external:tabletojson --external:html-to-text --external:@octokit/rest --external:@octokit/plugin-throttling --external:@octokit/plugin-retry --external:@octokit/plugin-paginate-rest --external:skia-canvas --external:@huggingface/transformers --external:@modelcontextprotocol/sdk --external:@anthropic-ai/sdk --external:@anthropic-ai/bedrock-sdk --external:es-toolkit --external:zod --external:zod-to-json-schema --external:fluent-ffmpeg --external:json-schema-generator && node ../../scripts/patch-cli.mjs",
+    "compile:cli": "esbuild src/main.ts --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity --external:gpt-tokenizer --external:playwright --external:@inquirer/prompts --external:jimp --external:turndown --external:turndown-plugin-gfm --external:vectra --external:tabletojson --external:html-to-text --external:@octokit/rest --external:@octokit/plugin-throttling --external:@octokit/plugin-retry --external:@octokit/plugin-paginate-rest --external:skia-canvas --external:@huggingface/transformers --external:@modelcontextprotocol/sdk --external:@anthropic-ai/sdk --external:@anthropic-ai/bedrock-sdk --external:es-toolkit --external:zod --external:zod-to-json-schema --external:fluent-ffmpeg --external:json-schema-generator --external:@inquirer/prompts && node ../../scripts/patch-cli.mjs",
     "compile": "yarn compile:api && yarn compile:runtime && yarn compile:cli",
-    "compile-debug": "esbuild src/main.ts --sourcemap --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity --external:gpt-tokenizer --external:playwright --external:@inquirer/prompts --external:jimp --external:turndown --external:turndown-plugin-gfm --external:vectra --external:tabletojson --external:html-to-text --external:@octokit/rest --external:@octokit/plugin-throttling --external:@octokit/plugin-retry --external:@octokit/plugin-paginate-rest --external:skia-canvas --external:@huggingface/transformers --external:@modelcontextprotocol/sdk --external:@anthropic-ai/sdk --external:@anthropic-ai/bedrock-sdk --external:es-toolkit --external:zod --external:zod-to-json-schema --external:fluent-ffmpeg --external:json-schema-generator",
+    "compile-debug": "esbuild src/main.ts --sourcemap --metafile=./esbuild.meta.json --bundle --platform=node --target=node20 --outfile=built/genaiscript.cjs --external:tsx --external:esbuild --external:get-tsconfig --external:resolve-pkg-maps --external:dockerode --external:pdfjs-dist --external:web-tree-sitter --external:tree-sitter-wasms --external:promptfoo --external:typescript --external:@lvce-editor/ripgrep --external:gpt-3-encoder --external:mammoth --external:xlsx --external:mathjs --external:@azure/identity --external:gpt-tokenizer --external:playwright --external:@inquirer/prompts --external:jimp --external:turndown --external:turndown-plugin-gfm --external:vectra --external:tabletojson --external:html-to-text --external:@octokit/rest --external:@octokit/plugin-throttling --external:@octokit/plugin-retry --external:@octokit/plugin-paginate-rest --external:skia-canvas --external:@huggingface/transformers --external:@modelcontextprotocol/sdk --external:@anthropic-ai/sdk --external:@anthropic-ai/bedrock-sdk --external:es-toolkit --external:zod --external:zod-to-json-schema --external:fluent-ffmpeg --external:json-schema-generator --external:@inquirer/prompts",
     "postcompile": "node built/genaiscript.cjs info help > ../../docs/src/content/docs/reference/cli/commands.md",
     "vis:treemap": "npx --yes esbuild-visualizer --metadata esbuild.meta.json --filename esbuild.treemap.html",
     "vis:network": "npx --yes esbuild-visualizer --metadata esbuild.meta.json --filename esbuild.network.html --template network",

packages/cli/src/cli.ts

@@ -19,6 +19,7 @@ import {
     parseHTMLToText,
     parseJinja2,
     parsePDF,
+    parseSecrets,
     parseTokens,
     prompty2genaiscript,
 } from "./parse" // Parsing functions
@@ -479,6 +480,11 @@ export async function cli() {
             "variables, as name=value passed to the template"
         )
         .action(parseJinja2)
+    parser
+        .command("secrets")
+        .description("Applies secret scanning and redaction to files")
+        .argument("<file...>", "input files")
+        .action(parseSecrets)
 
     // Define 'info' command group for utility information tasks
     const info = program.command("info").description("Utility tasks")

packages/cli/src/input.ts

@@ -17,6 +17,7 @@ export async function shellSelect(
     choices: string[],
     options?: ShellSelectOptions
 ): Promise<string> {
+    const { select } = await import("@inquirer/prompts")
     const res = await select<string>({
         ...(options || {}), // Spread operator to include any optional configurations
         message, // The message/question to display
@@ -35,6 +36,7 @@ export async function shellInput(
     message: string,
     options?: ShellInputOptions
 ): Promise<string> {
+    const { input } = await import("@inquirer/prompts")
     const res = await input({
         ...(options || {}), // Include optional configurations if any
         message, // The message to display to the user
@@ -52,6 +54,7 @@ export async function shellConfirm(
     message: string,
     options?: ShellConfirmOptions
 ): Promise<boolean> {
+    const { confirm } = await import("@inquirer/prompts")
     const res = await confirm({
         ...(options || {}), // Include optional configurations if any
         message, // The message to display, usually a yes/no question

packages/cli/src/parse.ts

@@ -25,6 +25,7 @@ import { splitMarkdown } from "../../core/src/frontmatter"
 import { parseOptionsVars } from "./vars"
 import { dataTryParse } from "../../core/src/data"
 import { resolveFileContent } from "../../core/src/file"
+import { redactSecrets } from "../../core/src/secretscanner"
 
 /**
  * This module provides various parsing utilities for different file types such
@@ -215,3 +216,20 @@ export async function prompty2genaiscript(
         await writeText(gf, script)
     }
 }
+
+export async function parseSecrets(files: string[]) {
+    const fs = await expandFiles(files)
+    let n = 0
+    for (const f of fs) {
+        const content = await readText(f)
+        const { found } = redactSecrets(content)
+        const entries = Object.entries(found)
+        if (entries.length) {
+            n++
+            console.log(
+                `${f}: ${entries.map(([k, v]) => `${k} (${v})`).join(", ")}`
+            )
+        }
+    }
+    if (n > 0) console.warn(`found secrets in ${n} of ${fs.length} files`)
+}

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "genaiscript-core-internal",
-  "version": "1.97.1",
+  "version": "1.97.2",
   "main": "src/index.ts",
   "license": "MIT",
   "private": true,