add redteam option

docs/src/content/docs/reference/cli/commands.md

@@ -109,6 +109,7 @@ Arguments:
                                       are tested
 
 Options:
+  --redteam                           run red team tests
   -p, --provider <string>             Preferred LLM provider aliases (choices:
                                       "openai", "azure", "azure_serverless",
                                       "azure_serverless_models", "anthropic",
@@ -147,6 +148,7 @@ Usage: genaiscript test list [options]
 List available tests in workspace
 
 Options:
+  --redteam                 run red team tests
   -g, --groups <groups...>  groups to include or exclude. Use :! prefix to
                             exclude
   -h, --help                display help for command

packages/cli/src/cli.ts

@@ -229,6 +229,7 @@ export async function cli() {
             "[script...]",
             "Script ids. If not provided, all scripts are tested"
         )
+        .option("--redteam", "run red team tests")
     addModelOptions(testRun) // Add model options to the command
         .option(
             "--models <models...>",
@@ -254,11 +255,12 @@ export async function cli() {
     // List available tests
     test.command("list")
         .description("List available tests in workspace")
-        .action(scriptTestList) // Action to list the tests
+        .option("--redteam", "list red team tests")
         .option(
             "-g, --groups <groups...>",
             "groups to include or exclude. Use :! prefix to exclude"
         )
+        .action(scriptTestList) // Action to list the tests
 
     // Launch test viewer
     test.command("view")

packages/cli/src/test.ts

@@ -48,6 +48,7 @@ import {
     CancellationOptions,
     checkCancelled,
 } from "../../core/src/cancellation"
+import { CORE_VERSION } from "../../core/src/version"
 
 /**
  * Parses model specifications from a string and returns a ModelOptions object.
@@ -107,13 +108,14 @@ export async function runPromptScriptTests(
         cache?: boolean
         verbose?: boolean
         write?: boolean
+        redteam?: boolean
         promptfooVersion?: string
         outSummary?: string
         testDelay?: string
     } & CancellationOptions
 ): Promise<PromptScriptTestRunResponse> {
     applyModelOptions(options, "cli")
-    const { cancellationToken } = options || {}
+    const { cancellationToken, redteam } = options || {}
     const scripts = await listTests({ ids, ...(options || {}) })
     if (!scripts.length)
         return {
@@ -155,7 +157,7 @@ export async function runPromptScriptTests(
 - Run this command to launch the promptfoo test viewer.
 
 \`\`\`sh
-genaiscript test view
+npx --yes genaiscript@${CORE_VERSION} test view
 \`\`\`
 
 `
@@ -187,6 +189,7 @@ genaiscript test view
             provider: "provider.mjs",
             chatInfo,
             embeddingsInfo,
+            redteam,
         })
         const yaml = YAMLStringify(config)
         await writeFile(fn, yaml)
@@ -277,11 +280,16 @@ genaiscript test view
  * @param options - Options to filter the test scripts by IDs or groups.
  * @returns A Promise resolving to an array of filtered scripts.
  */
-async function listTests(options: { ids?: string[]; groups?: string[] }) {
+async function listTests(options: {
+    ids?: string[]
+    groups?: string[]
+    redteam?: boolean
+}) {
     const prj = await buildProject()
     const scripts = filterScripts(prj.scripts, {
         ...(options || {}),
-        test: true,
+        test: options.redteam ? undefined : true,
+        redteam: options.redteam,
     })
     return scripts
 }
@@ -300,6 +308,7 @@ export async function scriptsTest(
         cache?: boolean
         verbose?: boolean
         write?: boolean
+        redteam?: boolean
         promptfooVersion?: string
         outSummary?: string
         testDelay?: string
@@ -320,7 +329,10 @@ export async function scriptsTest(
  * Lists available test scripts, printing their IDs and filenames.
  * @param options - Options to filter the scripts by groups.
  */
-export async function scriptTestList(options: { groups?: string[] }) {
+export async function scriptTestList(options: {
+    groups?: string[]
+    redteam?: boolean
+}) {
     const scripts = await listTests(options)
     console.log(scripts.map((s) => toStringList(s.id, s.filename)).join("\n"))
 }

packages/core/src/ast.ts

@@ -89,15 +89,17 @@ export interface ScriptFilterOptions {
     ids?: string[]
     groups?: string[]
     test?: boolean
+    redteam?: boolean
 }
 
 export function filterScripts(
     scripts: PromptScript[],
     options: ScriptFilterOptions
 ) {
-    const { ids, groups, test } = options || {}
+    const { ids, groups, test, redteam } = options || {}
     return scripts
         .filter((t) => !test || arrayify(t.tests)?.length)
+        .filter((t) => !redteam || t.redteam)
         .filter((t) => !ids?.length || ids.includes(t.id))
         .filter((t) => tagFilter(groups, t.group))
 }

packages/core/src/indent.ts

@@ -7,4 +7,14 @@ export function indent(text: string, indentation: string) {
         .join("\n")
 }
 
-export const dedent = tsDedent
+/**
+ * Unindents a string
+ */
+export function dedent(
+    templ: TemplateStringsArray | string,
+    ...values: unknown[]
+): string {
+    if (templ === undefined) return undefined
+    if (templ === null) return null
+    return tsDedent(templ, ...values)
+}

packages/core/src/promptfoo.ts

@@ -22,6 +22,7 @@ import { validateJSONWithSchema } from "./schema"
 import { TraceOptions } from "./trace"
 import { CancellationOptions } from "./cancellation"
 import { uniq } from "es-toolkit"
+import { dedent } from "./indent"
 
 /**
  * Convert GenAIScript connection info into prompt foo configuration
@@ -94,7 +95,8 @@ export async function generatePromptFooConfiguration(
         embeddingsInfo,
         trace,
     } = options || {}
-    const { description, title, id } = script
+    const { title, id } = script
+    const description = dedent(script.description)
     const models = options?.models || []
     const redteam: Partial<PromptRedteam> = options?.redteam
         ? script.redteam || {}
@@ -271,7 +273,9 @@ export async function generatePromptFooConfiguration(
             ? deleteEmptyValues({
                   injectVar: "fileContent",
                   numTests: redteam.numTests || PROMPTFOO_REDTEAM_NUM_TESTS,
-                  purpose: redteam.purpose || description || title || id,
+                  purpose: dedent(
+                      redteam.purpose || description || title || id
+                  ),
                   plugins: uniq(arrayify(redteam.plugins)),
                   strategies: uniq(arrayify(redteam.strategies)),
                   language: redteam.language,