✨ feat: update options and concurrency settings for tests

Enhanced CLI commands, added test max concurrency constant.
microsoft · Feb 10, 2025 · e697408 · e697408
1 parent 13cb79c
commit e697408
Show file tree

Hide file tree

Showing 3 changed files with 6 additions and 3 deletions.
diff --git a/docs/src/content/docs/reference/cli/commands.md b/docs/src/content/docs/reference/cli/commands.md
@@ -148,7 +148,7 @@ Usage: genaiscript test list [options]
 List available tests in workspace
 
 Options:
-  --redteam                 run red team tests
+  --redteam                 list red team tests
   -g, --groups <groups...>  groups to include or exclude. Use :! prefix to
                             exclude
   -h, --help                display help for command

diff --git a/packages/cli/src/test.ts b/packages/cli/src/test.ts
@@ -18,6 +18,7 @@ import {
     EMOJI_FAIL,
     TEST_RUNS_DIR_NAME,
     PROMPTFOO_REMOTE_API_PORT,
+    PROMPTFOO_TEST_MAX_CONCURRENCY,
 } from "../../core/src/constants"
 import { promptFooDriver } from "../../core/src/default_prompts"
 import { serializeError } from "../../core/src/error"
@@ -196,6 +197,7 @@ npx --yes genaiscript@${CORE_VERSION} test view
         configurations.push({ script, configuration: fn })
     }
 
+    const promptFooVersion = options.promptfooVersion || PROMPTFOO_VERSION
     const results: PromptScriptTestResult[] = []
     // Execute each configuration and gather results
     for (const config of configurations) {
@@ -205,12 +207,12 @@ npx --yes genaiscript@${CORE_VERSION} test view
         const cmd = "npx"
         const args = [
             "--yes",
-            `promptfoo@${options.promptfooVersion || PROMPTFOO_VERSION}`,
+            `promptfoo@${promptFooVersion}`,
             "eval",
             "--config",
             configuration,
             "--max-concurrency",
-            "1",
+            String(PROMPTFOO_TEST_MAX_CONCURRENCY),
             "--no-progress-bar",
         ]
         if (options.cache) args.push("--cache")

diff --git a/packages/core/src/constants.ts b/packages/core/src/constants.ts
@@ -136,6 +136,7 @@ export const PROMPTFOO_CACHE_PATH = ".genaiscript/cache/tests"
 export const PROMPTFOO_CONFIG_DIR = ".genaiscript/config/tests"
 export const PROMPTFOO_REMOTE_API_PORT = 15500
 export const PROMPTFOO_REDTEAM_NUM_TESTS = 5
+export const PROMPTFOO_TEST_MAX_CONCURRENCY = 1
 
 export const RUNS_DIR_NAME = "runs"
 export const CONVERTS_DIR_NAME = "converts"