promptfoo / redteam (#1114)

THIRD_PARTY_LICENSES.md

@@ -190,9 +190,9 @@ The following npm packages may be included in this product:
 
  - @aws-crypto/[email protected]
  - @aws-crypto/[email protected]
- - @huggingface/[email protected].2
- - flatbuffers@1.12.0
- - long@5.2.4
+ - @huggingface/[email protected].3
+ - flatbuffers@25.1.24
+ - long@5.3.0
 
 These packages each contain the following license:
 
@@ -1956,11 +1956,11 @@ Apache License
 
 The following npm packages may be included in this product:
 
- - @aws-sdk/client-cognito-identity@3.742.0
- - @aws-sdk/client-sso@3.734.0
- - @aws-sdk/credential-provider-cognito-identity@3.742.0
+ - @aws-sdk/client-cognito-identity@3.744.0
+ - @aws-sdk/client-sso@3.744.0
+ - @aws-sdk/credential-provider-cognito-identity@3.744.0
  - @aws-sdk/[email protected]
- - @aws-sdk/util-user-agent-node@3.734.0
+ - @aws-sdk/util-user-agent-node@3.744.0
  - @smithy/[email protected]
  - @smithy/[email protected]
  - @smithy/[email protected]
@@ -2182,7 +2182,7 @@ Apache License
 
 The following npm package may be included in this product:
 
- - @aws-sdk/client-bedrock-runtime@3.741.0
+ - @aws-sdk/client-bedrock-runtime@3.744.0
 
 This package contains the following license:
 
@@ -2394,7 +2394,7 @@ The following npm packages may be included in this product:
 
  - @aws-sdk/[email protected]
  - @aws-sdk/[email protected]
- - @aws-sdk/middleware-user-agent@3.734.0
+ - @aws-sdk/middleware-user-agent@3.744.0
  - @smithy/[email protected]
  - @smithy/[email protected]
  - @smithy/[email protected]
@@ -2862,7 +2862,7 @@ MIT License
 
 The following npm package may be included in this product:
 
- - genaiscript-vscode@1.101.1
+ - genaiscript-vscode@1.102.0
 
 This package contains the following license:
 
@@ -4211,14 +4211,14 @@ Apache License
 
 The following npm packages may be included in this product:
 
- - @aws-sdk/credential-provider-env@3.734.0
- - @aws-sdk/credential-provider-ini@3.741.0
- - @aws-sdk/credential-provider-node@3.741.0
- - @aws-sdk/credential-providers@3.742.0
+ - @aws-sdk/credential-provider-env@3.744.0
+ - @aws-sdk/credential-provider-ini@3.744.0
+ - @aws-sdk/credential-provider-node@3.744.0
+ - @aws-sdk/credential-providers@3.744.0
  - @aws-sdk/[email protected]
- - @aws-sdk/token-providers@3.734.0
+ - @aws-sdk/token-providers@3.744.0
  - @aws-sdk/[email protected]
- - @aws-sdk/util-endpoints@3.734.0
+ - @aws-sdk/util-endpoints@3.743.0
  - @aws-sdk/[email protected]
  - @aws-sdk/[email protected]
  - @smithy/[email protected]
@@ -4471,9 +4471,9 @@ Apache License
 
 The following npm packages may be included in this product:
 
- - @aws-sdk/credential-provider-process@3.734.0
- - @aws-sdk/credential-provider-sso@3.734.0
- - @aws-sdk/credential-provider-web-identity@3.734.0
+ - @aws-sdk/credential-provider-process@3.744.0
+ - @aws-sdk/credential-provider-sso@3.744.0
+ - @aws-sdk/credential-provider-web-identity@3.744.0
 
 These packages each contain the following license:
 
@@ -5374,9 +5374,9 @@ END OF TERMS AND CONDITIONS
 
 The following npm packages may be included in this product:
 
- - @aws-sdk/core@3.734.0
- - @aws-sdk/credential-provider-http@3.734.0
- - @aws-sdk/nested-clients@3.734.0
+ - @aws-sdk/core@3.744.0
+ - @aws-sdk/credential-provider-http@3.744.0
+ - @aws-sdk/nested-clients@3.744.0
  - [email protected]
 
 These packages each contain the following license:
@@ -6547,6 +6547,36 @@ EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 
 -----------
 
+The following npm package may be included in this product:
+
+ - @azure/[email protected]
+
+This package contains the following license:
+
+Copyright (c) Microsoft Corporation.
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED *AS IS*, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+-----------
+
 The following npm packages may be included in this product:
 
  - [email protected]
@@ -8009,24 +8039,24 @@ SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 The following npm packages may be included in this product:
 
  - @esbuild/[email protected]
- - @napi-rs/[email protected].66
- - @napi-rs/[email protected].66
+ - @napi-rs/[email protected].67
+ - @napi-rs/[email protected].67
  - @tokenizer/[email protected]
  - [email protected]
  - [email protected]
- - genaiscript-core-internal@1.101.1
- - genaiscript-sample@1.101.1
- - genaiscript-web@1.101.1
- - genaiscript@1.101.1
+ - genaiscript-core-internal@1.102.0
+ - genaiscript-sample@1.102.0
+ - genaiscript-web@1.102.0
+ - genaiscript@1.102.0
  - [email protected]
  - [email protected]
  - [email protected]
  - [email protected]
  - [email protected]
  - [email protected]
- - [email protected].20241212-1f88284f96
+ - [email protected].20250206-d981b153d3
  - [email protected]
- - [email protected].20250114-228dd16893
+ - [email protected].20250206-d981b153d3
  - [email protected]
  - [email protected]
  - [email protected]
@@ -8705,7 +8735,7 @@ SOFTWARE.
 
 The following npm package may be included in this product:
 
- - @napi-rs/[email protected].66
+ - @napi-rs/[email protected].67
 
 This package contains the following license:
 
@@ -9361,7 +9391,7 @@ The following npm packages may be included in this product:
  - [email protected]
  - [email protected]
  - [email protected]
- - [email protected].5
+ - [email protected].6
  - [email protected]
  - [email protected]
  - [email protected]
@@ -9384,7 +9414,7 @@ The following npm packages may be included in this product:
  - [email protected]
  - [email protected]
  - [email protected]
- - type-fest@4.33.0
+ - type-fest@4.34.1
  - [email protected]
  - [email protected]
  - [email protected]
@@ -9666,7 +9696,7 @@ SOFTWARE.
 The following npm packages may be included in this product:
 
  - @octokit/[email protected]
- - @octokit/[email protected].0
+ - @octokit/[email protected].1
  - @octokit/[email protected]
 
 These packages each contain the following license:
@@ -11735,7 +11765,6 @@ The following npm packages may be included in this product:
  - @azure/[email protected]
  - @azure/[email protected]
  - @azure/[email protected]
- - @azure/[email protected]
  - @azure/[email protected]
  - @azure/[email protected]
  - @azure/[email protected]

docs/src/content/docs/reference/cli/commands.md

@@ -85,7 +85,7 @@ Options:
 ## `test`
 
 ```
-Usage: genaiscript test [options] [command]
+Usage: genaiscript test|eval [options] [command]
 
 Options:
   -h, --help                 display help for command
@@ -109,6 +109,7 @@ Arguments:
                                       are tested
 
 Options:
+  --redteam                           run red team tests
   -p, --provider <string>             Preferred LLM provider aliases (choices:
                                       "openai", "azure", "azure_serverless",
                                       "azure_serverless_models", "anthropic",
@@ -147,6 +148,7 @@ Usage: genaiscript test list [options]
 List available tests in workspace
 
 Options:
+  --redteam                 list red team tests
   -g, --groups <groups...>  groups to include or exclude. Use :! prefix to
                             exclude
   -h, --help                display help for command