test: move prompt into one place #27
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: evaluate prompt | |
| on: | |
| pull_request: | |
| paths: | |
| - "evalprompt/**" | |
| - "src/prompts/**" | |
| branches: | |
| - main | |
| - dev | |
| jobs: | |
| prompt-evaluation: | |
| runs-on: ubuntu-latest | |
| environment: engineering | |
| permissions: | |
| contents: read # Required for actions/checkout | |
| pull-requests: write # Ability to post comments on Pull Requests | |
| id-token: write # Required for Azure login | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-node@v3 | |
| with: | |
| node-version: lts/* | |
| - name: Setup project | |
| run: | | |
| npm install | |
| - name: Azure login | |
| uses: azure/login@v2 | |
| with: | |
| client-id: ${{ secrets.EVALUATION_TEST_CLIENT_ID }} | |
| tenant-id: ${{ secrets.EVALUATION_TEST_TENANT_ID }} | |
| subscription-id: ${{ secrets.EVALUATION_TEST_SUB_ID }} | |
| enable-AzPSSession: true | |
| - name: Set up promptfoo cache | |
| uses: actions/cache@v3 | |
| with: | |
| path: ~/.cache/promptfoo | |
| key: ${{ runner.os }}-promptfoo-v1 | |
| restore-keys: | | |
| ${{ runner.os }}-promptfoo- | |
| - name: Prompt Evaluation | |
| continue-on-error: true | |
| env: | |
| PROMPTFOO_CACHE_PATH: ~/.cache/promptfoo | |
| run: | | |
| for dir in evalprompt/*/; do | |
| if [ -d "$dir" ] && [ "$(basename "$dir")" != "example" ]; then | |
| cd "$dir" || continue | |
| npx promptfoo eval -o output.json --share || true | |
| cd - > /dev/null || exit | |
| fi | |
| done | |
| - name: generate comments doc | |
| run: | | |
| node .github/workflows/scripts/collectPromptRes.js | |
| cat ./commentContent.txt | |
| - uses: actions/github-script@v7 | |
| with: | |
| github-token: ${{secrets.GITHUB_TOKEN}} | |
| script: | | |
| const fs = require('fs'); | |
| const content = fs.readFileSync('./commentContent.txt', 'utf8'); | |
| github.rest.issues.createComment({ | |
| issue_number: context.issue.number, | |
| owner: context.repo.owner, | |
| repo: context.repo.repo, | |
| body: content | |
| }) | |