Skip to content

Commit 8b81655

Browse files
placerdaplacerda
committedJul 29, 2024
flow evaluations
1 parent 8b07ba0 commit 8b81655

14 files changed

+545
-100
lines changed
 

‎.github/actions/config-env/action.yml

+47-10
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
name: Config Environment
22
description: >
33
This action will provision an Azure environment and read its variables (deployment outputs).
4-
If you don't want to do the provisioning, just get env variables by setting MANUAL_PROVISIONING input to true.
4+
Provision will be done in case there is no previous deployment or the resource group doesn't exist.
5+
If there's already a deployment, it will update the provisioned resources in case the template has changed.
6+
If you don't want to do the provisioning at all, just get env variables by setting MANUAL_PROVISIONING input to true.
57
inputs:
68
AZURE_CREDENTIALS:
79
description: 'Azure credentials'
@@ -63,14 +65,34 @@ runs:
6365
shell: bash
6466
id: setup-env-step
6567
run: |
66-
67-
VARIABLES=("AZUREAI_HUB_NAME" "AZUREAI_PROJECT_NAME" "AZURE_APP_INSIGHTS_NAME" "AZURE_APP_SERVICE_NAME"
68-
"AZURE_APP_SERVICE_PLAN_NAME" "AZURE_CONTAINER_REGISTRY_NAME" "AZURE_CONTAINER_REPOSITORY_NAME"
69-
"AZURE_KEY_VAULT_NAME" "AZURE_LOG_ANALYTICS_NAME" "AZURE_OPENAI_API_VERSION"
70-
"AZURE_OPENAI_CHAT_DEPLOYMENT" "AZURE_OPENAI_EMBEDDING_DEPLOYMENT" "AZURE_OPENAI_EMBEDDING_MODEL"
71-
"AZURE_OPENAI_ENDPOINT" "AZURE_OPENAI_NAME" "AZURE_PRINCIPAL_ID" "AZURE_RESOURCE_GROUP"
72-
"AZUREAI_RESOURCE_GROUP" "AZURE_SEARCH_ENDPOINT" "AZURE_SEARCH_INDEX_SAMPLE_DATA"
73-
"AZURE_SEARCH_NAME" "AZURE_STORAGE_ACCOUNT_NAME" "PROMPTFLOW_SERVING_ENGINE" "PROMPTFLOW_WORKER_NUM")
68+
VARIABLES=(
69+
"AZUREAI_HUB_NAME"
70+
"AZUREAI_PROJECT_NAME"
71+
"AZURE_APP_INSIGHTS_NAME"
72+
"AZURE_APP_SERVICE_NAME"
73+
"AZURE_SUBSCRIPTION_ID"
74+
"AZURE_LOCATION"
75+
"AZUREAI_RESOURCE_GROUP"
76+
"AZURE_APP_SERVICE_PLAN_NAME"
77+
"AZURE_CONTAINER_REGISTRY_NAME"
78+
"AZURE_CONTAINER_REPOSITORY_NAME"
79+
"AZURE_KEY_VAULT_NAME"
80+
"AZURE_LOG_ANALYTICS_NAME"
81+
"AZURE_OPENAI_API_VERSION"
82+
"AZURE_OPENAI_CHAT_DEPLOYMENT"
83+
"AZURE_OPENAI_EMBEDDING_DEPLOYMENT"
84+
"AZURE_OPENAI_EMBEDDING_MODEL"
85+
"AZURE_OPENAI_ENDPOINT"
86+
"AZURE_OPENAI_NAME"
87+
"AZURE_PRINCIPAL_ID"
88+
"AZURE_RESOURCE_GROUP"
89+
"AZURE_SEARCH_ENDPOINT"
90+
"AZURE_SEARCH_INDEX_SAMPLE_DATA"
91+
"AZURE_SEARCH_NAME"
92+
"AZURE_STORAGE_ACCOUNT_NAME"
93+
"PROMPTFLOW_SERVING_ENGINE"
94+
"PROMPTFLOW_WORKER_NUM"
95+
)
7496
7597
manual_environment=${{ inputs.MANUAL_PROVISIONING || 'false' }}
7698
if [ "$manual_environment" = "true" ]; then
@@ -192,9 +214,18 @@ runs:
192214
fi
193215
194216
outputs:
217+
AZUREAI_HUB_NAME:
218+
description: 'Azure AI Hub Name'
219+
value: ${{ steps.setup-env-step.outputs.AZUREAI_HUB_NAME }}
195220
AZUREAI_PROJECT_NAME:
196221
description: 'Azure AI Project Name'
197222
value: ${{ steps.setup-env-step.outputs.AZUREAI_PROJECT_NAME }}
223+
AZUREAI_RESOURCE_GROUP:
224+
description: 'Azure AI Resource Group Name'
225+
value: ${{ steps.setup-env-step.outputs.AZURE_RESOURCE_GROUP }}
226+
AZURE_APP_INSIGHTS_NAME:
227+
description: 'Azure App Insights Name'
228+
value: ${{ steps.setup-env-step.outputs.AZURE_APP_INSIGHTS_NAME }}
198229
AZURE_APP_SERVICE_NAME:
199230
description: 'Azure App Service Name'
200231
value: ${{ steps.setup-env-step.outputs.AZURE_APP_SERVICE_NAME }}
@@ -207,9 +238,15 @@ outputs:
207238
AZURE_CONTAINER_REPOSITORY_NAME:
208239
description: 'Azure Container Registry Repository Name'
209240
value: ${{ steps.setup-env-step.outputs.AZURE_CONTAINER_REPOSITORY_NAME }}
241+
AZURE_KEY_VAULT_NAME:
242+
description: 'Azure Key Vault Name'
243+
value: ${{ steps.setup-env-step.outputs.AZURE_KEY_VAULT_NAME }}
210244
AZURE_LOCATION:
211245
description: 'Azure location'
212246
value: ${{ steps.setup-env-step.outputs.AZURE_LOCATION }}
247+
AZURE_LOG_ANALYTICS_NAME:
248+
description: 'Azure Log Analytics Name'
249+
value: ${{ steps.setup-env-step.outputs.AZURE_LOG_ANALYTICS_NAME }}
213250
AZURE_OPENAI_API_VERSION:
214251
description: 'Azure OpenAI API Version'
215252
value: ${{ steps.setup-env-step.outputs.AZURE_OPENAI_API_VERSION }}
@@ -254,4 +291,4 @@ outputs:
254291
value: ${{ steps.setup-env-step.outputs.PROMPTFLOW_SERVING_ENGINE }}
255292
PROMPTFLOW_WORKER_NUM:
256293
description: 'Promptflow Worker Number'
257-
value: ${{ steps.setup-env-step.outputs.PROMPTFLOW_WORKER_NUM }}
294+
value: ${{ steps.setup-env-step.outputs.PROMPTFLOW_WORKER_NUM }}

‎.github/workflows/continuous_delivery_dev.yml

+2-14
Original file line numberDiff line numberDiff line change
@@ -10,26 +10,14 @@ jobs:
1010
runs-on: ubuntu-latest
1111
environment: dev
1212
outputs:
13-
AZUREAI_PROJECT_NAME: ${{ steps.config-env.outputs.AZUREAI_PROJECT_NAME }}
1413
AZURE_APP_SERVICE_NAME: ${{ steps.config-env.outputs.AZURE_APP_SERVICE_NAME }}
1514
AZURE_APP_SERVICE_PLAN_NAME: ${{ steps.config-env.outputs.AZURE_APP_SERVICE_PLAN_NAME }}
1615
AZURE_CONTAINER_REGISTRY_NAME: ${{ steps.config-env.outputs.AZURE_CONTAINER_REGISTRY_NAME }}
1716
AZURE_CONTAINER_REPOSITORY_NAME: ${{ steps.config-env.outputs.AZURE_CONTAINER_REPOSITORY_NAME }}
1817
AZURE_LOCATION: ${{ steps.config-env.outputs.AZURE_LOCATION }}
19-
AZURE_OPENAI_API_VERSION: ${{ steps.config-env.outputs.AZURE_OPENAI_API_VERSION }}
20-
AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ steps.config-env.outputs.AZURE_OPENAI_CHAT_DEPLOYMENT }}
21-
AZURE_OPENAI_EMBEDDING_DEPLOYMENT: ${{ steps.config-env.outputs.AZURE_OPENAI_EMBEDDING_DEPLOYMENT }}
22-
AZURE_OPENAI_EMBEDDING_MODEL: ${{ steps.config-env.outputs.AZURE_OPENAI_EMBEDDING_MODEL }}
23-
AZURE_OPENAI_ENDPOINT: ${{ steps.config-env.outputs.AZURE_OPENAI_ENDPOINT }}
24-
AZURE_OPENAI_NAME: ${{ steps.config-env.outputs.AZURE_OPENAI_NAME }}
2518
AZURE_RESOURCE_GROUP: ${{ steps.config-env.outputs.AZURE_RESOURCE_GROUP }}
26-
AZUREAI_RESOURCE_GROUP: ${{ steps.config-env.outputs.AZUREAI_RESOURCE_GROUP }}
27-
AZURE_SEARCH_ENDPOINT: ${{ steps.config-env.outputs.AZURE_SEARCH_ENDPOINT }}
28-
AZURE_SEARCH_INDEX_SAMPLE_DATA: ${{ steps.config-env.outputs.AZURE_SEARCH_INDEX_SAMPLE_DATA }}
29-
AZURE_SEARCH_NAME: ${{ steps.config-env.outputs.AZURE_SEARCH_NAME }}
19+
AZUREAI_RESOURCE_GROUP: ${{ steps.config-env.outputs.AZURE_RESOURCE_GROUP }}
3020
AZURE_SUBSCRIPTION_ID: ${{ steps.config-env.outputs.AZURE_SUBSCRIPTION_ID }}
31-
PROMPTFLOW_SERVING_ENGINE: ${{ steps.config-env.outputs.PROMPTFLOW_SERVING_ENGINE }}
32-
PROMPTFLOW_WORKER_NUM: ${{ steps.config-env.outputs.PROMPTFLOW_WORKER_NUM }}
3321
steps:
3422
- uses: actions/checkout@v4
3523
- name: Provision dev environment
@@ -55,7 +43,7 @@ jobs:
5543
AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
5644
AZURE_OPENAI_NAME: ${{ vars.AZURE_OPENAI_NAME }}
5745
AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }}
58-
AZUREAI_RESOURCE_GROUP: ${{ vars.AZUREAI_RESOURCE_GROUP }}
46+
AZUREAI_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }}
5947
AZURE_SEARCH_ENDPOINT: ${{ vars.AZURE_SEARCH_ENDPOINT }}
6048
AZURE_SEARCH_INDEX_SAMPLE_DATA: "false" # It's already populated during PR
6149
AZURE_SEARCH_NAME: ${{ vars.AZURE_SEARCH_NAME }}

‎.github/workflows/continuous_delivery_prod.yml

+6-5
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,6 @@ on:
55
branches:
66
- main
77

8-
98
jobs:
109

1110
setup-env-job:
@@ -19,7 +18,7 @@ jobs:
1918
AZURE_CONTAINER_REPOSITORY_NAME: ${{ steps.config-env.outputs.AZURE_CONTAINER_REPOSITORY_NAME }}
2019
AZURE_LOCATION: ${{ steps.config-env.outputs.AZURE_LOCATION }}
2120
AZURE_RESOURCE_GROUP: ${{ steps.config-env.outputs.AZURE_RESOURCE_GROUP }}
22-
AZUREAI_RESOURCE_GROUP: ${{ steps.config-env.outputs.AZUREAI_RESOURCE_GROUP }}
21+
AZUREAI_RESOURCE_GROUP: ${{ steps.config-env.outputs.AZURE_RESOURCE_GROUP }}
2322
AZURE_SUBSCRIPTION_ID: ${{ steps.config-env.outputs.AZURE_SUBSCRIPTION_ID }}
2423
steps:
2524
- uses: actions/checkout@v4
@@ -46,7 +45,7 @@ jobs:
4645
AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
4746
AZURE_OPENAI_NAME: ${{ vars.AZURE_OPENAI_NAME }}
4847
AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }}
49-
AZUREAI_RESOURCE_GROUP: ${{ vars.AZUREAI_RESOURCE_GROUP }}
48+
AZUREAI_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }}
5049
AZURE_SEARCH_ENDPOINT: ${{ vars.AZURE_SEARCH_ENDPOINT }}
5150
AZURE_SEARCH_INDEX_SAMPLE_DATA: ${{ vars.AZURE_SEARCH_INDEX_SAMPLE_DATA }}
5251
AZURE_SEARCH_NAME: ${{ vars.AZURE_SEARCH_NAME }}
@@ -278,10 +277,12 @@ jobs:
278277
echo "🔗 [View Resources Deployed Here]($deep_link)" >> $GITHUB_STEP_SUMMARY
279278
280279
smoke-tests:
281-
needs: deploy-flow
280+
needs: [setup-env-job, deploy-flow]
282281
runs-on: ubuntu-latest
283282
environment: prod
283+
env:
284+
AZURE_APP_SERVICE_NAME: ${{needs.setup-env-job.outputs.AZURE_APP_SERVICE_NAME}}
284285
steps:
285286
- name: Smoke Tests
286287
run: |
287-
echo "Run Smoke Tests"
288+
curl https://${{AZURE_APP_SERVICE_NAME}}.azurewebsites.net/score --data '{"question":"What is the size of the moon?", "chat_history":[]}' -X POST -H "Content-Type: application/json"

‎.github/workflows/continuous_delivery_qa.yml

+93-4
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,18 @@ jobs:
1818
AZURE_CONTAINER_REPOSITORY_NAME: ${{ steps.config-env.outputs.AZURE_CONTAINER_REPOSITORY_NAME }}
1919
AZURE_LOCATION: ${{ steps.config-env.outputs.AZURE_LOCATION }}
2020
AZURE_RESOURCE_GROUP: ${{ steps.config-env.outputs.AZURE_RESOURCE_GROUP }}
21-
AZUREAI_RESOURCE_GROUP: ${{ steps.config-env.outputs.AZUREAI_RESOURCE_GROUP }}
21+
AZUREAI_RESOURCE_GROUP: ${{ steps.config-env.outputs.AZURE_RESOURCE_GROUP }}
22+
AZUREAI_PROJECT_NAME: ${{ steps.config-env.outputs.AZUREAI_PROJECT_NAME }}
2223
AZURE_SUBSCRIPTION_ID: ${{ steps.config-env.outputs.AZURE_SUBSCRIPTION_ID }}
24+
AZURE_SEARCH_ENDPOINT: ${{ steps.config-env.outputs.AZURE_SEARCH_ENDPOINT }}
25+
AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ steps.config-env.outputs.AZURE_OPENAI_CHAT_DEPLOYMENT }}
26+
AZURE_OPENAI_EMBEDDING_DEPLOYMENT: ${{steps.config-env.outputs.AZURE_OPENAI_EMBEDDING_DEPLOYMENT}}
27+
AZURE_OPENAI_EMBEDDING_MODEL: ${{steps.config-env.outputs.AZURE_OPENAI_EMBEDDING_MODEL}}
28+
AZURE_OPENAI_API_VERSION: ${{steps.config-env.outputs.AZURE_OPENAI_API_VERSION}}
29+
AZURE_OPENAI_ENDPOINT: ${{steps.config-env.outputs.AZURE_OPENAI_ENDPOINT}}
30+
AZURE_OPENAI_NAME: ${{steps.config-env.outputs.AZURE_OPENAI_NAME}}
31+
PROMPTFLOW_WORKER_NUM: ${{steps.config-env.outputs.PROMPTFLOW_WORKER_NUM}}
32+
PROMPTFLOW_SERVING_ENGINE: ${{steps.config-env.outputs.PROMPTFLOW_SERVING_ENGINE}}
2333
steps:
2434
- uses: actions/checkout@v4
2535
- name: Provision qa environment
@@ -40,12 +50,13 @@ jobs:
4050
AZURE_CONTAINER_REPOSITORY_NAME: ${{ vars.AZURE_CONTAINER_REPOSITORY_NAME }}
4151
AZURE_OPENAI_API_VERSION: ${{ vars.AZURE_OPENAI_API_VERSION }}
4252
AZURE_OPENAI_CHAT_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT }}
53+
AZURE_OPENAI_DEPLOYMENT: ${{ vars.AZURE_OPENAI_CHAT_DEPLOYMENT }}
4354
AZURE_OPENAI_EMBEDDING_DEPLOYMENT: ${{ vars.AZURE_OPENAI_EMBEDDING_DEPLOYMENT }}
4455
AZURE_OPENAI_EMBEDDING_MODEL: ${{ vars.AZURE_OPENAI_EMBEDDING_MODEL }}
4556
AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }}
4657
AZURE_OPENAI_NAME: ${{ vars.AZURE_OPENAI_NAME }}
4758
AZURE_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }}
48-
AZUREAI_RESOURCE_GROUP: ${{ vars.AZUREAI_RESOURCE_GROUP }}
59+
AZUREAI_RESOURCE_GROUP: ${{ vars.AZURE_RESOURCE_GROUP }}
4960
AZURE_SEARCH_ENDPOINT: ${{ vars.AZURE_SEARCH_ENDPOINT }}
5061
AZURE_SEARCH_INDEX_SAMPLE_DATA: ${{ vars.AZURE_SEARCH_INDEX_SAMPLE_DATA }}
5162
AZURE_SEARCH_NAME: ${{ vars.AZURE_SEARCH_NAME }}
@@ -59,13 +70,91 @@ jobs:
5970
AZURE_STORAGE_ACCOUNT_NAME: ${{ vars.AZURE_STORAGE_ACCOUNT_NAME }}
6071

6172
qa-flow-evaluation:
73+
needs: [setup-env-job]
74+
runs-on: ubuntu-latest
75+
environment: qa
76+
steps:
77+
- name: Checkout
78+
uses: actions/checkout@v4
79+
80+
- name: Setup Python
81+
uses: actions/setup-python@v5
82+
with:
83+
python-version: 3.11
84+
85+
- name: Install dependencies
86+
run: |
87+
python -m pip install --upgrade pip
88+
pip install -r requirements.txt
89+
90+
- name: Login to Azure CLI
91+
uses: azure/login@v2
92+
with:
93+
creds: ${{ secrets.AZURE_CREDENTIALS }}
94+
95+
- name: AI-Based quality evaluation
96+
env:
97+
AZURE_OPENAI_ENDPOINT: ${{needs.setup-env-job.outputs.AZURE_OPENAI_ENDPOINT}}
98+
AZURE_OPENAI_DEPLOYMENT: ${{needs.setup-env-job.outputs.AZURE_OPENAI_CHAT_DEPLOYMENT}}
99+
AZUREAI_PROJECT_NAME: ${{needs.setup-env-job.outputs.AZUREAI_PROJECT_NAME}}
100+
AZURE_OPENAI_API_VERSION: ${{needs.setup-env-job.outputs.AZURE_OPENAI_API_VERSION}}
101+
AZURE_SUBSCRIPTION_ID: ${{needs.setup-env-job.outputs.AZURE_SUBSCRIPTION_ID}}
102+
AZURE_RESOURCE_GROUP: ${{needs.setup-env-job.outputs.AZURE_RESOURCE_GROUP}}
103+
AZURE_OPENAI_NAME: ${{needs.setup-env-job.outputs.AZURE_OPENAI_NAME}}
104+
LOCATION: ${{needs.setup-env-job.outputs.AZURE_LOCATION}}
105+
AZURE_CONTAINER_REGISTRY_NAME: ${{needs.setup-env-job.outputs.AZURE_CONTAINER_REGISTRY_NAME}}
106+
AZURE_CONTAINER_REPOSITORY_NAME: ${{needs.setup-env-job.outputs.AZURE_CONTAINER_REPOSITORY_NAME}}
107+
AZURE_SEARCH_ENDPOINT: ${{needs.setup-env-job.outputs.AZURE_SEARCH_ENDPOINT}}
108+
AZURE_OPENAI_CHAT_DEPLOYMENT: ${{needs.setup-env-job.outputs.AZURE_OPENAI_CHAT_DEPLOYMENT}}
109+
AZURE_OPENAI_EMBEDDING_DEPLOYMENT: ${{needs.setup-env-job.outputs.AZURE_OPENAI_EMBEDDING_DEPLOYMENT}}
110+
AZURE_OPENAI_EMBEDDING_MODEL: ${{needs.setup-env-job.outputs.AZURE_OPENAI_EMBEDDING_MODEL}}
111+
PROMPTFLOW_WORKER_NUM: ${{needs.setup-env-job.outputs.PROMPTFLOW_WORKER_NUM}}
112+
PROMPTFLOW_SERVING_ENGINE: ${{needs.setup-env-job.outputs.PROMPTFLOW_SERVING_ENGINE}}
113+
AZURE_APP_SERVICE_NAME: ${{needs.setup-env-job.outputs.AZURE_APP_SERVICE_NAME}}
114+
AZURE_APP_SERVICE_PLAN_NAME: ${{needs.setup-env-job.outputs.AZURE_APP_SERVICE_PLAN_NAME}}
115+
run: |
116+
echo "Run AI-Based flow evaluation"
117+
echo "AZURE_RESOURCE_GROUP=${AZURE_RESOURCE_GROUP}"
118+
echo "AZURE_OPENAI_NAME=${AZURE_OPENAI_NAME}"
119+
export AZURE_OPENAI_API_KEY=$(az cognitiveservices account keys list --resource-group ${AZURE_RESOURCE_GROUP} --name ${AZURE_OPENAI_NAME} --query "key1" --output tsv)
120+
# python evaluations/flow_eval.py
121+
python evaluations/qa_quality_eval.py
122+
shell: bash
123+
124+
- name: Upload evaluation results
125+
uses: actions/upload-artifact@v4
126+
with:
127+
name: flow-eval-results
128+
path: qa_flow_quality_eval.json.json
129+
130+
qa-safety-evaluation:
62131
needs: setup-env-job
63132
runs-on: ubuntu-latest
64133
environment: qa
65134
steps:
66-
- name: Evaluate Flow
135+
- name: AI-Based safety evaluation
136+
env:
137+
AZUREAI_PROJECT_NAME: ${{needs.setup-env-job.outputs.AZUREAI_PROJECT_NAME}}
138+
AZURE_SUBSCRIPTION_ID: ${{needs.setup-env-job.outputs.AZURE_SUBSCRIPTION_ID}}
139+
AZUREAI_RESOURCE_GROUP: ${{needs.setup-env-job.outputs.AZURE_RESOURCE_GROUP}}
140+
AZURE_LOCATION: ${{needs.setup-env-job.outputs.AZURE_LOCATION}}
141+
PREFIX: ${{ github.sha }}
67142
run: |
68-
echo "Run QA AI-Based Flow evaluation"
143+
echo "Run Automated Adversarial Testing"
144+
export PYTHONPATH=./src:$PYTHONPATH
145+
python evaluations/safety_eval.py
146+
147+
- name: Upload evaluation results
148+
uses: actions/upload-artifact@v4
149+
with:
150+
name: adversarial-test
151+
path: adversarial_test.json
152+
153+
- name: Upload evaluation results
154+
uses: actions/upload-artifact@v4
155+
with:
156+
name: adversarial-test-w-jailbreak
157+
path: adversarial_test_w_jailbreak.json
69158

70159
integration-testing:
71160
needs: [setup-env-job, qa-flow-evaluation]

0 commit comments

Comments
 (0)
Please sign in to comment.