diff --git a/.github/pipelines/compliance_check.yml b/.github/pipelines/compliance_check.yml new file mode 100644 index 00000000000..b1fe4b3713b --- /dev/null +++ b/.github/pipelines/compliance_check.yml @@ -0,0 +1,53 @@ +# Pipeline link: https://dev.azure.com/msdata/Vienna/_build?definitionId=26179&_a=summary +parameters: + - name: policyCulture + displayName: "Policy Culture" + type: string + # The culture used to run policy check scan, can be region codes separated by comma, e.g. 'en-US,de-DE' + default: 'en-US' + +name: $(BuildDefinitionName)_$(Date:yyyyMMdd)$(Rev:.r) # Configure run or build numbers + +variables: + - name: sourceLocation + value: $(System.DefaultWorkingDirectory) + +trigger: + - main + - releases/* + +pool: + vmImage: windows-latest + +steps: +- checkout: self + +# https://eng.ms/docs/microsoft-security/cloud-ecosystem-security/azure-security/cloudai-security-fundamentals-engineering/security-integration/guardian-wiki/sdl-azdo-extension/PoliCheck-build-task +- task: PoliCheck@2 + inputs: + targetType: 'F' + targetArgument: '$(sourceLocation)' + optionsPE: '1' + optionsUEPATH: '$(sourceLocation)/scripts/compliance-check/user_exclusion.xml' + result: '$(sourceLocation)/scripts/compliance-check/result.tsv' + optionsXCLASS: 'Geopolitical' + +- task: PowerShell@2 + inputs: + targetType: 'filePath' + filePath: '$(sourceLocation)/scripts/compliance-check/Check-PolicheckScan.ps1' + arguments: > + -policheckResult $(sourceLocation)/scripts/compliance-check/result.tsv + displayName: 'Check result' + +- task: PublishPipelineArtifact@1 + condition: failed() + inputs: + targetPath: '$(sourceLocation)/scripts/compliance-check/result.tsv' + artifactName: 'compliance-check-result' + publishLocation: 'pipeline' + +- task: CredScan@3 + displayName: 'CredScan' + inputs: + scanFolder: '$(sourceLocation)' diff --git a/docs/cloud/azureai/quick-start.md b/docs/cloud/azureai/quick-start.md index 710eb8f2a80..e0a7f0486fa 100644 --- a/docs/cloud/azureai/quick-start.md +++ b/docs/cloud/azureai/quick-start.md @@ -17,7 +17,7 @@ Benefits of use Azure AI comparison to just run locally: 3. A python environment, `python=3.9` is recommended. 4. Install `promptflow` with extra dependencies and `promptflow-tools`. ```sh -pip install promptflow[azure] promptflow-tools --extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ +pip install promptflow[azure] promptflow-tools ``` 5. Get the sample flows. - Get access to the flow sample repository. diff --git a/docs/how-to-guides/quick-start.md b/docs/how-to-guides/quick-start.md index 711e49a56a2..a8290fdee33 100644 --- a/docs/how-to-guides/quick-start.md +++ b/docs/how-to-guides/quick-start.md @@ -11,7 +11,7 @@ This guide will walk you through the main user journey of prompt flow code-first 1. A python environment, `python=3.9` is recommended. 2. Install `promptflow` and `promptflow-tools`. ```sh -pip install promptflow promptflow-tools --extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ +pip install promptflow promptflow-tools ``` 3. Get the sample flows. - Get access to the flow sample repository. diff --git a/examples/connections/requirements.txt b/examples/connections/requirements.txt index a3fae371d12..5f42f5b484f 100644 --- a/examples/connections/requirements.txt +++ b/examples/connections/requirements.txt @@ -1,4 +1,3 @@ ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ promptflow promptflow-tools python-dotenv diff --git a/examples/flows/chat/basic-chat/requirements.txt b/examples/flows/chat/basic-chat/requirements.txt index 16306751554..34d068f5f1c 100644 --- a/examples/flows/chat/basic-chat/requirements.txt +++ b/examples/flows/chat/basic-chat/requirements.txt @@ -1,3 +1,2 @@ ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ promptflow promptflow-tools \ No newline at end of file diff --git a/examples/flows/chat/chat-with-pdf/requirements.txt b/examples/flows/chat/chat-with-pdf/requirements.txt index 6234be00750..904344aa525 100644 --- a/examples/flows/chat/chat-with-pdf/requirements.txt +++ b/examples/flows/chat/chat-with-pdf/requirements.txt @@ -4,6 +4,5 @@ openai jinja2 python-dotenv tiktoken ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ promptflow[azure] promptflow-tools \ No newline at end of file diff --git a/examples/flows/chat/chat-with-wikipedia/requirements.txt b/examples/flows/chat/chat-with-wikipedia/requirements.txt index fda053472d8..a7db410630e 100644 --- a/examples/flows/chat/chat-with-wikipedia/requirements.txt +++ b/examples/flows/chat/chat-with-wikipedia/requirements.txt @@ -1,4 +1,3 @@ ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ promptflow promptflow-tools bs4 \ No newline at end of file diff --git a/examples/flows/evaluation/eval-basic/requirements.txt b/examples/flows/evaluation/eval-basic/requirements.txt index 16306751554..34d068f5f1c 100644 --- a/examples/flows/evaluation/eval-basic/requirements.txt +++ b/examples/flows/evaluation/eval-basic/requirements.txt @@ -1,3 +1,2 @@ ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ promptflow promptflow-tools \ No newline at end of file diff --git a/examples/flows/evaluation/eval-classification-accuracy/requirements.txt b/examples/flows/evaluation/eval-classification-accuracy/requirements.txt index 16306751554..34d068f5f1c 100644 --- a/examples/flows/evaluation/eval-classification-accuracy/requirements.txt +++ b/examples/flows/evaluation/eval-classification-accuracy/requirements.txt @@ -1,3 +1,2 @@ ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ promptflow promptflow-tools \ No newline at end of file diff --git a/examples/flows/evaluation/eval-entity-match-rate/requirements.txt b/examples/flows/evaluation/eval-entity-match-rate/requirements.txt index 16306751554..34d068f5f1c 100644 --- a/examples/flows/evaluation/eval-entity-match-rate/requirements.txt +++ b/examples/flows/evaluation/eval-entity-match-rate/requirements.txt @@ -1,3 +1,2 @@ ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ promptflow promptflow-tools \ No newline at end of file diff --git a/examples/flows/evaluation/eval-groundedness/flow.dag.yaml b/examples/flows/evaluation/eval-groundedness/flow.dag.yaml index 429c9ce9286..b62a6a240fb 100644 --- a/examples/flows/evaluation/eval-groundedness/flow.dag.yaml +++ b/examples/flows/evaluation/eval-groundedness/flow.dag.yaml @@ -38,7 +38,7 @@ inputs: Paraphrasing (IWP2005) . William Fedus, Ian Goodfellow, and Andrew M Dai. 2018. Maskgan: Better text generation via \ufb01lling in the.arXiv preprint arXiv:1801.07736 . Dan Hendrycks and Kevin Gimpel. 2016. Bridging - nonlinearities and stochastic regularizers with gaus- sian error linear + nonlinearities and stochastic regularizers with gaussian error linear units. CoRR , abs\/1606.08415. Felix Hill, Kyunghyun Cho, and Anna Korhonen. 2016. Learning distributed representations of sentences from unlabelled data. In Proceedings of the 2016 Conference of the North diff --git a/examples/flows/evaluation/eval-groundedness/requirements.txt b/examples/flows/evaluation/eval-groundedness/requirements.txt index 16306751554..34d068f5f1c 100644 --- a/examples/flows/evaluation/eval-groundedness/requirements.txt +++ b/examples/flows/evaluation/eval-groundedness/requirements.txt @@ -1,3 +1,2 @@ ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ promptflow promptflow-tools \ No newline at end of file diff --git a/examples/flows/evaluation/eval-perceived-intelligence/flow.dag.yaml b/examples/flows/evaluation/eval-perceived-intelligence/flow.dag.yaml index 8b36f2e832c..4bb014b2d94 100644 --- a/examples/flows/evaluation/eval-perceived-intelligence/flow.dag.yaml +++ b/examples/flows/evaluation/eval-perceived-intelligence/flow.dag.yaml @@ -38,7 +38,7 @@ inputs: Paraphrasing (IWP2005) . William Fedus, Ian Goodfellow, and Andrew M Dai. 2018. Maskgan: Better text generation via \ufb01lling in the.arXiv preprint arXiv:1801.07736 . Dan Hendrycks and Kevin Gimpel. 2016. Bridging - nonlinearities and stochastic regularizers with gaus- sian error linear + nonlinearities and stochastic regularizers with gaussian error linear units. CoRR , abs\/1606.08415. Felix Hill, Kyunghyun Cho, and Anna Korhonen. 2016. Learning distributed representations of sentences from unlabelled data. In Proceedings of the 2016 Conference of the North diff --git a/examples/flows/evaluation/eval-perceived-intelligence/requirements.txt b/examples/flows/evaluation/eval-perceived-intelligence/requirements.txt index 16306751554..34d068f5f1c 100644 --- a/examples/flows/evaluation/eval-perceived-intelligence/requirements.txt +++ b/examples/flows/evaluation/eval-perceived-intelligence/requirements.txt @@ -1,3 +1,2 @@ ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ promptflow promptflow-tools \ No newline at end of file diff --git a/examples/flows/standard/autonomous-agent/requirements.txt b/examples/flows/standard/autonomous-agent/requirements.txt index 88941ca6838..9abb95f309d 100644 --- a/examples/flows/standard/autonomous-agent/requirements.txt +++ b/examples/flows/standard/autonomous-agent/requirements.txt @@ -1,4 +1,3 @@ ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ promptflow promptflow-tools tiktoken diff --git a/examples/flows/standard/basic-with-builtin-llm/requirements.txt b/examples/flows/standard/basic-with-builtin-llm/requirements.txt index 4749440903e..c6b72c6ec33 100644 --- a/examples/flows/standard/basic-with-builtin-llm/requirements.txt +++ b/examples/flows/standard/basic-with-builtin-llm/requirements.txt @@ -1,4 +1,3 @@ ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ promptflow promptflow-tools python-dotenv \ No newline at end of file diff --git a/examples/flows/standard/basic-with-connection/requirements.txt b/examples/flows/standard/basic-with-connection/requirements.txt index 25f130ac9e2..35c1b55b32e 100644 --- a/examples/flows/standard/basic-with-connection/requirements.txt +++ b/examples/flows/standard/basic-with-connection/requirements.txt @@ -1,4 +1,3 @@ ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ promptflow[azure] promptflow-tools python-dotenv \ No newline at end of file diff --git a/examples/flows/standard/basic/requirements.txt b/examples/flows/standard/basic/requirements.txt index 25f130ac9e2..35c1b55b32e 100644 --- a/examples/flows/standard/basic/requirements.txt +++ b/examples/flows/standard/basic/requirements.txt @@ -1,4 +1,3 @@ ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ promptflow[azure] promptflow-tools python-dotenv \ No newline at end of file diff --git a/examples/flows/standard/customer-intent-extraction/requirements.txt b/examples/flows/standard/customer-intent-extraction/requirements.txt index 18e9929aa30..87e94a1fc7a 100644 --- a/examples/flows/standard/customer-intent-extraction/requirements.txt +++ b/examples/flows/standard/customer-intent-extraction/requirements.txt @@ -1,4 +1,3 @@ ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ promptflow promptflow-tools python-dotenv diff --git a/examples/flows/standard/flow-with-additional-includes/requirements.txt b/examples/flows/standard/flow-with-additional-includes/requirements.txt index 9e5b3b332ff..ccef8cfd3cc 100644 --- a/examples/flows/standard/flow-with-additional-includes/requirements.txt +++ b/examples/flows/standard/flow-with-additional-includes/requirements.txt @@ -1,4 +1,3 @@ ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ promptflow[azure] promptflow-tools bs4 \ No newline at end of file diff --git a/examples/flows/standard/flow-with-symlinks/requirements.txt b/examples/flows/standard/flow-with-symlinks/requirements.txt index 9e5b3b332ff..ccef8cfd3cc 100644 --- a/examples/flows/standard/flow-with-symlinks/requirements.txt +++ b/examples/flows/standard/flow-with-symlinks/requirements.txt @@ -1,4 +1,3 @@ ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ promptflow[azure] promptflow-tools bs4 \ No newline at end of file diff --git a/examples/flows/standard/named-entity-recognition/requirements.txt b/examples/flows/standard/named-entity-recognition/requirements.txt index 16306751554..34d068f5f1c 100644 --- a/examples/flows/standard/named-entity-recognition/requirements.txt +++ b/examples/flows/standard/named-entity-recognition/requirements.txt @@ -1,3 +1,2 @@ ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ promptflow promptflow-tools \ No newline at end of file diff --git a/examples/flows/standard/web-classification/requirements.txt b/examples/flows/standard/web-classification/requirements.txt index 9e5b3b332ff..ccef8cfd3cc 100644 --- a/examples/flows/standard/web-classification/requirements.txt +++ b/examples/flows/standard/web-classification/requirements.txt @@ -1,4 +1,3 @@ ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ promptflow[azure] promptflow-tools bs4 \ No newline at end of file diff --git a/examples/requirements.txt b/examples/requirements.txt index fdd06ef8581..971319a0d7e 100644 --- a/examples/requirements.txt +++ b/examples/requirements.txt @@ -1,6 +1,4 @@ -# remove when we publish to pypi ---extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ -promptflow[azure]==0.0.103036498 +promptflow[azure] promptflow-tools==0.1.0.b5 python-dotenv bs4 diff --git a/scripts/compliance-check/Check-PolicheckScan.ps1 b/scripts/compliance-check/Check-PolicheckScan.ps1 new file mode 100644 index 00000000000..41843f0a8a8 --- /dev/null +++ b/scripts/compliance-check/Check-PolicheckScan.ps1 @@ -0,0 +1,24 @@ +# Copyright (C) Microsoft Corporation. All rights reserved. + +<# + .SYNOPSIS + Check Policheck Scan result. + .DESCRIPTION + Helper script to check the Policheck result. + If there is policheck failure, show the error and throw exception. +#> + +[CmdLetbinding()] +param ( +[string]$policheckResult, +[string]$raiseError = $true +) + +$result = Get-Content -Path $policheckResult | Measure-Object -Line; +Write-Host("Number of errors found in this scan: " + ($result.Lines - 1)); +if ($raiseError -and ($result.Lines -gt 1)) +{ +Get-Content -Path $policheckResult; +throw "Policheck scan completed successfully but there are issues to fix."; +} +# Read-Host "Press enter to finish the process and close this window"; diff --git a/scripts/compliance-check/user_exclusion.xml b/scripts/compliance-check/user_exclusion.xml new file mode 100644 index 00000000000..f959304f804 --- /dev/null +++ b/scripts/compliance-check/user_exclusion.xml @@ -0,0 +1,5 @@ + + + + SRC\PROMPTFLOW\PROMPTFLOW\_SDK\_SERVING\STATIC\INDEX-AF571BFB.JS + diff --git a/src/promptflow/promptflow/_sdk/data/docker/Dockerfile.jinja2 b/src/promptflow/promptflow/_sdk/data/docker/Dockerfile.jinja2 index 932df27af50..75a2d05dec0 100644 --- a/src/promptflow/promptflow/_sdk/data/docker/Dockerfile.jinja2 +++ b/src/promptflow/promptflow/_sdk/data/docker/Dockerfile.jinja2 @@ -30,7 +30,7 @@ RUN conda create -n {{env.conda_env_name}} python=3.9.16 pip=23.0.1 -q -y && \ {% else %} pip install promptflow \ {% endif %} - promptflow-tools --extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ && \ + promptflow-tools && \ {% endif %} conda run -n {{env.conda_env_name}} pip install keyrings.alt && \ conda run -n {{env.conda_env_name}} pip cache purge && \