From f0fd77ef43474bad574f3c691cd2b1781407f7a0 Mon Sep 17 00:00:00 2001 From: crazygao Date: Wed, 30 Aug 2023 17:23:43 +0800 Subject: [PATCH] Testing matrix support for python version 3.9/3.10, linux/mac in ci pipeline (#197) # Description Testing matrix support for sdk cli tests, windows still need more time to pass. Bypass some mac tests on Heyi's suggestion. # All Promptflow Contribution checklist: - [x] **The pull request does not introduce [breaking changes]** - [ ] **CHANGELOG is updated for new features, bug fixes or other significant changes.** - [x] **I have read the [contribution guidelines](../CONTRIBUTING.md).** ## General Guidelines and Best Practices - [x] Title of the pull request is clear and informative. - [x] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines - [x] Pull request includes test coverage for the included changes. --- .github/CODEOWNERS | 2 +- .github/ISSUE_TEMPLATE/bug_report.md | 2 +- .github/PULL_REQUEST_TEMPLATE.md | 2 +- .../step_create_conda_environment/action.yml | 8 ++- .../actions/step_generate_configs/action.yml | 2 +- .../actions/step_initialize_build/action.yml | 13 ---- .../step_publish_test_results/action.yml | 40 +++++++++++ .github/actions/step_sdk_setup/action.yml | 2 +- .github/labeler.yml | 2 +- .../promptflow-executor-e2e-test.yml | 69 +++++++++++------- .../promptflow-executor-unit-test.yml | 67 ++++++++++++------ .github/workflows/promptflow-sdk-cli-test.yml | 70 ++++++++++++------- README.md | 10 +-- SECURITY.md | 2 +- SUPPORT.md | 8 +-- scripts/building/release-env.yml | 1 - src/promptflow/CHANGELOG.md | 4 +- src/promptflow/README.md | 6 +- .../promptflow/_core/tools_manager.py | 4 +- .../promptflow/_sdk/_orm/__init__.py | 3 +- src/promptflow/promptflow/_sdk/_orm/retry.py | 2 +- .../promptflow/_sdk/entities/_connection.py | 9 +++ .../promptflow/azure/_constants/__init__.py | 9 +-- .../promptflow/azure/_constants/_flow.py | 3 +- .../promptflow/azure/_entities/_flow.py | 11 +-- .../promptflow/azure/_load_functions.py | 1 - .../promptflow/azure/_ml/__init__.py | 5 +- .../promptflow/azure/_schemas/_flow_schema.py | 2 +- .../promptflow/azure/_utils/_url_utils.py | 12 ++-- .../promptflow/azure/_utils/gerneral.py | 2 +- .../operations/_fileshare_storeage_helper.py | 36 +++++----- src/promptflow/promptflow/contracts/flow.py | 5 +- .../executor/_line_execution_process_pool.py | 1 + .../promptflow/storage/run_records.py | 4 +- .../e2etests/test_executor_timeout.py | 13 +++- .../executor/e2etests/test_package_tool.py | 13 +++- .../tests/executor/e2etests/test_telemetry.py | 5 +- .../unittests/_utils/test_thread_utils.py | 2 + .../unittests/executor/test_flow_executor.py | 2 +- .../unittests/executor/test_flow_validator.py | 7 +- 40 files changed, 288 insertions(+), 173 deletions(-) delete mode 100644 .github/actions/step_initialize_build/action.yml create mode 100644 .github/actions/step_publish_test_results/action.yml diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index de3d188b01b..730c125512e 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +1 @@ -* @microsoft/prompt-flow-approvers \ No newline at end of file +* @microsoft/prompt-flow-approvers diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index 6cb54a19fae..28e4814b505 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -12,7 +12,7 @@ A clear and concise description of the bug. **How To Reproduce the bug** Steps to reproduce the behavior, how frequent can you experience the bug: -1. +1. **Expected behavior** A clear and concise description of what you expected to happen. diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 1d880b03139..fefe019b9bd 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -12,4 +12,4 @@ Please add an informative description that covers that changes made by the pull - [ ] There are a small number of commits, each of which have an informative message. This means that previously merged commits do not appear in the history of the PR. For more information on cleaning up the commits in your PR, [see this page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md). ### Testing Guidelines -- [ ] Pull request includes test coverage for the included changes. \ No newline at end of file +- [ ] Pull request includes test coverage for the included changes. diff --git a/.github/actions/step_create_conda_environment/action.yml b/.github/actions/step_create_conda_environment/action.yml index 878ba019bda..5e6ac08634a 100644 --- a/.github/actions/step_create_conda_environment/action.yml +++ b/.github/actions/step_create_conda_environment/action.yml @@ -4,6 +4,10 @@ inputs: required: false default: "scripts/building/release-env.yml" type: string + pythonVersion: + required: false + default: "3.9" + type: string runs: using: composite steps: @@ -13,11 +17,11 @@ runs: miniconda-version: "latest" activate-environment: release-env environment-file: ${{ inputs.condaEnvironmentFilePath }} - python-version: 3.9 + python-version: ${{ inputs.pythonVersion }} auto-activate-base: false auto-update-conda: true - run: | conda info conda list python --version - shell: bash -el {0} \ No newline at end of file + shell: bash -el {0} diff --git a/.github/actions/step_generate_configs/action.yml b/.github/actions/step_generate_configs/action.yml index da20f0db326..3cdec35e044 100644 --- a/.github/actions/step_generate_configs/action.yml +++ b/.github/actions/step_generate_configs/action.yml @@ -15,4 +15,4 @@ runs: conda activate release-env echo "Generating connection config file..." python3 ./scripts/building/generate_connection_config.py \ - --target_folder ${{ inputs.targetFolder }} \ No newline at end of file + --target_folder ${{ inputs.targetFolder }} diff --git a/.github/actions/step_initialize_build/action.yml b/.github/actions/step_initialize_build/action.yml deleted file mode 100644 index ca5f6d7c11a..00000000000 --- a/.github/actions/step_initialize_build/action.yml +++ /dev/null @@ -1,13 +0,0 @@ -name: pipelines_templates_steps_template_initialize_build_step -inputs: - condaEnvironmentFilePath: - required: false - default: "scripts/building/release-env.yml" - type: string -runs: - using: composite - steps: - - uses: "./.github/actions/pipelines_templates_steps_template_display_environment_variables_step" - - uses: "./.github/actions/pipelines_templates_steps_template_create_conda_environment_step" - with: - condaEnvironmentFilePath: ${{ inputs.condaEnvironmentFilePath }} \ No newline at end of file diff --git a/.github/actions/step_publish_test_results/action.yml b/.github/actions/step_publish_test_results/action.yml new file mode 100644 index 00000000000..38e3cfd8395 --- /dev/null +++ b/.github/actions/step_publish_test_results/action.yml @@ -0,0 +1,40 @@ +name: step_publish_test_results +inputs: + osVersion: + required: false + default: "ubuntu-latest" + type: string + pythonVersion: + required: false + default: "3.9" + type: string + coverageThreshold: + required: false + default: "0.3" + type: string + token: + description: 'A Github PAT' + required: true +runs: + using: composite + steps: + - name: Download Artifacts + uses: actions/download-artifact@v3 + with: + path: artifacts + - name: Display and Set Environment Variables + run: env | sort >> $GITHUB_OUTPUT + shell: bash -el {0} + id: display_env + - name: Publish Test Results + uses: EnricoMi/publish-unit-test-result-action@v2 + with: + check_name: "Executor Unit Test Result [${{ steps.display_env.outputs.GITHUB_HEAD_REF }}](https://github.com/microsoft/promptflow/actions/workflows/promptflow-executor-unit-test.yml?query=branch:${{ steps.display_env.outputs.GITHUB_HEAD_REF }}++)" + comment_title: "Executor Unit Test Result [${{ steps.display_env.outputs.GITHUB_HEAD_REF }}](https://github.com/microsoft/promptflow/actions/workflows/promptflow-executor-unit-test.yml?query=branch:${{ steps.display_env.outputs.GITHUB_HEAD_REF }}++)" + files: "artifacts/**/test-*.xml" + - name: Generate Coverage Report + uses: orgoro/coverage@v3.1 + with: + coverageFile: "artifacts/Test Results (Python ${{ inputs.pythonVersion }}) (OS ${{ inputs.osVersion }})/coverage.xml" + token: ${{ inputs.token }} + thresholdAll: ${{ inputs.coverageThreshold }} diff --git a/.github/actions/step_sdk_setup/action.yml b/.github/actions/step_sdk_setup/action.yml index 6a1876761b7..a00406f0e37 100644 --- a/.github/actions/step_sdk_setup/action.yml +++ b/.github/actions/step_sdk_setup/action.yml @@ -75,4 +75,4 @@ runs: pip install './dist/promptflow_sdk-0.0.1-py3-none-any.whl[azure]' echo "########### pip freeze ###########" pip freeze - working-directory: ${{ inputs.scriptPath }} \ No newline at end of file + working-directory: ${{ inputs.scriptPath }} diff --git a/.github/labeler.yml b/.github/labeler.yml index 6cfb6d455e1..2ad2c8869c2 100644 --- a/.github/labeler.yml +++ b/.github/labeler.yml @@ -13,4 +13,4 @@ promptflow-tools: - src/promptflow-tools/** fundamental: - scripts/** -- .github/** \ No newline at end of file +- .github/** diff --git a/.github/workflows/promptflow-executor-e2e-test.yml b/.github/workflows/promptflow-executor-e2e-test.yml index e1f8151e9f8..e8ec79523f3 100644 --- a/.github/workflows/promptflow-executor-e2e-test.yml +++ b/.github/workflows/promptflow-executor-e2e-test.yml @@ -1,5 +1,7 @@ name: promptflow-executor-e2e-test on: + schedule: + - cron: "40 20 * * *" # Every day starting at 4:40 BJT pull_request: branches: [ main ] paths: @@ -15,20 +17,31 @@ env: testWorkingDirectory: ${{ github.workspace }}/src/promptflow jobs: executor_e2e_tests: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} steps: - name: checkout uses: actions/checkout@v3 - name: Display and Set Environment Variables - run: env | sort >> $GITHUB_OUTPUT - shell: bash -el {0} + run: | + if [ "ubuntu-latest" == "${{ matrix.os }}" ]; then + export pyVersion="3.9"; + elif [ "macos-latest" == "${{ matrix.os }}" ]; then + export pyVersion="3.10"; + else + echo "Unsupported OS: ${{ matrix.os }}"; + exit 1; + fi + env | sort >> $GITHUB_OUTPUT id: display_env - - name: Conda Setup + shell: bash -el {0} + - name: Conda Setup - ${{ matrix.os }} - Python Version ${{ steps.display_env.outputs.pyVersion }} uses: "./.github/actions/step_create_conda_environment" - - run: | - echo ${{ env.packageSetupType }} - echo ${{ env.testWorkingDirectory }} - shell: bash -l {0} + with: + pythonVersion: ${{ steps.display_env.outputs.pyVersion }} - name: Build wheel uses: "./.github/actions/step_sdk_setup" with: @@ -42,6 +55,9 @@ jobs: uses: "./.github/actions/step_generate_configs" with: targetFolder: ${{ env.testWorkingDirectory }} + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v1 + id: cpu-cores - name: Run Coverage Test shell: bash -l {0} working-directory: ${{ github.workspace }} @@ -57,29 +73,34 @@ jobs: -t ${{ github.workspace }}/src/promptflow/tests/executor/e2etests \ -l eastus \ -m "all" \ - -n 4 \ + -n ${{ steps.cpu-cores.outputs.count }} \ --coverage-config ${{ github.workspace }}/src/promptflow/tests/executor/.coveragerc - name: Upload Test Results if: always() uses: actions/upload-artifact@v3 with: - name: pytest-results + name: Test Results (Python ${{ steps.display_env.outputs.pyVersion }}) (OS ${{ matrix.os }}) path: | ${{ github.workspace }}/*.xml ${{ github.workspace }}/htmlcov/ - # Use always() to always run this step to publish test results when there are test failures + publish-test-results: + name: "Publish Tests Results" + needs: executor_e2e_tests + runs-on: ubuntu-latest + permissions: + checks: write + pull-requests: write + contents: read + issues: read + if: always() + + steps: + - name: checkout + uses: actions/checkout@v3 - name: Publish Test Results - if: always() - uses: EnricoMi/publish-unit-test-result-action/composite@v2 - with: - check_name: "Executor E2E Test Result [${{ steps.display_env.outputs.GITHUB_HEAD_REF }}](https://github.com/microsoft/promptflow/actions/workflows/promptflow-executor-e2e-test.yml?query=branch:${{ steps.display_env.outputs.GITHUB_HEAD_REF }}++)" - comment_title: "Executor E2E Test Result [${{ steps.display_env.outputs.GITHUB_HEAD_REF }}](https://github.com/microsoft/promptflow/actions/workflows/promptflow-executor-e2e-test.yml?query=branch:${{ steps.display_env.outputs.GITHUB_HEAD_REF }}++)" - files: | - ${{ github.workspace }}/test-results.xml - - name: Generate Coverage Report - if: (success() || failure()) - uses: orgoro/coverage@v3.1 + uses: "./.github/actions/step_publish_test_results" with: - coverageFile: ${{ github.workspace }}/coverage.xml - token: ${{ secrets.GITHUB_TOKEN }} - thresholdAll: 0.55 + osVersion: ubuntu-latest + pythonVersion: 3.9 + coverageThreshold: 0.55 + token: ${{ secrets.GITHUB_TOKEN }} \ No newline at end of file diff --git a/.github/workflows/promptflow-executor-unit-test.yml b/.github/workflows/promptflow-executor-unit-test.yml index fdede59a3ed..3d4046a30c4 100644 --- a/.github/workflows/promptflow-executor-unit-test.yml +++ b/.github/workflows/promptflow-executor-unit-test.yml @@ -1,5 +1,7 @@ name: promptflow-executor-unit-test on: + schedule: + - cron: "40 19 * * *" # Every day starting at 3:40 BJT pull_request: branches: [ main ] paths: @@ -15,20 +17,31 @@ env: testWorkingDirectory: ${{ github.workspace }}/src/promptflow jobs: executor_unit_tests: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} steps: - name: checkout uses: actions/checkout@v3 - name: Display and Set Environment Variables - run: env | sort >> $GITHUB_OUTPUT - shell: bash -el {0} + run: | + if [ "ubuntu-latest" == "${{ matrix.os }}" ]; then + export pyVersion="3.9"; + elif [ "macos-latest" == "${{ matrix.os }}" ]; then + export pyVersion="3.10"; + else + echo "Unsupported OS: ${{ matrix.os }}"; + exit 1; + fi + env | sort >> $GITHUB_OUTPUT id: display_env - - name: Conda Setup + shell: bash -el {0} + - name: Conda Setup - ${{ matrix.os }} - Python Version ${{ steps.display_env.outputs.pyVersion }} uses: "./.github/actions/step_create_conda_environment" - - run: | - echo ${{ env.packageSetupType }} - echo ${{ env.testWorkingDirectory }} - shell: bash -l {0} + with: + pythonVersion: ${{ steps.display_env.outputs.pyVersion }} - name: Build wheel uses: "./.github/actions/step_sdk_setup" with: @@ -42,6 +55,9 @@ jobs: uses: "./.github/actions/step_generate_configs" with: targetFolder: ${{ env.testWorkingDirectory }} + - name: Get number of CPU cores + uses: SimenB/github-actions-cpu-cores@v1 + id: cpu-cores - name: Run Coverage Test shell: bash -l {0} working-directory: ${{ github.workspace }} @@ -56,29 +72,34 @@ jobs: -t ${{ github.workspace }}/src/promptflow/tests/executor/unittests \ -l eastus \ -m "all" \ - -n 4 \ + -n ${{ steps.cpu-cores.outputs.count }} \ --coverage-config ${{ github.workspace }}/src/promptflow/tests/executor/.coveragerc - name: Upload Test Results if: always() uses: actions/upload-artifact@v3 with: - name: pytest-results + name: Test Results (Python ${{ steps.display_env.outputs.pyVersion }}) (OS ${{ matrix.os }}) path: | ${{ github.workspace }}/*.xml ${{ github.workspace }}/htmlcov/ - # Use always() to always run this step to publish test results when there are test failures + publish-test-results: + name: "Publish Tests Results" + needs: executor_unit_tests + runs-on: ubuntu-latest + permissions: + checks: write + pull-requests: write + contents: read + issues: read + if: always() + + steps: + - name: checkout + uses: actions/checkout@v3 - name: Publish Test Results - if: always() - uses: EnricoMi/publish-unit-test-result-action/composite@v2 - with: - check_name: "Executor Unit Test Result [${{ steps.display_env.outputs.GITHUB_HEAD_REF }}](https://github.com/microsoft/promptflow/actions/workflows/promptflow-executor-unit-test.yml?query=branch:${{ steps.display_env.outputs.GITHUB_HEAD_REF }}++)" - comment_title: "Executor Unit Test Result [${{ steps.display_env.outputs.GITHUB_HEAD_REF }}](https://github.com/microsoft/promptflow/actions/workflows/promptflow-executor-unit-test.yml?query=branch:${{ steps.display_env.outputs.GITHUB_HEAD_REF }}++)" - files: | - ${{ github.workspace }}/test-results.xml - - name: Generate Coverage Report - if: (success() || failure()) - uses: orgoro/coverage@v3.1 + uses: "./.github/actions/step_publish_test_results" with: - coverageFile: ${{ github.workspace }}/coverage.xml + osVersion: ubuntu-latest + pythonVersion: 3.9 + coverageThreshold: 0.3 token: ${{ secrets.GITHUB_TOKEN }} - thresholdAll: 0.3 diff --git a/.github/workflows/promptflow-sdk-cli-test.yml b/.github/workflows/promptflow-sdk-cli-test.yml index 0c8441c55c3..3be58f2a377 100644 --- a/.github/workflows/promptflow-sdk-cli-test.yml +++ b/.github/workflows/promptflow-sdk-cli-test.yml @@ -1,5 +1,7 @@ name: promptflow-sdk-cli-test on: + schedule: + - cron: "40 18 * * *" # Every day starting at 2:40 BJT pull_request: branches: [ main ] paths: @@ -15,20 +17,31 @@ env: testWorkingDirectory: src/promptflow jobs: sdk_cli_tests: - runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} steps: - name: checkout uses: actions/checkout@v3 - name: Display and Set Environment Variables - run: env | sort >> $GITHUB_OUTPUT - shell: bash -el {0} + run: | + if [ "ubuntu-latest" == "${{ matrix.os }}" ]; then + export pyVersion="3.9"; + elif [ "macos-latest" == "${{ matrix.os }}" ]; then + export pyVersion="3.10"; + else + echo "Unsupported OS: ${{ matrix.os }}"; + exit 1; + fi + env | sort >> $GITHUB_OUTPUT id: display_env - - name: Conda Setup + shell: bash -el {0} + - name: Conda Setup - ${{ matrix.os }} - Python Version ${{ steps.display_env.outputs.pyVersion }} uses: "./.github/actions/step_create_conda_environment" - - run: | - echo ${{ env.packageSetupType }} - echo ${{ env.testWorkingDirectory }} - shell: bash -l {0} + with: + pythonVersion: ${{ steps.display_env.outputs.pyVersion }} - name: Build wheel uses: "./.github/actions/step_sdk_setup" with: @@ -54,8 +67,6 @@ jobs: az account show conda activate release-env export PYTHONPATH=${{ github.workspace }}/src/promptflow - echo "aaa=bbb" > ${{ github.workspace }}/src/promptflow/tests/test_configs/connections/.env - echo "ccc=ddd" >> ${{ github.workspace }}/src/promptflow/tests/test_configs/connections/.env python "../../scripts/building/run_coverage_tests.py" \ -p promptflow \ -t ${{ github.workspace }}/src/promptflow/tests/sdk_cli_azure_test ${{ github.workspace }}/src/promptflow/tests/sdk_cli_test \ @@ -67,23 +78,28 @@ jobs: if: always() uses: actions/upload-artifact@v3 with: - name: pytest-results - path: ${{ env.testWorkingDirectory }}/*.xml - # Use always() to always run this step to publish test results when there are test failures + name: Test Results (Python ${{ steps.display_env.outputs.pyVersion }}) (OS ${{ matrix.os }}) + path: | + ${{ env.testWorkingDirectory }}/*.xml + ${{ env.testWorkingDirectory }}/htmlcov/ + publish-test-results: + name: "Publish Tests Results" + needs: sdk_cli_tests + runs-on: ubuntu-latest + permissions: + checks: write + pull-requests: write + contents: read + issues: read + if: always() + + steps: + - name: checkout + uses: actions/checkout@v3 - name: Publish Test Results - if: always() - uses: EnricoMi/publish-unit-test-result-action/composite@v2 - with: - check_name: "SDK CLI Test Result [${{ steps.display_env.outputs.GITHUB_HEAD_REF }}](https://github.com/microsoft/promptflow/actions/workflows/promptflow-sdk-cli-test.yml?query=branch:${{ steps.display_env.outputs.GITHUB_HEAD_REF }}++)" - comment_title: "SDK CLI Test Result [${{ steps.display_env.outputs.GITHUB_HEAD_REF }}](https://github.com/microsoft/promptflow/actions/workflows/promptflow-sdk-cli-test.yml?query=branch:${{ steps.display_env.outputs.GITHUB_HEAD_REF }}++)" - files: | - ${{ env.testWorkingDirectory }}/test-results.xml - - name: Generate Coverage Report - if: (success() || failure()) - uses: orgoro/coverage@v3.1 + uses: "./.github/actions/step_publish_test_results" with: - coverageFile: ${{ env.testWorkingDirectory }}/coverage.xml + osVersion: ubuntu-latest + pythonVersion: 3.9 + coverageThreshold: 0.4 token: ${{ secrets.GITHUB_TOKEN }} - thresholdAll: 0.4 - - diff --git a/README.md b/README.md index 1d90971ec9e..3f3b9e30fbf 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,11 @@ > opening [issues](https://github.com/microsoft/promptflow/issues/new/choose), > submitting [PRs](https://github.com/microsoft/promptflow/pulls). -**Prompt flow** is a suite of development tools designed to streamline the end-to-end development cycle of LLM-based AI applications, from ideation, prototyping, testing, evaluation to production deployment and monitoring. It makes prompt engineering much easier and enables you to build LLM apps with production quality. +**Prompt flow** is a suite of development tools designed to streamline the end-to-end development cycle of LLM-based AI applications, from ideation, prototyping, testing, evaluation to production deployment and monitoring. It makes prompt engineering much easier and enables you to build LLM apps with production quality. -With prompt flow, you will be able to: +With prompt flow, you will be able to: -- Create executable workflows that link LLMs, prompts, Python code and other tools together. +- Create executable workflows that link LLMs, prompts, Python code and other tools together. - Debug and iterate your flows, especially the interaction with LLMs with ease. - Evaluate your flow's quality and performance with larger datasets. - Integrate the testing and evaluation into your CI/CD system to ensure quality of your flow. @@ -43,8 +43,8 @@ contact [opencode@microsoft.com](mailto:opencode@microsoft.com) with any additio ## Trademarks -This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft -trademarks or logos is subject to and must follow +This project may contain trademarks or logos for projects, products, or services. Authorized use of Microsoft +trademarks or logos is subject to and must follow [Microsoft's Trademark & Brand Guidelines](https://www.microsoft.com/en-us/legal/intellectualproperty/trademarks/usage/general). Use of Microsoft trademarks or logos in modified versions of this project must not cause confusion or imply Microsoft sponsorship. Any use of third-party trademarks or logos are subject to those third-party's policies. diff --git a/SECURITY.md b/SECURITY.md index e138ec5d6a7..9dc63164721 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -14,7 +14,7 @@ Instead, please report them to the Microsoft Security Response Center (MSRC) at If you prefer to submit without logging in, send email to [secure@microsoft.com](mailto:secure@microsoft.com). If possible, encrypt your message with our PGP key; please download it from the [Microsoft Security Response Center PGP Key page](https://aka.ms/opensource/security/pgpkey). -You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). +You should receive a response within 24 hours. If for some reason you do not, please follow up via email to ensure we received your original message. Additional information can be found at [microsoft.com/msrc](https://aka.ms/opensource/security/msrc). Please include the requested information listed below (as much as you can provide) to help us better understand the nature and scope of the possible issue: diff --git a/SUPPORT.md b/SUPPORT.md index 0e45e97b135..8d7be6abc3b 100644 --- a/SUPPORT.md +++ b/SUPPORT.md @@ -1,11 +1,11 @@ # Support -## How to file issues and get help +## How to file issues and get help -This project uses GitHub Issues to track bugs and feature requests. Please search the existing -issues before filing new issues to avoid duplicates. For new issues, file your bug or +This project uses GitHub Issues to track bugs and feature requests. Please search the existing +issues before filing new issues to avoid duplicates. For new issues, file your bug or feature request as a new Issue. -## Microsoft Support Policy +## Microsoft Support Policy Support for this **PROJECT or PRODUCT** is limited to the resources listed above. diff --git a/scripts/building/release-env.yml b/scripts/building/release-env.yml index 373d54f0c91..5425145602d 100644 --- a/scripts/building/release-env.yml +++ b/scripts/building/release-env.yml @@ -3,7 +3,6 @@ channels: - defaults - conda-forge dependencies: - - python=3.9 - pip - pip: - setuptools diff --git a/src/promptflow/CHANGELOG.md b/src/promptflow/CHANGELOG.md index 04b9dbf1103..a85e8dc4144 100644 --- a/src/promptflow/CHANGELOG.md +++ b/src/promptflow/CHANGELOG.md @@ -8,10 +8,10 @@ - **pf flow**: init/test/serve/export - **pf run**: create/update/stream/list/show/show-details/show-metrics/visualize/archive/restore/export - **pf connection**: create/update/show/list/delete -- Azure AI support: +- Azure AI support: - **pfazure run**: create/list/stream/show/show-details/show-metrics/visualize ## v0.1.0b1 (2022.07.20) -- Stub version in Pypi. \ No newline at end of file +- Stub version in Pypi. diff --git a/src/promptflow/README.md b/src/promptflow/README.md index 011cc6d4d9d..57556f9d8f2 100644 --- a/src/promptflow/README.md +++ b/src/promptflow/README.md @@ -8,11 +8,11 @@ > opening [issues](https://github.com/microsoft/promptflow/issues/new/choose), > submitting [PRs](https://github.com/microsoft/promptflow/pulls). -**Prompt flow** is a suite of development tools designed to streamline the end-to-end development cycle of LLM-based AI applications, from ideation, prototyping, testing, evaluation to production deployment and monitoring. It makes prompt engineering much easier and enables you to build LLM apps with production quality. +**Prompt flow** is a suite of development tools designed to streamline the end-to-end development cycle of LLM-based AI applications, from ideation, prototyping, testing, evaluation to production deployment and monitoring. It makes prompt engineering much easier and enables you to build LLM apps with production quality. -With prompt flow, you will be able to: +With prompt flow, you will be able to: -- Create executable workflows that link LLMs, prompts, Python code and other tools together. +- Create executable workflows that link LLMs, prompts, Python code and other tools together. - Debug and iterate your flows, especially the interaction with LLMs with ease. - Evaluate your flow's quality and performance with larger datasets. - Integrate the testing and evaluation into your CI/CD system to ensure quality of your flow. diff --git a/src/promptflow/promptflow/_core/tools_manager.py b/src/promptflow/promptflow/_core/tools_manager.py index aa2fcb960bb..faa0b7f1dd1 100644 --- a/src/promptflow/promptflow/_core/tools_manager.py +++ b/src/promptflow/promptflow/_core/tools_manager.py @@ -266,9 +266,7 @@ def _register(provider_cls, collection, type): for name, value in provider_cls.__dict__.items(): if hasattr(value, "__original_function"): name = value.__original_function.__qualname__ - value.__tool = function_to_tool_definition( - value, type=type, initialize_inputs=initialize_inputs - ) + value.__tool = function_to_tool_definition(value, type=type, initialize_inputs=initialize_inputs) collection[name] = value.__tool module_logger.debug(f"Registered {name} as a builtin function") # Get the connection type - provider name mapping for execution use diff --git a/src/promptflow/promptflow/_sdk/_orm/__init__.py b/src/promptflow/promptflow/_sdk/_orm/__init__.py index 713928f6a87..5ca254e51a6 100644 --- a/src/promptflow/promptflow/_sdk/_orm/__init__.py +++ b/src/promptflow/promptflow/_sdk/_orm/__init__.py @@ -4,8 +4,9 @@ __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore -from .connection import Connection from promptflow._sdk._orm.run_info import RunInfo + +from .connection import Connection from .session import mgmt_db_session __all__ = [ diff --git a/src/promptflow/promptflow/_sdk/_orm/retry.py b/src/promptflow/promptflow/_sdk/_orm/retry.py index 0f4b0faeff7..a381da07885 100644 --- a/src/promptflow/promptflow/_sdk/_orm/retry.py +++ b/src/promptflow/promptflow/_sdk/_orm/retry.py @@ -3,9 +3,9 @@ # --------------------------------------------------------- import time - from functools import partial, wraps from typing import Tuple, Union + from sqlalchemy.exc import OperationalError diff --git a/src/promptflow/promptflow/_sdk/entities/_connection.py b/src/promptflow/promptflow/_sdk/entities/_connection.py index 5c554bdf0fd..1d2eaf0edcd 100644 --- a/src/promptflow/promptflow/_sdk/entities/_connection.py +++ b/src/promptflow/promptflow/_sdk/entities/_connection.py @@ -297,6 +297,7 @@ class AzureOpenAIConnection(_StrongTypeConnection): :param api_version: The api version, default "2023-07-01-preview". :param name: Connection name. """ + TYPE = ConnectionType.AZURE_OPEN_AI def __init__( @@ -341,6 +342,7 @@ class OpenAIConnection(_StrongTypeConnection): :param organization: The organization, optional. :param name: Connection name. """ + TYPE = ConnectionType.OPEN_AI def __init__(self, api_key: str, organization: str = None, **kwargs): @@ -366,6 +368,7 @@ class SerpConnection(_StrongTypeConnection): :param api_key: The api key. :param name: Connection name. """ + TYPE = ConnectionType.SERP def __init__(self, api_key: str, **kwargs): @@ -400,6 +403,7 @@ class QdrantConnection(_EmbeddingStoreConnection): :param api_base: The api base. :param name: Connection name. """ + TYPE = ConnectionType.QDRANT @classmethod @@ -413,6 +417,7 @@ class WeaviateConnection(_EmbeddingStoreConnection): :param api_base: The api base. :param name: Connection name. """ + TYPE = ConnectionType.WEAVIATE @classmethod @@ -427,6 +432,7 @@ class CognitiveSearchConnection(_StrongTypeConnection): :param api_version: The api version, default "2023-07-01-Preview". :param name: Connection name. """ + TYPE = ConnectionType.COGNITIVE_SEARCH def __init__(self, api_key: str, api_base: str, api_version: str = "2023-07-01-Preview", **kwargs): @@ -463,6 +469,7 @@ class AzureContentSafetyConnection(_StrongTypeConnection): :param api_type: The api type, default "Content Safety". :param name: Connection name. """ + TYPE = ConnectionType.AZURE_CONTENT_SAFETY def __init__( @@ -514,6 +521,7 @@ class FormRecognizerConnection(AzureContentSafetyConnection): :param api_type: The api type, default "Form Recognizer". :param name: Connection name. """ + # Note: FormRecognizer and ContentSafety are using CognitiveService type in ARM, so keys are the same. TYPE = ConnectionType.FORM_RECOGNIZER @@ -533,6 +541,7 @@ class CustomConnection(_Connection): :param secrets: The secrets kv pairs. :param name: Connection name """ + TYPE = ConnectionType.CUSTOM def __init__(self, secrets: Dict[str, str], configs: Dict[str, str] = None, **kwargs): diff --git a/src/promptflow/promptflow/azure/_constants/__init__.py b/src/promptflow/promptflow/azure/_constants/__init__.py index ce75b56ff2c..bc886bec8f7 100644 --- a/src/promptflow/promptflow/azure/_constants/__init__.py +++ b/src/promptflow/promptflow/azure/_constants/__init__.py @@ -4,12 +4,7 @@ __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore +from ._component import COMMAND_COMPONENT_SPEC_TEMPLATE, DEFAULT_PYTHON_VERSION from ._flow import FlowJobType, FlowType -from ._component import DEFAULT_PYTHON_VERSION, COMMAND_COMPONENT_SPEC_TEMPLATE -__all__ = [ - "FlowJobType", - "FlowType", - "DEFAULT_PYTHON_VERSION", - "COMMAND_COMPONENT_SPEC_TEMPLATE" -] +__all__ = ["FlowJobType", "FlowType", "DEFAULT_PYTHON_VERSION", "COMMAND_COMPONENT_SPEC_TEMPLATE"] diff --git a/src/promptflow/promptflow/azure/_constants/_flow.py b/src/promptflow/promptflow/azure/_constants/_flow.py index 7a995945e08..b1e0578d190 100644 --- a/src/promptflow/promptflow/azure/_constants/_flow.py +++ b/src/promptflow/promptflow/azure/_constants/_flow.py @@ -2,6 +2,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- + class FlowType: STANDARD = "standard" CHAT = "chat" @@ -14,7 +15,7 @@ class FlowJobType: # Use this storage since it's the storage used by notebook -DEFAULT_STORAGE = 'workspaceworkingdirectory' +DEFAULT_STORAGE = "workspaceworkingdirectory" PROMPTFLOW_FILE_SHARE_DIR = "Promptflows" CHILD_RUNS_PAGE_SIZE = 25 # align with UX diff --git a/src/promptflow/promptflow/azure/_entities/_flow.py b/src/promptflow/promptflow/azure/_entities/_flow.py index 91d5de78a5b..61c6d445c7c 100644 --- a/src/promptflow/promptflow/azure/_entities/_flow.py +++ b/src/promptflow/promptflow/azure/_entities/_flow.py @@ -1,17 +1,18 @@ # --------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. # --------------------------------------------------------- +import logging import os.path from contextlib import contextmanager from os import PathLike from pathlib import Path -import logging -from typing import Optional, Union, List +from typing import List, Optional, Union -from promptflow.azure._ml import Code, AdditionalIncludesMixin from promptflow._sdk._constants import DAG_FILE_NAME -from .._constants._flow import DEFAULT_STORAGE +from promptflow.azure._ml import AdditionalIncludesMixin, Code + from ..._sdk._utils import PromptflowIgnoreFile +from .._constants._flow import DEFAULT_STORAGE # pylint: disable=redefined-builtin, unused-argument, f-string-without-interpolation @@ -20,7 +21,6 @@ class Flow(AdditionalIncludesMixin): - def __init__( self, path: Union[str, PathLike], @@ -61,6 +61,7 @@ def _get_all_additional_includes_configs(self) -> List: For flow, its additional include need to be read from dag with a helper function. """ from promptflow._sdk._utils import _get_additional_includes + return _get_additional_includes(os.path.join(self.code, self.path)) # endregion diff --git a/src/promptflow/promptflow/azure/_load_functions.py b/src/promptflow/promptflow/azure/_load_functions.py index f4851b0b78f..04b6900db87 100644 --- a/src/promptflow/promptflow/azure/_load_functions.py +++ b/src/promptflow/promptflow/azure/_load_functions.py @@ -5,7 +5,6 @@ from pathlib import Path from typing import IO, AnyStr, Optional, Union - from ._utils import is_arm_id diff --git a/src/promptflow/promptflow/azure/_ml/__init__.py b/src/promptflow/promptflow/azure/_ml/__init__.py index e8ea0109680..f2f7adf2805 100644 --- a/src/promptflow/promptflow/azure/_ml/__init__.py +++ b/src/promptflow/promptflow/azure/_ml/__init__.py @@ -24,11 +24,11 @@ def __call__(self, *args, **kwargs): # TODO: avoid import azure.ai.ml if promptflow.azure.configure is not called try: - from azure.ai.ml import load_component, MLClient + from azure.ai.ml import MLClient, load_component from azure.ai.ml.entities import Component - from azure.ai.ml.entities._load_functions import load_common from azure.ai.ml.entities._assets import Code from azure.ai.ml.entities._component._additional_includes import AdditionalIncludesMixin + from azure.ai.ml.entities._load_functions import load_common except ImportError: class load_component(_DummyCallableClassForLazyImportError): @@ -49,6 +49,7 @@ class Code(_DummyCallableClassForLazyImportError): class AdditionalIncludesMixin(_DummyCallableClassForLazyImportError): pass + __all__ = [ "load_component", "Component", diff --git a/src/promptflow/promptflow/azure/_schemas/_flow_schema.py b/src/promptflow/promptflow/azure/_schemas/_flow_schema.py index 2f7f1b5b8e1..f77c1ff4123 100644 --- a/src/promptflow/promptflow/azure/_schemas/_flow_schema.py +++ b/src/promptflow/promptflow/azure/_schemas/_flow_schema.py @@ -5,7 +5,7 @@ import logging from pathlib import Path -from azure.ai.ml._schema import YamlFileSchema, UnionField +from azure.ai.ml._schema import UnionField, YamlFileSchema from azure.ai.ml._schema.core.fields import LocalPathField from marshmallow import fields, post_load diff --git a/src/promptflow/promptflow/azure/_utils/_url_utils.py b/src/promptflow/promptflow/azure/_utils/_url_utils.py index c99b6b595da..dde716000f3 100644 --- a/src/promptflow/promptflow/azure/_utils/_url_utils.py +++ b/src/promptflow/promptflow/azure/_utils/_url_utils.py @@ -6,9 +6,12 @@ class BulkRunURL: """Parser for a flow run URL.""" + REGEX_PATTERN = ".*prompts/flow/([^/]+)/([^/]+)/bulktest/([^/]+).*" - RUN_URL_FORMAT = "https://ml.azure.com/prompts/flow/{}/{}/bulktest/{}/details?wsid=" \ - "/subscriptions/{}/resourcegroups/{}/providers/Microsoft.MachineLearningServices/workspaces/{}" + RUN_URL_FORMAT = ( + "https://ml.azure.com/prompts/flow/{}/{}/bulktest/{}/details?wsid=" + "/subscriptions/{}/resourcegroups/{}/providers/Microsoft.MachineLearningServices/workspaces/{}" + ) def __init__(self, url: str): if url: @@ -29,6 +32,7 @@ def get_url(cls, experiment_id, flow_id, bulk_test_id, subscription_id, resource class BulkRunId: """Parser for a flow run ID.""" + REGEX_PATTERN = "azureml://experiment/([^/]+)/flow/([^/]+)/bulktest/([^/]+)(/run/[^/]+)?" RUN_ID_FORMAT = "azureml://experiment/{}/flow/{}/bulktest/{}" @@ -48,9 +52,7 @@ def __init__(self, arm_id: str): @classmethod def get_url(cls, experiment_id, flow_id, bulk_test_id, *, run_id=None): - arm_id = cls.RUN_ID_FORMAT.format( - experiment_id, flow_id, bulk_test_id - ) + arm_id = cls.RUN_ID_FORMAT.format(experiment_id, flow_id, bulk_test_id) if run_id: arm_id += "/run/{}".format(run_id) return arm_id diff --git a/src/promptflow/promptflow/azure/_utils/gerneral.py b/src/promptflow/promptflow/azure/_utils/gerneral.py index 4b362a2a016..d40f68f2225 100644 --- a/src/promptflow/promptflow/azure/_utils/gerneral.py +++ b/src/promptflow/promptflow/azure/_utils/gerneral.py @@ -15,7 +15,7 @@ def is_arm_id(obj) -> bool: def get_user_alias_from_credential(credential): token = credential.get_token("https://storage.azure.com/.default").token - decode_json = jwt.decode(token, options={'verify_signature': False, 'verify_aud': False}) + decode_json = jwt.decode(token, options={"verify_signature": False, "verify_aud": False}) try: email = decode_json["upn"] return email.split("@")[0] diff --git a/src/promptflow/promptflow/azure/operations/_fileshare_storeage_helper.py b/src/promptflow/promptflow/azure/operations/_fileshare_storeage_helper.py index 9b387479f74..fd5a102903a 100644 --- a/src/promptflow/promptflow/azure/operations/_fileshare_storeage_helper.py +++ b/src/promptflow/promptflow/azure/operations/_fileshare_storeage_helper.py @@ -4,9 +4,8 @@ import os from collections import defaultdict from multiprocessing import Lock - from pathlib import Path -from typing import Any, Optional, Dict +from typing import Any, Dict, Optional from azure.ai.ml._artifacts._fileshare_storage_helper import FileStorageClient from azure.ai.ml._utils._asset_utils import ( @@ -16,12 +15,11 @@ get_directory_size, ) from azure.core.exceptions import ResourceExistsError -from azure.storage.fileshare import ShareDirectoryClient, DirectoryProperties +from azure.storage.fileshare import DirectoryProperties, ShareDirectoryClient from promptflow._sdk._vendor import get_upload_files_from_folder -from promptflow.azure._utils.gerneral import get_user_alias_from_credential from promptflow.azure._constants._flow import PROMPTFLOW_FILE_SHARE_DIR - +from promptflow.azure._utils.gerneral import get_user_alias_from_credential uploading_lock = defaultdict(Lock) @@ -94,14 +92,14 @@ def upload( return artifact_info def upload_file( - self, - source: str, - dest: str, - show_progress: Optional[bool] = None, - msg: Optional[str] = None, - in_directory: bool = False, - subdirectory_client: Optional[ShareDirectoryClient] = None, - callback: Optional[Any] = None, + self, + source: str, + dest: str, + show_progress: Optional[bool] = None, + msg: Optional[str] = None, + in_directory: bool = False, + subdirectory_client: Optional[ShareDirectoryClient] = None, + callback: Optional[Any] = None, ) -> None: """ " Upload a single file to a path inside the file system directory.""" @@ -143,12 +141,12 @@ def upload_file( self.uploaded_file_count = self.uploaded_file_count + 1 def upload_dir( - self, - source: str, - dest: str, - msg: str, - show_progress: bool, - ignore_file: IgnoreFile, + self, + source: str, + dest: str, + msg: str, + show_progress: bool, + ignore_file: IgnoreFile, ) -> None: """Upload a directory to a path inside the fileshare directory.""" subdir = self.directory_client.create_subdirectory(dest) diff --git a/src/promptflow/promptflow/contracts/flow.py b/src/promptflow/promptflow/contracts/flow.py index 4ab5fdd2755..a31997105f8 100644 --- a/src/promptflow/promptflow/contracts/flow.py +++ b/src/promptflow/promptflow/contracts/flow.py @@ -335,10 +335,9 @@ def deserialize(data: dict) -> "Flow": node_variants={name: NodeVariants.deserialize(v) for name, v in (data.get("node_variants") or {}).items()}, ) - def _apply_default_node_variants(self: 'Flow'): + def _apply_default_node_variants(self: "Flow"): self.nodes = [ - self._apply_default_node_variant(node, self.node_variants) - if node.use_variants else node + self._apply_default_node_variant(node, self.node_variants) if node.use_variants else node for node in self.nodes ] return self diff --git a/src/promptflow/promptflow/executor/_line_execution_process_pool.py b/src/promptflow/promptflow/executor/_line_execution_process_pool.py index 6eb84e1ee0c..88cab14608e 100644 --- a/src/promptflow/promptflow/executor/_line_execution_process_pool.py +++ b/src/promptflow/promptflow/executor/_line_execution_process_pool.py @@ -365,6 +365,7 @@ def create_executor_legacy(*, flow, connections, loaded_tools, cache_manager, st def get_available_max_worker_count(): import math import os + import psutil pid = os.getpid() diff --git a/src/promptflow/promptflow/storage/run_records.py b/src/promptflow/promptflow/storage/run_records.py index 1c560e5d3da..e41aef61c81 100644 --- a/src/promptflow/promptflow/storage/run_records.py +++ b/src/promptflow/promptflow/storage/run_records.py @@ -3,7 +3,7 @@ # --------------------------------------------------------- import json -from dataclasses import dataclass, asdict +from dataclasses import asdict, dataclass from datetime import datetime from promptflow._utils.dataclass_serializer import serialize @@ -21,6 +21,7 @@ class NodeRunRecord: :param datetime end_time: The time the node finished running :param str status: The status of the node run """ + node_name: str line_number: int run_info: str @@ -67,6 +68,7 @@ class LineRunRecord: :param str status: The status of the line execution :param str tags: The tags associated with the line run """ + line_number: int run_info: str start_time: datetime diff --git a/src/promptflow/tests/executor/e2etests/test_executor_timeout.py b/src/promptflow/tests/executor/e2etests/test_executor_timeout.py index e349a40a4a0..b1d89e801ce 100644 --- a/src/promptflow/tests/executor/e2etests/test_executor_timeout.py +++ b/src/promptflow/tests/executor/e2etests/test_executor_timeout.py @@ -65,7 +65,12 @@ def skip_serp(self, flow_folder, dev_connections): ], ) def test_executor_exec_bulk_with_timeout(self, flow_folder, dev_connections): - executor = FlowExecutor.create(get_yaml_file(flow_folder), dev_connections, raise_ex=True, line_timeout_sec=5) + executor = FlowExecutor.create( + get_yaml_file(flow_folder), + dev_connections, + raise_ex=True, + line_timeout_sec=5, + ) run_id = str(uuid.uuid4()) bulk_inputs = self.get_bulk_inputs() nlines = len(bulk_inputs) @@ -88,7 +93,11 @@ def test_executor_exec_bulk_with_timeout(self, flow_folder, dev_connections): def test_executor_exec_bulk_with_one_line_timeout(self, flow_folder, dev_connections): mem_run_storage = MemoryRunStorage() executor = FlowExecutor.create( - get_yaml_file(flow_folder), dev_connections, raise_ex=False, storage=mem_run_storage, line_timeout_sec=15 + get_yaml_file(flow_folder), + dev_connections, + raise_ex=False, + storage=mem_run_storage, + line_timeout_sec=15, ) run_id = str(uuid.uuid4()) bulk_inputs = self.get_bulk_inputs(flow_folder=flow_folder) diff --git a/src/promptflow/tests/executor/e2etests/test_package_tool.py b/src/promptflow/tests/executor/e2etests/test_package_tool.py index b1f7e8f06c2..b49e984770a 100644 --- a/src/promptflow/tests/executor/e2etests/test_package_tool.py +++ b/src/promptflow/tests/executor/e2etests/test_package_tool.py @@ -43,9 +43,17 @@ def get_bulk_inputs(self, nlinee=4, flow_folder=""): def test_executor_package_tool_with_conn(self, mocker): flow_folder = PACKAGE_TOOL_BASE / "tool_with_connection" package_tool_definition = get_flow_package_tool_definition(flow_folder) - mocker.patch("promptflow.tools.list.list_package_tools", return_value=package_tool_definition) + mocker.patch( + "promptflow.tools.list.list_package_tools", + return_value=package_tool_definition, + ) name, secret = "dummy_name", "dummy_secret" - connections = {"test_conn": {"type": "TestConnection", "value": {"name": name, "secret": secret}}} + connections = { + "test_conn": { + "type": "TestConnection", + "value": {"name": name, "secret": secret}, + } + } executor = FlowExecutor.create(get_yaml_file(flow_folder), connections, raise_ex=True) flow_result = executor.exec_line({}) assert flow_result.run_info.status == Status.Completed @@ -54,6 +62,7 @@ def test_executor_package_tool_with_conn(self, mocker): assert v.status == Status.Completed assert v.output == name + secret + @pytest.mark.skipif(sys.platform == "darwin", reason="Skip on Mac") def test_executor_package_with_prompt_tool(self, dev_connections, mocker): flow_folder = PACKAGE_TOOL_BASE / "custom_llm_tool" package_tool_definition = get_flow_package_tool_definition(flow_folder) diff --git a/src/promptflow/tests/executor/e2etests/test_telemetry.py b/src/promptflow/tests/executor/e2etests/test_telemetry.py index dc8b033ca2d..ecc626e9c6e 100644 --- a/src/promptflow/tests/executor/e2etests/test_telemetry.py +++ b/src/promptflow/tests/executor/e2etests/test_telemetry.py @@ -1,4 +1,5 @@ import json +import sys import uuid from collections import namedtuple from unittest.mock import patch @@ -24,13 +25,15 @@ def mock_stream_chat(**kwargs): return stream_response(kwargs) +@pytest.mark.skipif(sys.platform == "darwin", reason="Skip on Mac") @pytest.mark.usefixtures("dev_connections") @pytest.mark.e2etest class TestExecutorTelemetry: def test_executor_openai_telemetry(self, dev_connections): """This test validates telemetry info header is correctly injected to OpenAI API by mocking openai.ChatCompletion.create method. The mock method will return a generator - that yields a namedtuple with a json string of the headers passed to the method.""" + that yields a namedtuple with a json string of the headers passed to the method. + """ with patch("openai.ChatCompletion.create", new=mock_stream_chat): operation_context = OperationContext.get_instance() diff --git a/src/promptflow/tests/executor/unittests/_utils/test_thread_utils.py b/src/promptflow/tests/executor/unittests/_utils/test_thread_utils.py index f94ed4dd52c..45b03092652 100644 --- a/src/promptflow/tests/executor/unittests/_utils/test_thread_utils.py +++ b/src/promptflow/tests/executor/unittests/_utils/test_thread_utils.py @@ -1,3 +1,4 @@ +import sys import time from io import StringIO from logging import WARNING, Logger, StreamHandler @@ -12,6 +13,7 @@ class DummyException(Exception): pass +@pytest.mark.skipif(sys.platform == "darwin", reason="Skip on Mac") @pytest.mark.unittest class TestRepeatLogTimer: def test_context_manager(self): diff --git a/src/promptflow/tests/executor/unittests/executor/test_flow_executor.py b/src/promptflow/tests/executor/unittests/executor/test_flow_executor.py index 3b022185653..ad949b70237 100644 --- a/src/promptflow/tests/executor/unittests/executor/test_flow_executor.py +++ b/src/promptflow/tests/executor/unittests/executor/test_flow_executor.py @@ -67,7 +67,7 @@ def test_apply_inputs_mapping(self, inputs, inputs_mapping, expected): "Couldn't find these mapping relations: ${baseline.output}, ${data.output}. " "Please make sure your input mapping keys and values match your YAML input section and input data. " "If a mapping value has a '${data' prefix, it might be generated from the YAML input section, " - "and you may need to manually assign input mapping based on your input data." + "and you may need to manually assign input mapping based on your input data.", ), ], ) diff --git a/src/promptflow/tests/executor/unittests/executor/test_flow_validator.py b/src/promptflow/tests/executor/unittests/executor/test_flow_validator.py index 7c5ee582c47..014ba284b79 100644 --- a/src/promptflow/tests/executor/unittests/executor/test_flow_validator.py +++ b/src/promptflow/tests/executor/unittests/executor/test_flow_validator.py @@ -5,7 +5,7 @@ from promptflow.executor._errors import InvalidFlowRequest from promptflow.executor.flow_validator import FlowValidator -from ...utils import get_yaml_file, WRONG_FLOW_ROOT +from ...utils import WRONG_FLOW_ROOT, get_yaml_file @pytest.mark.unittest @@ -28,10 +28,7 @@ def test_ensure_nodes_order(self, flow_folder, expected_node_order): @pytest.mark.parametrize( "flow_folder, error_message", [ - ( - "nodes_cycle", - "There is a circular dependency in the flow 'node_cycle'." - ), + ("nodes_cycle", "There is a circular dependency in the flow 'node_cycle'."), ( "nodes_cycle_with_skip", "There is a circular dependency in the flow 'node_cycle_with_skip'.",