[Separation] Separation of devkit and azure (microsoft#2502)

# Description

Move devkits and azures


![image](https://github.com/microsoft/promptflow/assets/2208599/bec0ee1d-2d40-46b3-845f-296653fb793a)

![image](https://github.com/microsoft/promptflow/assets/2208599/bb409b09-9ffe-4ad1-bd3b-6bf80f9cff96)

![image](https://github.com/microsoft/promptflow/assets/2208599/296aec64-2305-4863-803d-88b41eb4d382)

We have some critical issues when install streamlit with poetry. And pf
scripts, will fix in later patches.

# All Promptflow Contribution checklist:
- [ ] **The pull request does not introduce [breaking changes].**
- [ ] **CHANGELOG is updated for new features, bug fixes or other
significant changes.**
- [ ] **I have read the [contribution guidelines](../CONTRIBUTING.md).**
- [ ] **Create an issue and link to the pull request to get dedicated
review from promptflow team. Learn more: [suggested
workflow](../CONTRIBUTING.md#suggested-workflow).**

## General Guidelines and Best Practices
- [ ] Title of the pull request is clear and informative.
- [ ] There are a small number of commits, each of which have an
informative message. This means that previously merged commits do not
appear in the history of the PR. For more information on cleaning up the
commits in your PR, [see this
page](https://github.com/Azure/azure-powershell/blob/master/documentation/development-docs/cleaning-up-commits.md).

### Testing Guidelines
- [ ] Pull request includes test coverage for the included changes.

.cspell.json

@@ -25,18 +25,21 @@
     "**/*.txt",
     ".gitignore",
     "scripts/docs/_build/**",
-    "src/promptflow/promptflow/azure/_restclient/flow/**",
-    "src/promptflow/promptflow/azure/_restclient/swagger.json",
-    "src/promptflow/promptflow/azure/_models/**",
+    "src/promptflow-azure/promptflow/azure/_restclient/flow/**",
+    "src/promptflow-azure/promptflow/azure/_restclient/swagger.json",
+    "src/promptflow-azure/promptflow/azure/_models/**",
+    "src/promptflow-azure/tests/**",
     "src/promptflow-core/promptflow/core/_connection_provider/_models/**",
     "src/promptflow/tests/**",
     "src/promptflow-recording/**",
     "src/promptflow-tools/tests/**",
     "**/flow.dag.yaml",
+    "**/pyproject.toml",
     "**/setup.py",
     "scripts/installer/curl_install_pypi/**",
     "scripts/installer/windows/**",
     "src/promptflow/promptflow/_sdk/_service/pfsvc.py",
+    "src/promptflow-devkit/promptflow/_sdk/_service/pfsvc.py",
     ".github/workflows/**",
     ".github/actions/**",
     ".github/pipelines/**"

.github/workflows/promptflow-executor-e2e-test.yml

@@ -106,14 +106,16 @@ jobs:
         Set-PSDebug -Trace 1
         pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
         pip install ${{ github.workspace }}/src/promptflow-tracing
-        pip install ${{ github.workspace }}/src/promptflow-core
-        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[executor-service]"}}
+        pip install ${{ github.workspace }}/src/promptflow-core[executor-service]
+        pip install ${{ github.workspace }}/src/promptflow-devkit
+        pip install ${{ github.workspace }}/src/promptflow-azure
+        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
         pip freeze
     - name: install recording
       run: |
         pip install vcrpy
-        pip install -e .
+        pip install .
       working-directory: ${{ env.RECORD_DIRECTORY }}
     - name: Azure Login
       uses: azure/login@v1

.github/workflows/promptflow-executor-unit-test.yml

@@ -110,14 +110,16 @@ jobs:
         Set-PSDebug -Trace 1
         pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
         pip install ${{ github.workspace }}/src/promptflow-tracing
-        pip install ${{ github.workspace }}/src/promptflow-core
-        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[executor-service]"}}
+        pip install ${{ github.workspace }}/src/promptflow-core[executor-service]
+        pip install ${{ github.workspace }}/src/promptflow-devkit
+        pip install ${{ github.workspace }}/src/promptflow-azure
+        gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
         pip freeze
     - name: install recording
       run: |
         pip install vcrpy
-        pip install -e .
+        pip install .
       working-directory: ${{ env.RECORD_DIRECTORY }}
     - name: Azure Login
       uses: azure/login@v1

.github/workflows/promptflow-global-config-test.yml

@@ -69,6 +69,10 @@ jobs:
         echo "Installed promptflow-tracing"
         pip uninstall -y promptflow-core
         pip install ${{ github.workspace }}/src/promptflow-core
+        pip uninstall -y promptflow-devkit
+        pip install ${{ github.workspace }}/src/promptflow-devkit
+        pip uninstall -y promptflow-azure
+        pip install ${{ github.workspace }}/src/promptflow-azure
         pip freeze
     - name: Azure Login
       uses: azure/login@v1

.github/workflows/promptflow-import-linter.yml

@@ -18,6 +18,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v4
+    - name: Set up Python 3.11
+      uses: actions/setup-python@v5
+      with:
+        python-version: 3.11
     - uses: snok/install-poetry@v1
     - name: Install all packages
       run: |

.github/workflows/promptflow-sdk-cli-test.yml

@@ -5,6 +5,7 @@ on:
   pull_request:
     paths:
       - src/promptflow-core/**
+      - src/promptflow-devkit/**
       - src/promptflow/**
       - src/promptflow-tracing/**
       - scripts/building/**
@@ -81,13 +82,14 @@ jobs:
         pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
         pip install ${{ github.workspace }}/src/promptflow-tracing
         pip install ${{ github.workspace }}/src/promptflow-core
+        pip install ${{ github.workspace }}/src/promptflow-devkit[pyarrow]
         gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
         pip freeze
     - name: install recording
       run: |
         pip install vcrpy
-        pip install -e .
+        pip install .
       working-directory: ${{ env.RECORD_DIRECTORY }}
     - name: Azure login (non pull_request workflow)
       if: github.event_name != 'pull_request'
@@ -142,6 +144,7 @@ jobs:
       run: |
         Set-PSDebug -Trace 1
         pip uninstall -y promptflow promptflow-sdk promptflow-tools
+        pip install ${{ github.workspace }}/src/promptflow-devkit
         gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}}
         gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
         pip freeze

.github/workflows/sdk-cli-azure-test-production.yml

@@ -89,6 +89,8 @@ jobs:
           pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
           pip install ${{ github.workspace }}/src/promptflow-tracing
           pip install ${{ github.workspace }}/src/promptflow-core
+          pip install ${{ github.workspace }}/src/promptflow-devkit
+          pip install ${{ github.workspace }}/src/promptflow-azure
           gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}}
           gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
           pip freeze

.github/workflows/sdk-cli-azure-test-pull-request.yml

@@ -7,6 +7,7 @@ on:
   pull_request:
     paths:
       - src/promptflow-core/**
+      - src/promptflow-devkit/**
       - src/promptflow/**
       - scripts/building/**
       - src/promptflow-tracing/**
@@ -89,6 +90,8 @@ jobs:
           pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
           pip install ${{ github.workspace }}/src/promptflow-tracing
           pip install ${{ github.workspace }}/src/promptflow-core
+          pip install ${{ github.workspace }}/src/promptflow-devkit
+          pip install ${{ github.workspace }}/src/promptflow-azure
           gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}}
           gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
           pip freeze

.github/workflows/sdk-cli-azure-test.yml

@@ -88,6 +88,8 @@ jobs:
           pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
           pip install ${{ github.workspace }}/src/promptflow-tracing
           pip install ${{ github.workspace }}/src/promptflow-core
+          pip install ${{ github.workspace }}/src/promptflow-devkit
+          pip install ${{ github.workspace }}/src/promptflow-azure
           gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure]"}}
           gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
           pip freeze

.github/workflows/sdk-cli-perf-monitor-test.yml

@@ -7,6 +7,8 @@ on:
   pull_request:
     paths:
       - src/promptflow/**
+      - src/promptflow-core/**
+      - src/promptflow-devkit/**
       - scripts/building/**
       - .github/workflows/sdk-cli-perf-monitor-test.yml
 
@@ -81,7 +83,9 @@ jobs:
           pip install -r ${{ github.workspace }}/src/promptflow/dev_requirements.txt
           pip install ${{ github.workspace }}/src/promptflow-tracing
           pip install ${{ github.workspace }}/src/promptflow-core
-          gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[azure,executable,azureml-serving]"}}
+          pip install ${{ github.workspace }}/src/promptflow-devkit
+          pip install ${{ github.workspace }}/src/promptflow-azure
+          gci ./promptflow -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install "$($_.FullName)[all]"}}
           gci ./promptflow-tools -Recurse | % {if ($_.Name.Contains('.whl')) {python -m pip install $_.FullName}}
           pip freeze
 

.gitignore

@@ -165,7 +165,6 @@ cython_debug/
 
 # ignore
 !flow.tools.json
-**/.promptflow/*
 .runs
 connection.json
 .env

.pre-commit-config.yaml

@@ -1,7 +1,7 @@
 # See https://pre-commit.com for more information
 # See https://pre-commit.com/hooks.html for more hooks
 
-exclude: '(^docs/)|flows|scripts|src/promptflow/promptflow/azure/_restclient/|src/promptflow-core/promptflow/core/_connection_provider/_models/|src/promptflow/promptflow/azure/_models/|src/promptflow/tests/test_configs|src/promptflow-tools'
+exclude: '(^docs/)|flows|scripts|src/promptflow-azure/promptflow/azure/_restclient/|src/promptflow-core/promptflow/core/_connection_provider/_models/|src/promptflow-azure/promptflow/azure/_models/|src/promptflow/tests/test_configs|src/promptflow-tools'
 
 repos:
     - repo: https://github.com/pre-commit/pre-commit-hooks

scripts/check_enforcer/check_enforcer.py

@@ -47,11 +47,11 @@
         "scripts/building/**",
         ".github/workflows/promptflow-sdk-cli-test.yml",
     ],
-    "sdk_cli_global_config_tests": [
-        "src/promptflow/**",
-        "scripts/building/**",
-        ".github/workflows/promptflow-global-config-test.yml",
-    ],
+    # "sdk_cli_global_config_tests": [
+    #     "src/promptflow/**",
+    #     "scripts/building/**",
+    #     ".github/workflows/promptflow-global-config-test.yml",
+    # ],
     "sdk_cli_azure_test_replay": [
         "src/promptflow/**",
         "scripts/building/**",

scripts/compliance-check/user_exclusion.xml

@@ -3,5 +3,5 @@
   <!--index-xxx.js is an auto-generated javascript file - skipped given it's not expected to be readable -->
   <Exclusion Type="FileName">SRC\PROMPTFLOW-CORE\PROMPTFLOW\CORE\_SERVING\STATIC\INDEX.JS</Exclusion>
   <Exclusion Type="FileType">.MIN.JS</Exclusion>
-  <Exclusion Type="FolderPathFull">SRC\PROMPTFLOW\PROMPTFLOW\_SDK\_SERVICE\STATIC\</Exclusion>
+  <Exclusion Type="FolderPathFull">SRC\PROMPTFLOW-DEVKIT\PROMPTFLOW\_SDK\_SERVICE\STATIC\</Exclusion>
 </PoliCheckExclusions>

scripts/dev-setup/main.py

@@ -15,9 +15,8 @@
 PROMPT_FLOW_PKGS = [
     "promptflow-tracing",
     "promptflow-core",
-    # TODO: uncomment below lines when the packages are ready
-    # "promptflow-devkit",
-    # "promptflow-azure",
+    "promptflow-devkit",
+    "promptflow-azure",
     "promptflow[azure]",
     "promptflow-tools",
 ]

setup.cfg

@@ -11,8 +11,8 @@ exclude =
     docs/*
     venv,.pytest_cache
     build
-    src/promptflow/promptflow/azure/_restclient
-    src/promptflow/promptflow/azure/_models
+    src/promptflow-azure/promptflow/azure/_restclient
+    src/promptflow-azure/promptflow/azure/_models
     src/promptflow-core/promptflow/core/_connection_provider/_models
     src/promptflow/tests/test_configs/*
 import-order-style = google

src/promptflow-azure/promptflow/azure/__init__.py

@@ -2,6 +2,8 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-from ._version import __version__
+__path__ = __import__("pkgutil").extend_path(__path__, __name__)  # type: ignore
 
-__all__ = ["__version__"]
+from ._pf_client import PFClient
+
+__all__ = ["PFClient"]

src/promptflow-azure/pyproject.toml

@@ -2,6 +2,9 @@
 name = "promptflow-azure"
 version = "0.1.0b1"
 description = "Prompt flow azure"
+include = [
+  "promptflow/azure/resources/*"
+]
 
 license = "MIT"
 
@@ -36,10 +39,18 @@ packages = [
 
 [tool.poetry.dependencies]
 python = "<4.0,>=3.8"
+azure-core = ">=1.26.4,<2.0.0"
+"azure-storage-blob[aio]" = ">=12.17.0,<13.0.0"  # add [aio] for async run download feature
+azure-identity = ">=1.12.0,<2.0.0"
+azure-ai-ml = ">=1.14.0,<2.0.0"
+azure-cosmos = ">=4.5.1,<5.0.0"  # used to upload trace to cloud
+pyjwt = ">=2.4.0,<3.0.0"  # requirement of control plane SDK
+promptflow-devkit = "<2.0.0"
 
 [tool.poetry.group.dev.dependencies]
 pre-commit = "*"
 import-linter = "*"
+promptflow-devkit = { path = "../promptflow-devkit" }
 
 [tool.poetry.group.test.dependencies]
 pytest = "*"
@@ -50,6 +61,9 @@ pytest-xdist = "*"
 requires = ["poetry-core>=1.5.0"]
 build-backend = "poetry.core.masonry.api"
 
+[tool.poetry.scripts]
+pfazure = "promptflow.azure._cli.entry:main"
+
 [tool.pytest.ini_options]
 markers = [
   "unittest",

src/promptflow-core/pyproject.toml

@@ -54,6 +54,7 @@ executor-service = ["fastapi"]
 [tool.poetry.group.dev.dependencies]
 pre-commit = "*"
 import-linter = "*"
+promptflow-tracing = { path = "../promptflow-tracing" }
 
 [tool.poetry.group.test.dependencies]
 pytest = "*"
@@ -74,11 +75,6 @@ markers = [
 # durations - list the slowest test durations
 addopts = """
 --junit-xml=test-results.xml \
---cov=promptflow \
---cov-config=pyproject.toml \
---cov-report=term \
---cov-report=html \
---cov-report=xml \
 --dist loadfile \
 --log-level=info \
 --log-format="%(asctime)s %(levelname)s %(message)s" \

src/promptflow-devkit/pf

@@ -0,0 +1,10 @@
+#!/usr/bin/env python
+
+import sys
+import os
+
+def main():
+    if os.environ.get('PF_INSTALLER') is None:
+        os.environ['PF_INSTALLER'] = 'PIP'
+
+    os.execl(sys.executable, sys.executable, '-m', 'promptflow._cli._pf.entry', *sys.argv[1:])

src/promptflow-devkit/promptflow/_cli/data/entry_flow/requirements_txt

@@ -0,0 +1 @@
+promptflow

src/promptflow/promptflow/_cli/data/package_tool/README.md.jinja2 → src/promptflow-devkit/promptflow/_cli/data/package_tool/README.md.jinja2

@@ -11,4 +11,4 @@ The directory structure in the package tool is as follows:
         __init__.py
 ```
 
-Please refer to [tool doc](https://microsoft.github.io/promptflow/how-to-guides/develop-a-tool/index.html) for more details about how to develop a tool.
+Please refer to [tool doc](https://microsoft.github.io/promptflow/how-to-guides/develop-a-tool/index.html) for more details about how to develop a tool.

src/promptflow/promptflow/_sdk/_service/static/chat_index.html → src/promptflow-devkit/promptflow/_sdk/_service/static/chat_index.html

@@ -20,4 +20,4 @@
     <div id="root"></div>
     <script src="{{ url_for('static', filename='assets/index.mjs') }}"></script>
   </body>
-</html>
+</html>

src/promptflow/promptflow/_sdk/data/docker/runit/promptflow-serve/finish.jinja2 → src/promptflow-devkit/promptflow/_sdk/data/docker/runit/promptflow-serve/finish.jinja2

@@ -10,4 +10,4 @@ while pgrep gunicorn >/dev/null; do
   sleep 1
 done
 
-echo "$(date -uIns) - Stopped all Gunicorn processes"
+echo "$(date -uIns) - Stopped all Gunicorn processes"

src/promptflow/promptflow/_sdk/data/docker_csharp/runit/promptflow-serve/finish.jinja2 → src/promptflow-devkit/promptflow/_sdk/data/docker_csharp/runit/promptflow-serve/finish.jinja2

@@ -1,4 +1,4 @@
 #!/bin/bash
 
 echo "$(date -uIns) - promptflow-serve/finish $@"
-echo "$(date -uIns) - Stopped all Gunicorn processes"
+echo "$(date -uIns) - Stopped all Gunicorn processes"

src/promptflow/promptflow/_sdk/data/docker_csharp/runit/promptflow-serve/run.jinja2 → src/promptflow-devkit/promptflow/_sdk/data/docker_csharp/runit/promptflow-serve/run.jinja2

@@ -2,4 +2,4 @@
 
 echo "start promptflow serving"
 cd /flow
-dotnet Promptflow.dll --port "8080" --yaml_path "flow.dag.yaml" --assembly_folder "." --connection_folder_path  "../connections" --log_path "" --serving
+dotnet Promptflow.dll --port "8080" --yaml_path "flow.dag.yaml" --assembly_folder "." --connection_folder_path  "../connections" --log_path "" --serving

src/promptflow-devkit/promptflow/devkit/__init__.py → src/promptflow-devkit/promptflow/_version.py

@@ -2,6 +2,4 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 
-from ._version import __version__
-
-__all__ = ["__version__"]
+VERSION = "0.0.1"