From b9cb0e1b3bd122f89403f5f1659bbdc87396b0e8 Mon Sep 17 00:00:00 2001 From: Clement Wang <47586720+wangchao1230@users.noreply.github.com> Date: Thu, 3 Aug 2023 18:39:45 +0800 Subject: [PATCH] Preview samples of pf code first experience (#1) Signed-off-by: Brynn Yin Co-authored-by: Clement Wang Co-authored-by: Korin <0mza987@gmail.com> Co-authored-by: Brynn Yin Co-authored-by: Han Wang Co-authored-by: Brynn Yin <24237253+brynn-code@users.noreply.github.com> Co-authored-by: zhangxingzhi Co-authored-by: Xingzhi Zhang <37076709+elliotzh@users.noreply.github.com> Co-authored-by: Zhengfei Wang <38847871+zhengfeiwang@users.noreply.github.com> Co-authored-by: Daniel Schneider Co-authored-by: zhen Co-authored-by: Philip Gao (from Dev Box) Co-authored-by: Ying Chen Co-authored-by: Ying Chen <2601502859@qq.com> --- .github/workflows/configuration.yml | 44 ++ .github/workflows/connections_connection.yml | 44 ++ .../tutorials_flowinpipeline_pipeline.yml | 44 ++ .../tutorials_getstarted_quickstart.yml | 44 ++ .../tutorials_getstarted_quickstartazure.yml | 44 ++ ...rials_runmanagement_cloudrunmanagement.yml | 44 ++ .gitignore | 9 + README.md | 8 +- docs/README.md | 3 + docs/dev/dev_setup.md | 1 + examples/README.md | 64 +++ examples/configuration.ipynb | 159 +++++++ examples/connections/.env.example | 1 + examples/connections/README.md | 38 ++ examples/connections/azure_content_safety.yml | 7 + examples/connections/azure_openai.yml | 7 + examples/connections/cognitive_search.yml | 6 + examples/connections/connection.ipynb | 236 ++++++++++ examples/connections/custom.yml | 7 + examples/connections/openai.yml | 5 + examples/connections/serp.yml | 4 + examples/dev_requirements.txt | 7 + examples/flows/chat/basic-chat/README.md | 63 +++ .../flows/chat/basic-chat/azure_openai.yml | 7 + examples/flows/chat/basic-chat/chat.jinja2 | 12 + examples/flows/chat/basic-chat/data.jsonl | 0 examples/flows/chat/basic-chat/flow.dag.yaml | 29 ++ .../flows/chat/chat-with-wikipedia/README.md | 67 +++ .../chat-with-wikipedia/augmented_chat.jinja2 | 17 + .../chat/chat-with-wikipedia/azure_openai.yml | 7 + .../flows/chat/chat-with-wikipedia/data.jsonl | 1 + .../extract_query_from_question.jinja2 | 36 ++ .../chat/chat-with-wikipedia/flow.dag.yaml | 72 +++ .../chat/chat-with-wikipedia/get_wiki_url.py | 57 +++ .../process_search_result.py | 17 + .../search_result_from_url.py | 75 +++ .../flows/evaluation/basic-eval/README.md | 33 ++ .../flows/evaluation/basic-eval/aggregate.py | 38 ++ .../flows/evaluation/basic-eval/data.jsonl | 1 + .../flows/evaluation/basic-eval/flow.dag.yaml | 38 ++ .../evaluation/basic-eval/line_process.py | 17 + .../classification-accuracy-eval/README.md | 38 ++ .../calculate_accuracy.py | 21 + .../classification-accuracy-eval/data.jsonl | 3 + .../flow.dag.yaml | 39 ++ .../classification-accuracy-eval/grade.py | 6 + .../standard/basic-with-builtin-llm/README.md | 66 +++ .../basic-with-builtin-llm/azure_openai.yml | 7 + .../basic-with-builtin-llm/data.jsonl | 2 + .../basic-with-builtin-llm/flow.dag.yaml | 29 ++ .../basic-with-builtin-llm/hello.jinja2 | 2 + .../basic-with-builtin-llm/requirements.txt | 6 + .../standard/basic-with-connection/README.md | 101 ++++ .../basic-with-connection/azure_openai.yml | 7 + .../standard/basic-with-connection/custom.yml | 9 + .../standard/basic-with-connection/data.jsonl | 2 + .../basic-with-connection/flow.dag.yaml | 26 ++ .../basic-with-connection/hello.jinja2 | 2 + .../standard/basic-with-connection/hello.py | 71 +++ .../basic-with-connection/requirements.txt | 6 + examples/flows/standard/basic/.env.example | 4 + examples/flows/standard/basic/README.md | 122 +++++ .../flows/standard/basic/azure_openai.yml | 7 + examples/flows/standard/basic/data.jsonl | 2 + examples/flows/standard/basic/flow.dag.yaml | 25 + examples/flows/standard/basic/hello.jinja2 | 2 + examples/flows/standard/basic/hello.py | 77 +++ .../flows/standard/basic/requirements.txt | 6 + examples/flows/standard/basic/run.yml | 9 + .../flow-with-additional-includes/README.md | 76 +++ .../azure_openai.yml | 7 + .../flow-with-additional-includes/data.jsonl | 3 + .../flow.dag.yaml | 94 ++++ .../requirements.txt | 3 + .../flow-with-additional-includes/run.yml | 4 + .../run_evaluation.yml | 7 + .../.promptflow/flow.tools.json | 30 ++ .../standard/flow-with-symlinks/README.md | 83 ++++ .../flow-with-symlinks/create_symlinks.py | 15 + .../standard/flow-with-symlinks/flow.dag.yaml | 83 ++++ .../flows/standard/intent-copilot/.amlignore | 7 + .../standard/intent-copilot/.env.example | 3 + .../.promptflow/flow.tools.json | 48 ++ .../flows/standard/intent-copilot/README.md | 106 +++++ .../data/denormalized-flat.jsonl | 39 ++ .../intent-copilot/extract_intent_tool.py | 26 ++ .../standard/intent-copilot/flow.dag.yaml | 28 ++ .../flows/standard/intent-copilot/inputs.json | 4 + .../flows/standard/intent-copilot/intent.py | 67 +++ .../standard/intent-copilot/requirements.txt | 6 + .../user_intent_few_shot.jinja2 | 43 ++ .../user_intent_zero_shot.jinja2 | 37 ++ .../summarizing-film-with-autogpt/README.md | 61 +++ .../autogpt_class.py | 144 ++++++ .../autogpt_easy_start.py | 30 ++ .../azure_openai.yml | 7 + .../summarizing-film-with-autogpt/data.jsonl | 1 + .../flow.dag.yaml | 59 +++ .../functions.py | 59 +++ .../generate_goal.py | 15 + .../python_repl.py | 38 ++ .../requirements.txt | 5 + .../system_prompt.jinja2 | 1 + .../triggering_prompt.jinja2 | 1 + .../user_prompt.jinja2 | 1 + .../summarizing-film-with-autogpt/util.py | 163 +++++++ .../wiki_search.py | 62 +++ .../.promptflow/flow.tools.json | 73 +++ .../standard/web-classification/README.md | 107 +++++ .../web-classification/azure_openai.yml | 7 + .../classify_with_llm.jinja2 | 16 + .../web-classification/convert_to_dict.py | 12 + .../standard/web-classification/data.jsonl | 3 + .../fetch_text_content_from_url.py | 30 ++ .../standard/web-classification/flow.dag.yaml | 83 ++++ .../web-classification/prepare_examples.py | 44 ++ .../web-classification/requirements.txt | 4 + .../flows/standard/web-classification/run.yml | 4 + .../web-classification/run_evaluation.yml | 7 + .../summarize_text_content.jinja2 | 5 + .../summarize_text_content__variant_1.jinja2 | 5 + examples/requirements.txt | 8 + examples/setup.sh | 7 + examples/tutorials/flow-deploy/.gitignore | 2 + examples/tutorials/flow-deploy/deploy.md | 44 ++ .../tutorials/flow-deploy/linux/Dockerfile | 25 + .../tutorials/flow-deploy/linux/README.md | 115 +++++ .../flow-deploy/linux/connections_setup.py | 52 +++ .../flow-deploy/linux/docker-compose.yaml | 17 + examples/tutorials/flow-deploy/linux/start.sh | 2 + examples/tutorials/flow-in-pipeline/data.tsv | 21 + .../tutorials/flow-in-pipeline/pipeline.ipynb | 208 +++++++++ .../tsv2jsonl-component/component_spec.yaml | 27 ++ .../tsv2jsonl-component/main.py | 17 + .../get-started/quickstart-azure.ipynb | 440 ++++++++++++++++++ .../tutorials/get-started/quickstart.ipynb | 423 +++++++++++++++++ .../run-management/cloud-run-management.ipynb | 258 ++++++++++ scripts/ghactions_driver/driver.py | 80 ++++ scripts/ghactions_driver/steps.py | 130 ++++++ scripts/workflow_generator.py | 112 +++++ src/promptflow-tools/README.md | 2 +- 141 files changed, 5961 insertions(+), 5 deletions(-) create mode 100644 .github/workflows/configuration.yml create mode 100644 .github/workflows/connections_connection.yml create mode 100644 .github/workflows/tutorials_flowinpipeline_pipeline.yml create mode 100644 .github/workflows/tutorials_getstarted_quickstart.yml create mode 100644 .github/workflows/tutorials_getstarted_quickstartazure.yml create mode 100644 .github/workflows/tutorials_runmanagement_cloudrunmanagement.yml create mode 100644 docs/README.md create mode 100644 examples/README.md create mode 100644 examples/configuration.ipynb create mode 100644 examples/connections/.env.example create mode 100644 examples/connections/README.md create mode 100644 examples/connections/azure_content_safety.yml create mode 100644 examples/connections/azure_openai.yml create mode 100644 examples/connections/cognitive_search.yml create mode 100644 examples/connections/connection.ipynb create mode 100644 examples/connections/custom.yml create mode 100644 examples/connections/openai.yml create mode 100644 examples/connections/serp.yml create mode 100644 examples/dev_requirements.txt create mode 100644 examples/flows/chat/basic-chat/README.md create mode 100644 examples/flows/chat/basic-chat/azure_openai.yml create mode 100644 examples/flows/chat/basic-chat/chat.jinja2 create mode 100644 examples/flows/chat/basic-chat/data.jsonl create mode 100644 examples/flows/chat/basic-chat/flow.dag.yaml create mode 100644 examples/flows/chat/chat-with-wikipedia/README.md create mode 100644 examples/flows/chat/chat-with-wikipedia/augmented_chat.jinja2 create mode 100644 examples/flows/chat/chat-with-wikipedia/azure_openai.yml create mode 100644 examples/flows/chat/chat-with-wikipedia/data.jsonl create mode 100644 examples/flows/chat/chat-with-wikipedia/extract_query_from_question.jinja2 create mode 100644 examples/flows/chat/chat-with-wikipedia/flow.dag.yaml create mode 100644 examples/flows/chat/chat-with-wikipedia/get_wiki_url.py create mode 100644 examples/flows/chat/chat-with-wikipedia/process_search_result.py create mode 100644 examples/flows/chat/chat-with-wikipedia/search_result_from_url.py create mode 100644 examples/flows/evaluation/basic-eval/README.md create mode 100644 examples/flows/evaluation/basic-eval/aggregate.py create mode 100644 examples/flows/evaluation/basic-eval/data.jsonl create mode 100644 examples/flows/evaluation/basic-eval/flow.dag.yaml create mode 100644 examples/flows/evaluation/basic-eval/line_process.py create mode 100644 examples/flows/evaluation/classification-accuracy-eval/README.md create mode 100644 examples/flows/evaluation/classification-accuracy-eval/calculate_accuracy.py create mode 100644 examples/flows/evaluation/classification-accuracy-eval/data.jsonl create mode 100644 examples/flows/evaluation/classification-accuracy-eval/flow.dag.yaml create mode 100644 examples/flows/evaluation/classification-accuracy-eval/grade.py create mode 100644 examples/flows/standard/basic-with-builtin-llm/README.md create mode 100644 examples/flows/standard/basic-with-builtin-llm/azure_openai.yml create mode 100644 examples/flows/standard/basic-with-builtin-llm/data.jsonl create mode 100644 examples/flows/standard/basic-with-builtin-llm/flow.dag.yaml create mode 100644 examples/flows/standard/basic-with-builtin-llm/hello.jinja2 create mode 100644 examples/flows/standard/basic-with-builtin-llm/requirements.txt create mode 100644 examples/flows/standard/basic-with-connection/README.md create mode 100644 examples/flows/standard/basic-with-connection/azure_openai.yml create mode 100644 examples/flows/standard/basic-with-connection/custom.yml create mode 100644 examples/flows/standard/basic-with-connection/data.jsonl create mode 100644 examples/flows/standard/basic-with-connection/flow.dag.yaml create mode 100644 examples/flows/standard/basic-with-connection/hello.jinja2 create mode 100644 examples/flows/standard/basic-with-connection/hello.py create mode 100644 examples/flows/standard/basic-with-connection/requirements.txt create mode 100644 examples/flows/standard/basic/.env.example create mode 100644 examples/flows/standard/basic/README.md create mode 100644 examples/flows/standard/basic/azure_openai.yml create mode 100644 examples/flows/standard/basic/data.jsonl create mode 100644 examples/flows/standard/basic/flow.dag.yaml create mode 100644 examples/flows/standard/basic/hello.jinja2 create mode 100644 examples/flows/standard/basic/hello.py create mode 100644 examples/flows/standard/basic/requirements.txt create mode 100644 examples/flows/standard/basic/run.yml create mode 100644 examples/flows/standard/flow-with-additional-includes/README.md create mode 100644 examples/flows/standard/flow-with-additional-includes/azure_openai.yml create mode 100644 examples/flows/standard/flow-with-additional-includes/data.jsonl create mode 100644 examples/flows/standard/flow-with-additional-includes/flow.dag.yaml create mode 100644 examples/flows/standard/flow-with-additional-includes/requirements.txt create mode 100644 examples/flows/standard/flow-with-additional-includes/run.yml create mode 100644 examples/flows/standard/flow-with-additional-includes/run_evaluation.yml create mode 100644 examples/flows/standard/flow-with-symlinks/.promptflow/flow.tools.json create mode 100644 examples/flows/standard/flow-with-symlinks/README.md create mode 100644 examples/flows/standard/flow-with-symlinks/create_symlinks.py create mode 100644 examples/flows/standard/flow-with-symlinks/flow.dag.yaml create mode 100644 examples/flows/standard/intent-copilot/.amlignore create mode 100644 examples/flows/standard/intent-copilot/.env.example create mode 100644 examples/flows/standard/intent-copilot/.promptflow/flow.tools.json create mode 100644 examples/flows/standard/intent-copilot/README.md create mode 100644 examples/flows/standard/intent-copilot/data/denormalized-flat.jsonl create mode 100644 examples/flows/standard/intent-copilot/extract_intent_tool.py create mode 100644 examples/flows/standard/intent-copilot/flow.dag.yaml create mode 100644 examples/flows/standard/intent-copilot/inputs.json create mode 100644 examples/flows/standard/intent-copilot/intent.py create mode 100644 examples/flows/standard/intent-copilot/requirements.txt create mode 100644 examples/flows/standard/intent-copilot/user_intent_few_shot.jinja2 create mode 100644 examples/flows/standard/intent-copilot/user_intent_zero_shot.jinja2 create mode 100644 examples/flows/standard/summarizing-film-with-autogpt/README.md create mode 100644 examples/flows/standard/summarizing-film-with-autogpt/autogpt_class.py create mode 100644 examples/flows/standard/summarizing-film-with-autogpt/autogpt_easy_start.py create mode 100644 examples/flows/standard/summarizing-film-with-autogpt/azure_openai.yml create mode 100644 examples/flows/standard/summarizing-film-with-autogpt/data.jsonl create mode 100644 examples/flows/standard/summarizing-film-with-autogpt/flow.dag.yaml create mode 100644 examples/flows/standard/summarizing-film-with-autogpt/functions.py create mode 100644 examples/flows/standard/summarizing-film-with-autogpt/generate_goal.py create mode 100644 examples/flows/standard/summarizing-film-with-autogpt/python_repl.py create mode 100644 examples/flows/standard/summarizing-film-with-autogpt/requirements.txt create mode 100644 examples/flows/standard/summarizing-film-with-autogpt/system_prompt.jinja2 create mode 100644 examples/flows/standard/summarizing-film-with-autogpt/triggering_prompt.jinja2 create mode 100644 examples/flows/standard/summarizing-film-with-autogpt/user_prompt.jinja2 create mode 100644 examples/flows/standard/summarizing-film-with-autogpt/util.py create mode 100644 examples/flows/standard/summarizing-film-with-autogpt/wiki_search.py create mode 100644 examples/flows/standard/web-classification/.promptflow/flow.tools.json create mode 100644 examples/flows/standard/web-classification/README.md create mode 100644 examples/flows/standard/web-classification/azure_openai.yml create mode 100644 examples/flows/standard/web-classification/classify_with_llm.jinja2 create mode 100644 examples/flows/standard/web-classification/convert_to_dict.py create mode 100644 examples/flows/standard/web-classification/data.jsonl create mode 100644 examples/flows/standard/web-classification/fetch_text_content_from_url.py create mode 100644 examples/flows/standard/web-classification/flow.dag.yaml create mode 100644 examples/flows/standard/web-classification/prepare_examples.py create mode 100644 examples/flows/standard/web-classification/requirements.txt create mode 100644 examples/flows/standard/web-classification/run.yml create mode 100644 examples/flows/standard/web-classification/run_evaluation.yml create mode 100644 examples/flows/standard/web-classification/summarize_text_content.jinja2 create mode 100644 examples/flows/standard/web-classification/summarize_text_content__variant_1.jinja2 create mode 100644 examples/requirements.txt create mode 100644 examples/setup.sh create mode 100644 examples/tutorials/flow-deploy/.gitignore create mode 100644 examples/tutorials/flow-deploy/deploy.md create mode 100644 examples/tutorials/flow-deploy/linux/Dockerfile create mode 100644 examples/tutorials/flow-deploy/linux/README.md create mode 100644 examples/tutorials/flow-deploy/linux/connections_setup.py create mode 100644 examples/tutorials/flow-deploy/linux/docker-compose.yaml create mode 100644 examples/tutorials/flow-deploy/linux/start.sh create mode 100644 examples/tutorials/flow-in-pipeline/data.tsv create mode 100644 examples/tutorials/flow-in-pipeline/pipeline.ipynb create mode 100644 examples/tutorials/flow-in-pipeline/tsv2jsonl-component/component_spec.yaml create mode 100644 examples/tutorials/flow-in-pipeline/tsv2jsonl-component/main.py create mode 100644 examples/tutorials/get-started/quickstart-azure.ipynb create mode 100644 examples/tutorials/get-started/quickstart.ipynb create mode 100644 examples/tutorials/run-management/cloud-run-management.ipynb create mode 100644 scripts/ghactions_driver/driver.py create mode 100644 scripts/ghactions_driver/steps.py create mode 100644 scripts/workflow_generator.py diff --git a/.github/workflows/configuration.yml b/.github/workflows/configuration.yml new file mode 100644 index 00000000000..157a1b4b629 --- /dev/null +++ b/.github/workflows/configuration.yml @@ -0,0 +1,44 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: configuration +on: + pull_request: + branches: [ main,preview/code-first ] + workflow_dispatch: + +jobs: + configuration: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Generate config.json + run: echo ${{ secrets.TEST_WORKSPACE_CONFIG_JSON }} > ${{ github.workspace }}/examples/config.json + - name: Azure Login + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: Setup Python 3.9 environment + uses: actions/setup-python@v4 + with: + python-version: "3.9" + - name: Prepare requirements + run: | + python -m pip install --upgrade pip + pip install -r ${{ github.workspace }}/examples/requirements.txt + pip install -r ${{ github.workspace }}/examples/dev_requirements.txt + - name: Create Aoai Connection + run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY }}" api_base="${{ secrets.AOAI_API_ENDPOINT }}" + - name: Test Notebook + working-directory: examples + run: | + papermill -k python configuration.ipynb configuration.output.ipynb + - name: Upload artifact + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: artifact + path: examples \ No newline at end of file diff --git a/.github/workflows/connections_connection.yml b/.github/workflows/connections_connection.yml new file mode 100644 index 00000000000..1da48273cec --- /dev/null +++ b/.github/workflows/connections_connection.yml @@ -0,0 +1,44 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: connections_connection +on: + pull_request: + branches: [ main,preview/code-first ] + workflow_dispatch: + +jobs: + connection: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Generate config.json + run: echo ${{ secrets.TEST_WORKSPACE_CONFIG_JSON }} > ${{ github.workspace }}/examples/config.json + - name: Azure Login + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: Setup Python 3.9 environment + uses: actions/setup-python@v4 + with: + python-version: "3.9" + - name: Prepare requirements + run: | + python -m pip install --upgrade pip + pip install -r ${{ github.workspace }}/examples/requirements.txt + pip install -r ${{ github.workspace }}/examples/dev_requirements.txt + - name: Create Aoai Connection + run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY }}" api_base="${{ secrets.AOAI_API_ENDPOINT }}" + - name: Test Notebook + working-directory: examples/connections + run: | + papermill -k python connection.ipynb connection.output.ipynb + - name: Upload artifact + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: artifact + path: examples/connections \ No newline at end of file diff --git a/.github/workflows/tutorials_flowinpipeline_pipeline.yml b/.github/workflows/tutorials_flowinpipeline_pipeline.yml new file mode 100644 index 00000000000..b923837d1d9 --- /dev/null +++ b/.github/workflows/tutorials_flowinpipeline_pipeline.yml @@ -0,0 +1,44 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: tutorials_flowinpipeline_pipeline +on: + pull_request: + branches: [ main,preview/code-first ] + workflow_dispatch: + +jobs: + pipeline: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Generate config.json + run: echo ${{ secrets.TEST_WORKSPACE_CONFIG_JSON }} > ${{ github.workspace }}/examples/config.json + - name: Azure Login + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: Setup Python 3.9 environment + uses: actions/setup-python@v4 + with: + python-version: "3.9" + - name: Prepare requirements + run: | + python -m pip install --upgrade pip + pip install -r ${{ github.workspace }}/examples/requirements.txt + pip install -r ${{ github.workspace }}/examples/dev_requirements.txt + - name: Create Aoai Connection + run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY }}" api_base="${{ secrets.AOAI_API_ENDPOINT }}" + - name: Test Notebook + working-directory: examples/tutorials/flow-in-pipeline + run: | + papermill -k python pipeline.ipynb pipeline.output.ipynb + - name: Upload artifact + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: artifact + path: examples/tutorials/flow-in-pipeline \ No newline at end of file diff --git a/.github/workflows/tutorials_getstarted_quickstart.yml b/.github/workflows/tutorials_getstarted_quickstart.yml new file mode 100644 index 00000000000..9063ceebed6 --- /dev/null +++ b/.github/workflows/tutorials_getstarted_quickstart.yml @@ -0,0 +1,44 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: tutorials_getstarted_quickstart +on: + pull_request: + branches: [ main,preview/code-first ] + workflow_dispatch: + +jobs: + quickstart: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Generate config.json + run: echo ${{ secrets.TEST_WORKSPACE_CONFIG_JSON }} > ${{ github.workspace }}/examples/config.json + - name: Azure Login + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: Setup Python 3.9 environment + uses: actions/setup-python@v4 + with: + python-version: "3.9" + - name: Prepare requirements + run: | + python -m pip install --upgrade pip + pip install -r ${{ github.workspace }}/examples/requirements.txt + pip install -r ${{ github.workspace }}/examples/dev_requirements.txt + - name: Create Aoai Connection + run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY }}" api_base="${{ secrets.AOAI_API_ENDPOINT }}" + - name: Test Notebook + working-directory: examples/tutorials/get-started + run: | + papermill -k python quickstart.ipynb quickstart.output.ipynb + - name: Upload artifact + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: artifact + path: examples/tutorials/get-started \ No newline at end of file diff --git a/.github/workflows/tutorials_getstarted_quickstartazure.yml b/.github/workflows/tutorials_getstarted_quickstartazure.yml new file mode 100644 index 00000000000..7c4c36206d2 --- /dev/null +++ b/.github/workflows/tutorials_getstarted_quickstartazure.yml @@ -0,0 +1,44 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: tutorials_getstarted_quickstartazure +on: + pull_request: + branches: [ main,preview/code-first ] + workflow_dispatch: + +jobs: + quickstart-azure: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Generate config.json + run: echo ${{ secrets.TEST_WORKSPACE_CONFIG_JSON }} > ${{ github.workspace }}/examples/config.json + - name: Azure Login + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: Setup Python 3.9 environment + uses: actions/setup-python@v4 + with: + python-version: "3.9" + - name: Prepare requirements + run: | + python -m pip install --upgrade pip + pip install -r ${{ github.workspace }}/examples/requirements.txt + pip install -r ${{ github.workspace }}/examples/dev_requirements.txt + - name: Create Aoai Connection + run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY }}" api_base="${{ secrets.AOAI_API_ENDPOINT }}" + - name: Test Notebook + working-directory: examples/tutorials/get-started + run: | + papermill -k python quickstart-azure.ipynb quickstart-azure.output.ipynb + - name: Upload artifact + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: artifact + path: examples/tutorials/get-started \ No newline at end of file diff --git a/.github/workflows/tutorials_runmanagement_cloudrunmanagement.yml b/.github/workflows/tutorials_runmanagement_cloudrunmanagement.yml new file mode 100644 index 00000000000..1748a138263 --- /dev/null +++ b/.github/workflows/tutorials_runmanagement_cloudrunmanagement.yml @@ -0,0 +1,44 @@ +# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: tutorials_runmanagement_cloudrunmanagement +on: + pull_request: + branches: [ main,preview/code-first ] + workflow_dispatch: + +jobs: + cloud-run-management: + runs-on: ubuntu-latest + steps: + - name: Checkout repository + uses: actions/checkout@v3 + - name: Generate config.json + run: echo ${{ secrets.TEST_WORKSPACE_CONFIG_JSON }} > ${{ github.workspace }}/examples/config.json + - name: Azure Login + uses: azure/login@v1 + with: + creds: ${{ secrets.AZURE_CREDENTIALS }} + - name: Setup Python 3.9 environment + uses: actions/setup-python@v4 + with: + python-version: "3.9" + - name: Prepare requirements + run: | + python -m pip install --upgrade pip + pip install -r ${{ github.workspace }}/examples/requirements.txt + pip install -r ${{ github.workspace }}/examples/dev_requirements.txt + - name: Create Aoai Connection + run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml --set api_key="${{ secrets.AOAI_API_KEY }}" api_base="${{ secrets.AOAI_API_ENDPOINT }}" + - name: Test Notebook + working-directory: examples/tutorials/run-management + run: | + papermill -k python cloud-run-management.ipynb cloud-run-management.output.ipynb + - name: Upload artifact + if: ${{ always() }} + uses: actions/upload-artifact@v3 + with: + name: artifact + path: examples/tutorials/run-management \ No newline at end of file diff --git a/.gitignore b/.gitignore index 3894b41a4ad..2526387df60 100644 --- a/.gitignore +++ b/.gitignore @@ -82,6 +82,8 @@ target/ profile_default/ ipython_config.py +# Vscode +.vscode/ # pyenv # For a library or package, you might want to ignore these files since the code is # intended to run in multiple environments; otherwise, check them in: @@ -159,5 +161,12 @@ cython_debug/ # option (not recommended) you can uncomment the following to ignore the entire idea folder. #.idea/ +# ignore +# TODO: ignore when PFS supports submit without tools.json +.promptflow +.runs +connection.json +.env +.azureml # secrets **/connections.json diff --git a/README.md b/README.md index 87e405730d6..12ca9f2da62 100644 --- a/README.md +++ b/README.md @@ -21,13 +21,13 @@ With prompt flow, you will be able to: You can develope your flow locally and seamlessly move the experience to azure cloud, learn more: [Azure Machine Learning prompt flow](https://learn.microsoft.com/en-us/azure/machine-learning/prompt-flow/overview-what-is-prompt-flow?view=azureml-api-2). -By joining the Prompt Flow community, you can build AI-first apps faster and have a front-row -peek at how the SDK is being built. Prompt Flow has been released as open-source so that more -pioneering developers can join us in crafting the future of how people building AI applications. +Prompt Flow has been released as open-source so that more pioneering developers can join us in crafting the future of how people building AI applications. ## Get Started with Prompt flow ⚡ -WIP +For users: please start with our [examples](./examples/README.md) & [docs](./docs/README.md). + +For developers: please start with our dev setup guide: [dev_setup.md](./docs/dev/dev_setup.md). ## Contributing diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000000..16229b73621 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,3 @@ +# Promptflow Docs + +WIP. \ No newline at end of file diff --git a/docs/dev/dev_setup.md b/docs/dev/dev_setup.md index 6d3f66599c9..2d2ccfdf5dd 100644 --- a/docs/dev/dev_setup.md +++ b/docs/dev/dev_setup.md @@ -1 +1,2 @@ +# Dev Steup WIP \ No newline at end of file diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 00000000000..670264eeb0c --- /dev/null +++ b/examples/README.md @@ -0,0 +1,64 @@ +# Promptflow examples + +[![code style: black](https://img.shields.io/badge/code%20style-black-000000.svg)](https://github.com/psf/black) +[![license: MIT](https://img.shields.io/badge/License-MIT-purple.svg)](../LICENSE) + +## Prerequisites + +- Bootstrap your python env. + - e.g: create a new [conda](https://conda.io/projects/conda/en/latest/user-guide/getting-started.html) environment. `conda create -n pf-examples python=3.9`. + - install required packages in python environment : `pip install -r requirements.txt` + - show installed sdk: `pip show promptflow` + + +## Examples available + +NOTE: the status in below table is a fake one. WIP. + +**Tutorials** ([tutorials](tutorials)) +path|status|description +-|-|- +[quickstart.ipynb](tutorials/get-started/quickstart.ipynb)|[![tutorials_getstarted_quickstart](https://github.com/microsoft/promptflow/actions/workflows/tutorials_getstarted_quickstart.yml/badge.svg)](https://github.com/microsoft/promptflow/actions/workflows/tutorials_getstarted_quickstart.yml)| get started +[deploy.md](tutorials/flow-deploy/deploy.md)|[![batch-score-rest](https://github.com/Azure/azureml-examples/workflows/cli-scripts-batch-score-rest/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-batch-score-rest.yml)| deploy flow as endpoint +[run.ipynb](tutorials/advanced-run-management/run.ipynb)|[![batch-score-rest](https://github.com/Azure/azureml-examples/workflows/cli-scripts-batch-score-rest/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-batch-score-rest.yml)| advanced flow run management +[quickstart-azure.ipynb](tutorials/get-started/quickstart-azure.ipynb)|[![tutorials_getstarted_quickstartazure](https://github.com/microsoft/promptflow/actions/workflows/tutorials_getstarted_quickstartazure.yml/badge.svg)](https://github.com/microsoft/promptflow/actions/workflows/tutorials_getstarted_quickstartazure.yml)| get started - local to cloud +[pipeline.ipynb](tutorials/flow-in-pipeline/pipeline.ipynb)|[![tutorials_flowinpipeline_pipeline](https://github.com/microsoft/promptflow/actions/workflows/tutorials_flowinpipeline_pipeline.yml/badge.svg)](https://github.com/microsoft/promptflow/actions/workflows/tutorials_flowinpipeline_pipeline.yml)| flow as component in pipeline +[cloud-run-management.ipynb](tutorials/run-management/cloud-run-management.ipynb)|[![tutorials_runmanagement_cloudrunmanagement](https://github.com/microsoft/promptflow/actions/workflows/tutorials_runmanagement_cloudrunmanagement.yml/badge.svg)](https://github.com/microsoft/promptflow/actions/workflows/tutorials_runmanagement_cloudrunmanagement.yml)| cloud run management + + +**Flows** ([flows](flows)) + +[Standard](flows/standard/) flows + +path|status|description +-|-|- +[basic](flows/standard/basic/flow.dag.yaml)|[![batch-score-rest](https://github.com/Azure/azureml-examples/workflows/cli-scripts-batch-score-rest/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-batch-score-rest.yml)| a basic flow with prompt and python tool. +[basic-with-connection](flows/standard/basic-with-connection/flow.dag.yaml)|[![batch-score-rest](https://github.com/Azure/azureml-examples/workflows/cli-scripts-batch-score-rest/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-batch-score-rest.yml)| a basic flow using custom connection with prompt and python tool +[basic-with-builtin-llm](flows/standard/basic-with-builtin-llm/flow.dag.yaml)|[![batch-score-rest](https://github.com/Azure/azureml-examples/workflows/cli-scripts-batch-score-rest/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-batch-score-rest.yml)| a basic flow using builtin llm tool +[intent-copilot](flows/standard/intent-copilot/flow.dag.yaml)|[![batch-score-rest](https://github.com/Azure/azureml-examples/workflows/cli-scripts-batch-score-rest/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-batch-score-rest.yml)| a flow created from existing langchain python code +[flow-with-symlinks](flows/standard/flow-with-symlinks/flow.dag.yaml)|[![batch-score-rest](https://github.com/Azure/azureml-examples/workflows/cli-scripts-batch-score-rest/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-batch-score-rest.yml)| a flow created with external code reference +[web-classification](flows/standard/web-classification/flow.dag.yaml)|[![batch-score-rest](https://github.com/Azure/azureml-examples/workflows/cli-scripts-batch-score-rest/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-batch-score-rest.yml)| a flow demonstrating multi-class classification with LLM. Given an url, it will classify the url into one web category with just a few shots, simple summarization and classification prompts. + + +[Evaluation](flows/evaluation/) flows + +path|status|description +-|-|- +[basic-eval](flows/standard/basic-eval/flow.dag.yaml)|[![batch-score-rest](https://github.com/Azure/azureml-examples/workflows/cli-scripts-batch-score-rest/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-batch-score-rest.yml)| a basic evaluation flow. +[classification-accuracy-eval](flows/standard/classification-accuracy-eval/flow.dag.yaml)|[![batch-score-rest](https://github.com/Azure/azureml-examples/workflows/cli-scripts-batch-score-rest/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-batch-score-rest.yml)| a flow illustrating how to evaluate the performance of a classification system. + +[Chat](flows/chat/) flows +path|status|description +-|-|- +[basic-chat](flows/standard/basic-chat/flow.dag.yaml)|[![batch-score-rest](https://github.com/Azure/azureml-examples/workflows/cli-scripts-batch-score-rest/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-batch-score-rest.yml)| a basic chat flow. +[chat-with-wikipedia](flows/standard/chat-with-wikipedia/flow.dag.yaml)|[![batch-score-rest](https://github.com/Azure/azureml-examples/workflows/cli-scripts-batch-score-rest/badge.svg?branch=main)](https://github.com/Azure/azureml-examples/actions/workflows/cli-scripts-batch-score-rest.yml)| a flow demonstrating Q&A with GPT3.5 using information from Wikipedia to make the answer more grounded. + +**Connections** ([connections](connections)) +path|status|description +-|-|- +[connection.ipynb](connections/connection.ipynb)|[![connections_connection](https://github.com/microsoft/promptflow/actions/workflows/connections_connection.yml/badge.svg)](https://github.com/microsoft/promptflow/actions/workflows/connections_connection.yml)| connections sdk experience + +## Reference + +* [Promptflow public documentation](https://learn.microsoft.com/en-us/azure/machine-learning/prompt-flow/overview-what-is-prompt-flow?view=azureml-api-2) +* [Promptflow internal documentation](https://promptflow.azurewebsites.net/) \ No newline at end of file diff --git a/examples/configuration.ipynb b/examples/configuration.ipynb new file mode 100644 index 00000000000..eb6c13a8cb5 --- /dev/null +++ b/examples/configuration.ipynb @@ -0,0 +1,159 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Configuration\n", + "\n", + "_**Setting up your Azure Machine Learning services workspace and configuring needed resources**_\n", + "\n", + "---\n", + "---\n", + "\n", + "**Requirements** - In order to benefit from this tutorial, you will need:\n", + "- A basic understanding of Machine Learning\n", + "- An Azure account with an active subscription. [Create an account for free](https://azure.microsoft.com/free/?WT.mc_id=A261C142F)\n", + "- An Azure ML workspace\n", + "- A python environment\n", + "- Install dependent packages for samples via `pip install -r requirements-azure.txt`\n", + "- Installed Azure Machine Learning Python SDK v2 - [install instructions](../README.md) - check the getting started section\n", + "\n", + "**Learning Objectives** - By the end of this tutorial, you should be able to:\n", + "- Connect to your AML workspace from the Python SDK using different auth credentials\n", + "- Create workspace config file\n", + "- Create Compute clusters which required by jobs notebooks. [Check this notebook to create a compute cluster](../resources/compute/compute.ipynb)\n", + "\n", + "**Motivations** - This notebook covers the scenario that user define components using yaml then use these components to build pipeline." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 1. Import the required libraries & set dependent environment variables" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import required libraries\n", + "from azure.ai.ml import MLClient" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 2. Configure credential\n", + "\n", + "We are using `DefaultAzureCredential` to get access to workspace. When an access token is needed, it requests one using multiple identities(`EnvironmentCredential, ManagedIdentityCredential, SharedTokenCacheCredential, VisualStudioCodeCredential, AzureCliCredential, AzurePowerShellCredential`) in turn, stopping when one provides a token.\n", + "Reference [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity.defaultazurecredential?view=azure-python) for more information.\n", + "\n", + "`DefaultAzureCredential` should be capable of handling most Azure SDK authentication scenarios. \n", + "Reference [here](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity?view=azure-python) for all available credentials if it does not work for you. " + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.identity import (\n", + " InteractiveBrowserCredential,\n", + " AzureCliCredential,\n", + " DefaultAzureCredential,\n", + ")\n", + "\n", + "try:\n", + " credential = DefaultAzureCredential()\n", + " # Check if given credential can get token successfully.\n", + " credential.get_token(\"https://management.azure.com/.default\")\n", + "except Exception as ex:\n", + " # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential not work\n", + " credential = InteractiveBrowserCredential()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 3. Connect to Azure Machine Learning Workspace\n", + "\n", + "The [workspace](https://docs.microsoft.com/en-us/azure/machine-learning/concept-workspace) is the top-level resource for Azure Machine Learning, providing a centralized place to work with all the artifacts you create when you use Azure Machine Learning. In this section we will connect to the workspace in which the job will be run. \n", + "\n", + "Check this notebook for creating a [workspace](../resources/workspace/workspace.ipynb).\n", + "\n", + "To connect to a workspace, we need identifier parameters - a subscription, resource group and workspace name. \n", + "The config details of a workspace can be saved to a file from the Azure Machine Learning [portal](https://ml.azure.com/). Click on the name of the portal on the top right corner to see the link to save the config file.\n", + "This config file can be used to load a workspace using `MLClient`. If no path is mentioned, path is defaulted to current folder. If no file name is mentioned, file name will be defaulted to `config.json`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " ml_client = MLClient.from_config(credential=credential)\n", + "except Exception as ex:\n", + " # NOTE: Update following workspace information if not correctly configure before\n", + " client_config = {\n", + " \"subscription_id\": \"\",\n", + " \"resource_group\": \"\",\n", + " \"workspace_name\": \"\",\n", + " }\n", + "\n", + " if client_config[\"subscription_id\"].startswith(\"<\"):\n", + " print(\n", + " \"please update your in notebook cell\"\n", + " )\n", + " raise ex\n", + " else: # write and reload from config file\n", + " import json, os\n", + "\n", + " config_path = \"../.azureml/config.json\"\n", + " os.makedirs(os.path.dirname(config_path), exist_ok=True)\n", + " with open(config_path, \"w\") as fo:\n", + " fo.write(json.dumps(client_config))\n", + " ml_client = MLClient.from_config(credential=credential, path=config_path)\n", + "print(ml_client)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# TODO: set up connections" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.17" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/connections/.env.example b/examples/connections/.env.example new file mode 100644 index 00000000000..21c453497d1 --- /dev/null +++ b/examples/connections/.env.example @@ -0,0 +1 @@ +api_key: "" \ No newline at end of file diff --git a/examples/connections/README.md b/examples/connections/README.md new file mode 100644 index 00000000000..604522b7e2e --- /dev/null +++ b/examples/connections/README.md @@ -0,0 +1,38 @@ +## Working with Connection in Prompt Flow +This repository contains example `YAML` files for creating `connection` using prompt-flow cli. Learn more on all the [connections types](https://promptflow.azurewebsites.net/concepts/concept-connections.html). + + +- To create a connection using any of the sample `YAML` files provided in this directory, execute following command: +```bash +# Override keys with --set to avoid yaml file changes +pf connection create -f custom.yml --set configs.key1='abc' +``` + +- To create a custom connection using an `.env` file, execute following command: +```bash +pf connection create -f .env --name custom_connection +``` + +- To list the created connection, execute following command: +```bash +pf connection list +``` + +- To show one connection details, execute following command: +```bash +pf connection show --name custom_connection +``` + +- To update a connection that in workspace, execute following command. Currently only a few fields(description, display_name) support update: +```bash +# Update an existing connection with --set to override values +# Update an azure open ai connection with a new api base +pf connection update -n my_azure_open_ai_connection --set api_base='new_value' +# Update a custom connection +pf connection update -n custom_connection --set configs.key1='abc' secrets.key2='xyz' +``` + +- To delete a connection: +```bash +pf connection delete -n +``` diff --git a/examples/connections/azure_content_safety.yml b/examples/connections/azure_content_safety.yml new file mode 100644 index 00000000000..c50f4fde38b --- /dev/null +++ b/examples/connections/azure_content_safety.yml @@ -0,0 +1,7 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/AzureContentSafetyConnection.schema.json +name: azure_content_safety_connection +type: azure_content_safety +api_key: "" +endpoint: "endpoint" +api_version: "2023-04-30-preview" + diff --git a/examples/connections/azure_openai.yml b/examples/connections/azure_openai.yml new file mode 100644 index 00000000000..5646047bebe --- /dev/null +++ b/examples/connections/azure_openai.yml @@ -0,0 +1,7 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/AzureOpenAIConnection.schema.json +name: azure_open_ai_connection +type: azure_open_ai +api_key: "" +api_base: "aoai-api-endpoint" +api_type: "azure" +api_version: "2023-03-15-preview" diff --git a/examples/connections/cognitive_search.yml b/examples/connections/cognitive_search.yml new file mode 100644 index 00000000000..586fe17cd05 --- /dev/null +++ b/examples/connections/cognitive_search.yml @@ -0,0 +1,6 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/CognitiveSearchConnection.schema.json +name: cognitive_search_connection +type: cognitive_search +api_key: "" +api_base: "endpoint" +api_version: "2023-07-01-Preview" diff --git a/examples/connections/connection.ipynb b/examples/connections/connection.ipynb new file mode 100644 index 00000000000..0bf7bc738a3 --- /dev/null +++ b/examples/connections/connection.ipynb @@ -0,0 +1,236 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Connection\n", + "\n", + "Prompt flow provides various prebuilt connections, including Azure Open AI, Open AI, Azure Content Safety, etc. Prebuilt connections enable seamless integration with these resources within the built-in tools. \n", + "\n", + "Additionally, users have the flexibility to create custom connection types using key-value pairs, empowering them to tailor the connections to their specific requirements, particularly in Python tools.\n", + "\n", + "Reach more details about connection types [here](https://learn.microsoft.com/en-us/azure/machine-learning/prompt-flow/concept-connections?view=azureml-api-2).\n", + "## Create different type of connections\n", + "We will use Azure Open AI connection and custom connection as example to show how to create connection with promptflow sdk." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Install dependent packages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -r ../requirements.txt" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Initialize a pf client" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from promptflow import PFClient\n", + "\n", + "# client can help manage your runs and connections.\n", + "client = PFClient()" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create an Azure Open AI connection\n", + "\n", + "Prepare your Azure Open AI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from promptflow.entities import AzureOpenAIConnection\n", + "\n", + "# Initialize an AzureOpenAIConnection object\n", + "connection = AzureOpenAIConnection(\n", + " name=\"my_azure_open_ai_connection\",\n", + " api_key=\"\",\n", + " api_base=\"\",\n", + ")\n", + "# Create the connection, note that api_key will be scrubbed in the returned result\n", + "result = client.connections.create_or_update(connection)\n", + "print(result)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create a custom connection" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from promptflow.entities import CustomConnection\n", + "\n", + "# Initialize a custom connection object\n", + "connection = CustomConnection(\n", + " name=\"my_custom_connection\",\n", + " # Secrets is a required field for custom connection\n", + " secrets={\"my_key\": \"\"},\n", + " configs={\"endpoint\": \"\", \"other_config\": \"other_value\"},\n", + ")\n", + "# Create the connection, note that all secret values will be scrubbed in the returned result\n", + "result = client.connections.create_or_update(connection)\n", + "print(result)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## List all connections" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "connections = client.connections.list()\n", + "for connection in connections:\n", + " print(connection)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Get a connection by name" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "connection = client.connections.get(name=\"my_custom_connection\")\n", + "print(connection)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Delete a connection by name" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Update a connection\n", + "### Update an Azure Open AI connection" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "connection = client.connections.get(name=\"my_azure_open_ai_connection\")\n", + "connection.api_base = \"new_value\"\n", + "connection.api_key = (\n", + " \"\" # secrets are required again when updating connection using sdk\n", + ")\n", + "result = client.connections.create_or_update(connection)\n", + "print(connection)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Update a custom connection" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "connection = client.connections.get(name=\"my_custom_connection\")\n", + "connection.configs[\"other_config\"] = \"new_value\"\n", + "connection.secrets[\n", + " \"my_key\"\n", + "] = \"new_secret_value\" # ValueError: Connection 'my_custom_connection' secrets ['my_key'] must be filled again when updating it.\n", + "result = client.connections.create_or_update(connection)\n", + "print(connection)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# client.connections.delete(name=\"my_custom_connection\")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "promptflow-sdk", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.17" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/connections/custom.yml b/examples/connections/custom.yml new file mode 100644 index 00000000000..17d5de3f6ab --- /dev/null +++ b/examples/connections/custom.yml @@ -0,0 +1,7 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/CustomConnection.schema.json +name: custom_connection +type: custom +configs: + key1: "test1" +secrets: # required + api-key: "" \ No newline at end of file diff --git a/examples/connections/openai.yml b/examples/connections/openai.yml new file mode 100644 index 00000000000..c536e68e75b --- /dev/null +++ b/examples/connections/openai.yml @@ -0,0 +1,5 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/OpenAIConnection.schema.json +name: open_ai_connection +type: open_ai +api_key: "" +organization: "org" diff --git a/examples/connections/serp.yml b/examples/connections/serp.yml new file mode 100644 index 00000000000..dd4f7ee84d8 --- /dev/null +++ b/examples/connections/serp.yml @@ -0,0 +1,4 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/SerpConnection.schema.json +name: serp_connection +type: serp +api_key: "" diff --git a/examples/dev_requirements.txt b/examples/dev_requirements.txt new file mode 100644 index 00000000000..caa7946ce9e --- /dev/null +++ b/examples/dev_requirements.txt @@ -0,0 +1,7 @@ +# required for notebook sample ci +ipython_genutils +ipykernel +papermill +pyyaml +keyrings.alt +black-nb \ No newline at end of file diff --git a/examples/flows/chat/basic-chat/README.md b/examples/flows/chat/basic-chat/README.md new file mode 100644 index 00000000000..faab5f2ce7a --- /dev/null +++ b/examples/flows/chat/basic-chat/README.md @@ -0,0 +1,63 @@ +# Basic Chat +This example shows how to create a basic chat flow. It demonstrates how to create a chatbot that can remember previous interactions and use the conversation history to generate next message. + +Tools used in this flow： +- `llm` tool + +## What you will learn + +In this flow, you will learn +- how to compose a chat flow. +- prompt template format of LLM tool chat api. Message delimiter is a separate line containing role name and colon: "system:", "user:", "assistant:". +See OpenAI Chat for more about message role. + ```jinja + system: + You are a chatbot having a conversation with a human. + + user: + {{question}} + ``` +- how to consume chat history in prompt. + ```jinja + {% for item in chat_history %} + user: + {{item.inputs.question}} + assistant: + {{item.outputs.answer}} + {% endfor %} + ``` + +## Getting started + +### 1 Create connection for LLM tool to use +Go to "Prompt flow" "Connections" tab. Click on "Create" button, select one of LLM tool supported connection types and fill in the configurations. + +Currently, there are two connection types supported by LLM tool: "AzureOpenAI" and "OpenAI". If you want to use "AzureOpenAI" connection type, you need to create an Azure OpenAI service first. Please refer to [Azure OpenAI Service](https://azure.microsoft.com/en-us/products/cognitive-services/openai-service/) for more details. If you want to use "OpenAI" connection type, you need to create an OpenAI account first. Please refer to [OpenAI](https://platform.openai.com/) for more details. + +```bash +# Override keys with --set to avoid yaml file changes +pf connection create --file azure_openai.yml --set api_key= api_base= +``` + +Note in [flow.dag.yaml](flow.dag.yaml) we are using connection named `azure_open_ai_connection`. +```bash +# show registered connection +pf connection show --name azure_open_ai_connection +``` + +### 2 Start chatting + +```bash +# run chat flow with default question in flow.dag.yaml +pf flow test --flow . + +# run chat flow with new question +pf flow test --flow . --inputs question="What's Azure Machine Learning?" + +# start a interactive chat session in CLI +pf flow test --flow . --interactive + +# start a interactive chat session in CLI with verbose info +pf flow test --flow . --interactive --verbose +``` + diff --git a/examples/flows/chat/basic-chat/azure_openai.yml b/examples/flows/chat/basic-chat/azure_openai.yml new file mode 100644 index 00000000000..5646047bebe --- /dev/null +++ b/examples/flows/chat/basic-chat/azure_openai.yml @@ -0,0 +1,7 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/AzureOpenAIConnection.schema.json +name: azure_open_ai_connection +type: azure_open_ai +api_key: "" +api_base: "aoai-api-endpoint" +api_type: "azure" +api_version: "2023-03-15-preview" diff --git a/examples/flows/chat/basic-chat/chat.jinja2 b/examples/flows/chat/basic-chat/chat.jinja2 new file mode 100644 index 00000000000..c5e811e1969 --- /dev/null +++ b/examples/flows/chat/basic-chat/chat.jinja2 @@ -0,0 +1,12 @@ +system: +You are a helpful assistant. + +{% for item in chat_history %} +user: +{{item.inputs.question}} +assistant: +{{item.outputs.answer}} +{% endfor %} + +user: +{{question}} \ No newline at end of file diff --git a/examples/flows/chat/basic-chat/data.jsonl b/examples/flows/chat/basic-chat/data.jsonl new file mode 100644 index 00000000000..e69de29bb2d diff --git a/examples/flows/chat/basic-chat/flow.dag.yaml b/examples/flows/chat/basic-chat/flow.dag.yaml new file mode 100644 index 00000000000..bd5b0fe6bc3 --- /dev/null +++ b/examples/flows/chat/basic-chat/flow.dag.yaml @@ -0,0 +1,29 @@ +inputs: + chat_history: + type: list + default: [] + question: + type: string + is_chat_input: true + default: What is ChatGPT? +outputs: + answer: + type: string + reference: ${chat.output} + is_chat_output: true +nodes: +- inputs: + deployment_name: gpt-35-turbo + max_tokens: "256" + temperature: "0.7" + chat_history: ${inputs.chat_history} + question: ${inputs.question} + name: chat + type: llm + source: + type: code + path: chat.jinja2 + api: chat + provider: AzureOpenAI + connection: azure_open_ai_connection +node_variants: {} diff --git a/examples/flows/chat/chat-with-wikipedia/README.md b/examples/flows/chat/chat-with-wikipedia/README.md new file mode 100644 index 00000000000..95046fce408 --- /dev/null +++ b/examples/flows/chat/chat-with-wikipedia/README.md @@ -0,0 +1,67 @@ +# Chat With Wikipedia + +This is a companion flow to "Ask Wikipedia". It demonstrates how to create a chatbot that can remember previous interactions and use the conversation history to generate next message. + +Tools used in this flow： +- `llm` tool +- custom `python` Tool + + +## What you will learn + +In this flow, you will learn +- how to compose a chat flow. +- prompt template format of LLM tool chat api. Message delimiter is a separate line containing role name and colon: "system:", "user:", "assistant:". +See OpenAI Chat for more about message role. + ```jinja + system: + You are a chatbot having a conversation with a human. + + user: + {{question}} + ``` +- how to consume chat history in prompt. + ```jinja + {% for item in chat_history %} + user: + {{item.inputs.question}} + assistant: + {{item.outputs.answer}} + {% endfor %} + ``` + +## Getting started + +### 1 Create connection for LLM tool to use +Go to "Prompt flow" "Connections" tab. Click on "Create" button, select one of LLM tool supported connection types and fill in the configurations. + +Currently, there are two connection types supported by LLM tool: "AzureOpenAI" and "OpenAI". If you want to use "AzureOpenAI" connection type, you need to create an Azure OpenAI service first. Please refer to [Azure OpenAI Service](https://azure.microsoft.com/en-us/products/cognitive-services/openai-service/) for more details. If you want to use "OpenAI" connection type, you need to create an OpenAI account first. Please refer to [OpenAI](https://platform.openai.com/) for more details. + +```bash +# Override keys with --set to avoid yaml file changes +pf connection create --file azure_openai.yml --set api_key= api_base= +``` + +Note in [flow.dag.yaml](flow.dag.yaml) we are using connection named `azure_open_ai_connection`. +```bash +# show registered connection +pf connection show --name azure_open_ai_connection +``` + +### 2 Start chatting + +```bash +# run chat flow with default question in flow.dag.yaml +pf flow test --flow . + +# run chat flow with new question +pf flow test --flow . --inputs question="What's Azure Machine Learning?" + +# start a interactive chat session in CLI +pf flow test --flow . --interactive + +# start a interactive chat session in CLI with verbose info +pf flow test --flow . --interactive --verbose +``` + + diff --git a/examples/flows/chat/chat-with-wikipedia/augmented_chat.jinja2 b/examples/flows/chat/chat-with-wikipedia/augmented_chat.jinja2 new file mode 100644 index 00000000000..0719d1fa09a --- /dev/null +++ b/examples/flows/chat/chat-with-wikipedia/augmented_chat.jinja2 @@ -0,0 +1,17 @@ +system: +You are a chatbot having a conversation with a human. +Given the following extracted parts of a long document and a question, create a final answer with references ("SOURCES"). +If you don't know the answer, just say that you don't know. Don't try to make up an answer. +ALWAYS return a "SOURCES" part in your answer. + +{{contexts}} + +{% for item in chat_history %} +user: +{{item.inputs.question}} +assistant: +{{item.outputs.answer}} +{% endfor %} + +user: +{{question}} diff --git a/examples/flows/chat/chat-with-wikipedia/azure_openai.yml b/examples/flows/chat/chat-with-wikipedia/azure_openai.yml new file mode 100644 index 00000000000..5646047bebe --- /dev/null +++ b/examples/flows/chat/chat-with-wikipedia/azure_openai.yml @@ -0,0 +1,7 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/AzureOpenAIConnection.schema.json +name: azure_open_ai_connection +type: azure_open_ai +api_key: "" +api_base: "aoai-api-endpoint" +api_type: "azure" +api_version: "2023-03-15-preview" diff --git a/examples/flows/chat/chat-with-wikipedia/data.jsonl b/examples/flows/chat/chat-with-wikipedia/data.jsonl new file mode 100644 index 00000000000..4cdcac94445 --- /dev/null +++ b/examples/flows/chat/chat-with-wikipedia/data.jsonl @@ -0,0 +1 @@ +{"chat_history":[{"inputs":{"question":"What did OpenAI announce on March 2023?"},"outputs":{"answer":"On March 2023, OpenAI announced the release of GPT-4, a large multimodal model that exhibits human-level performance on various professional and academic benchmarks. This model accepts image and text inputs and emits text outputs. SOURCE: https://en.wikipedia.org/w/index.php?search=GPT-4"}}],"question":"What is the difference between this model and previous ones?"} diff --git a/examples/flows/chat/chat-with-wikipedia/extract_query_from_question.jinja2 b/examples/flows/chat/chat-with-wikipedia/extract_query_from_question.jinja2 new file mode 100644 index 00000000000..c32c569ce40 --- /dev/null +++ b/examples/flows/chat/chat-with-wikipedia/extract_query_from_question.jinja2 @@ -0,0 +1,36 @@ +system: +You are an AI assistant reading the transcript of a conversation between an AI and a human. Given an input question and conversation history, infer user real intent. + +The conversation history is provided just in case of a coreference (e.g. "What is this?" where "this" is defined in previous conversation). + +Return the output as query used for next round user message. + +user: +EXAMPLE +Conversation history: +Human: I want to find the best restaurants nearby, could you recommend some? +AI: Sure, I can help you with that. Here are some of the best restaurants nearby: Rock Bar. +Human: How do I get to Rock Bar? + +Output: directions to Rock Bar +END OF EXAMPLE + +EXAMPLE +Conversation history: +Human: I want to find the best restaurants nearby, could you recommend some? +AI: Sure, I can help you with that. Here are some of the best restaurants nearby: Rock Bar. +Human: How do I get to Rock Bar? +AI: To get to Rock Bar, you need to go to the 52nd floor of the Park A. You can take the subway to Station A and walk for about 8 minutes from exit A53. Alternatively, you can take the train to S Station and walk for about 12 minutes from the south exit3. +Human: Show me more restaurants. + +Output: best restaurants nearby +END OF EXAMPLE + +Conversation history (for reference only): +{% for item in chat_history %} +Human: {{item.inputs.question}} +AI: {{item.outputs.answer}} +{% endfor %} +Human: {{question}} + +Output: diff --git a/examples/flows/chat/chat-with-wikipedia/flow.dag.yaml b/examples/flows/chat/chat-with-wikipedia/flow.dag.yaml new file mode 100644 index 00000000000..f57a5cd7265 --- /dev/null +++ b/examples/flows/chat/chat-with-wikipedia/flow.dag.yaml @@ -0,0 +1,72 @@ +inputs: + chat_history: + type: list + default: [] + question: + type: string + default: What is ChatGPT? + is_chat_input: true +outputs: + answer: + type: string + reference: ${augmented_chat.output} + is_chat_output: true +nodes: +- name: extract_query_from_question + type: llm + source: + type: code + path: extract_query_from_question.jinja2 + inputs: + deployment_name: gpt-35-turbo + temperature: '0.7' + top_p: '1.0' + stop: '' + max_tokens: '256' + presence_penalty: '0' + frequency_penalty: '0' + logit_bias: '' + question: ${inputs.question} + chat_history: ${inputs.chat_history} + provider: AzureOpenAI + connection: azure_open_ai_connection + api: chat +- name: get_wiki_url + type: python + source: + type: code + path: get_wiki_url.py + inputs: + entity: ${extract_query_from_question.output} + count: '2' +- name: search_result_from_url + type: python + source: + type: code + path: search_result_from_url.py + inputs: + url_list: ${get_wiki_url.output} + count: '10' +- name: process_search_result + type: python + source: + type: code + path: process_search_result.py + inputs: + search_result: ${search_result_from_url.output} +- name: augmented_chat + type: llm + source: + type: code + path: augmented_chat.jinja2 + inputs: + deployment_name: gpt-35-turbo + temperature: '0.8' + question: ${inputs.question} + chat_history: ${inputs.chat_history} + contexts: ${process_search_result.output} + provider: AzureOpenAI + connection: azure_open_ai_connection + api: chat +id: chat_with_wikipedia +name: Chat With Wikipedia diff --git a/examples/flows/chat/chat-with-wikipedia/get_wiki_url.py b/examples/flows/chat/chat-with-wikipedia/get_wiki_url.py new file mode 100644 index 00000000000..bc4578f1b21 --- /dev/null +++ b/examples/flows/chat/chat-with-wikipedia/get_wiki_url.py @@ -0,0 +1,57 @@ +import re + +import bs4 +import requests + +from promptflow import tool + + +def decode_str(string): + return string.encode().decode("unicode-escape").encode("latin1").decode("utf-8") + + +def remove_nested_parentheses(string): + pattern = r"\([^()]+\)" + while re.search(pattern, string): + string = re.sub(pattern, "", string) + return string + + +@tool +def get_wiki_url(entity: str, count=2): + # Send a request to the URL + url = f"https://en.wikipedia.org/w/index.php?search={entity}" + url_list = [] + try: + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.35" + } + response = requests.get(url, headers=headers) + if response.status_code == 200: + # Parse the HTML content using BeautifulSoup + soup = bs4.BeautifulSoup(response.text, "html.parser") + mw_divs = soup.find_all("div", {"class": "mw-search-result-heading"}) + if mw_divs: # mismatch + result_titles = [decode_str(div.get_text().strip()) for div in mw_divs] + result_titles = [remove_nested_parentheses(result_title) for result_title in result_titles] + print(f"Could not find {entity}. Similar ententity: {result_titles[:count]}.") + url_list.extend( + [f"https://en.wikipedia.org/w/index.php?search={result_title}" for result_title in result_titles] + ) + else: + page_content = [p_ul.get_text().strip() for p_ul in soup.find_all("p") + soup.find_all("ul")] + if any("may refer to:" in p for p in page_content): + url_list.extend(get_wiki_url("[" + entity + "]")) + else: + url_list.append(url) + else: + msg = ( + f"Get url failed with status code {response.status_code}.\nURL: {url}\nResponse: " + f"{response.text[:100]}" + ) + print(msg) + return url_list[:count] + except Exception as e: + print("Get url failed with error: {}".format(e)) + return url_list diff --git a/examples/flows/chat/chat-with-wikipedia/process_search_result.py b/examples/flows/chat/chat-with-wikipedia/process_search_result.py new file mode 100644 index 00000000000..4248ac0f3c1 --- /dev/null +++ b/examples/flows/chat/chat-with-wikipedia/process_search_result.py @@ -0,0 +1,17 @@ +from promptflow import tool + + +@tool +def process_search_result(search_result): + def format(doc: dict): + return f"Content: {doc['Content']}\nSource: {doc['Source']}" + + try: + context = [] + for url, content in search_result: + context.append({"Content": content, "Source": url}) + context_str = "\n\n".join([format(c) for c in context]) + return context_str + except Exception as e: + print(f"Error: {e}") + return "" diff --git a/examples/flows/chat/chat-with-wikipedia/search_result_from_url.py b/examples/flows/chat/chat-with-wikipedia/search_result_from_url.py new file mode 100644 index 00000000000..f4008199bed --- /dev/null +++ b/examples/flows/chat/chat-with-wikipedia/search_result_from_url.py @@ -0,0 +1,75 @@ +import random +import time +from concurrent.futures import ThreadPoolExecutor +from functools import partial + +import bs4 +import requests + +from promptflow import tool + +session = requests.Session() + + +def decode_str(string): + return string.encode().decode("unicode-escape").encode("latin1").decode("utf-8") + + +def get_page_sentence(page, count: int = 10): + # find all paragraphs + paragraphs = page.split("\n") + paragraphs = [p.strip() for p in paragraphs if p.strip()] + + # find all sentence + sentences = [] + for p in paragraphs: + sentences += p.split(". ") + sentences = [s.strip() + "." for s in sentences if s.strip()] + # get first `count` number of sentences + return " ".join(sentences[:count]) + + +def fetch_text_content_from_url(url: str, count: int = 10): + # Send a request to the URL + try: + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.35" + } + delay = random.uniform(0, 0.5) + time.sleep(delay) + response = session.get(url, headers=headers) + if response.status_code == 200: + # Parse the HTML content using BeautifulSoup + soup = bs4.BeautifulSoup(response.text, "html.parser") + page_content = [p_ul.get_text().strip() for p_ul in soup.find_all("p") + soup.find_all("ul")] + page = "" + for content in page_content: + if len(content.split(" ")) > 2: + page += decode_str(content) + if not content.endswith("\n"): + page += "\n" + text = get_page_sentence(page, count=count) + return (url, text) + else: + msg = ( + f"Get url failed with status code {response.status_code}.\nURL: {url}\nResponse: " + f"{response.text[:100]}" + ) + print(msg) + return (url, "No available content") + + except Exception as e: + print("Get url failed with error: {}".format(e)) + return (url, "No available content") + + +@tool +def search_result_from_url(url_list: list, count: int = 10): + results = [] + patial_func_of_fetch_text_content_from_url = partial(fetch_text_content_from_url, count=count) + with ThreadPoolExecutor(max_workers=5) as executor: + futures = executor.map(patial_func_of_fetch_text_content_from_url, url_list) + for feature in futures: + results.append(feature) + return results diff --git a/examples/flows/evaluation/basic-eval/README.md b/examples/flows/evaluation/basic-eval/README.md new file mode 100644 index 00000000000..756e8102676 --- /dev/null +++ b/examples/flows/evaluation/basic-eval/README.md @@ -0,0 +1,33 @@ +# Basic Eval +This example shows how to create a basic evaluation flow. + +Tools used in this flow： +- `python` tool + +## What you will learn + +In this flow, you will learn +- how to compose a point based evaluation flow, where you can calculate point-wise metrics. +- the way to log metrics. use `from promptflow import log_metric` + - see file [aggregate](aggregate.py). TODO. + +### 1. Test flow with single line data + +Testing flow/node: +```bash +# test with default input value in flow.dag.yaml +pf flow test --flow . + +# test with flow inputs +pf flow test --flow . --inputs groundtruth=groundtruth prediction=prediction + +# test node with inputs +pf flow test --flow . --node line_process --inputs groundtruth=groundtruth prediction=prediction +``` + +### 2. create flow run with multi line data +There are two ways to evaluate an classification flow. + +```bash +pf run create --flow . --data ./data.jsonl --stream +``` \ No newline at end of file diff --git a/examples/flows/evaluation/basic-eval/aggregate.py b/examples/flows/evaluation/basic-eval/aggregate.py new file mode 100644 index 00000000000..496acb0cc42 --- /dev/null +++ b/examples/flows/evaluation/basic-eval/aggregate.py @@ -0,0 +1,38 @@ +from typing import List + +from promptflow import tool + + +@tool +def aggregate(processed_results: List[str], variant_ids: List[str], line_numbers: List[int]): + """ + This tool aggregates the processed result of all lines to the variant level and log metric for each variant. + + :param processed_results: List of the output of line_process node. + :param variant_ids: List of variant ids that can be used to group the results by variant. + :param line_numbers: List of line numbers of the variants. If provided, this can be used to + group the results by line number. + """ + + # Add your aggregation logic here + # aggregated_results should be a dictionary with variant id as key, and for each variant id, it is also a + # dictionary with the metric name as the key and the metric value as the value. + # For example: { + # "variant_0": { + # "metric_name_0": metric_value_0, + # "metric_name_1": metric_value_1, + # ... + # }, + # "variant_1": { + # ... + # }, + # ... + # } + aggregated_results = {} + + # Log metric for each variant + # from promptflow import log_metric + # log_metric(key="", value=aggregated_results[""][""], + # variant_id="") + + return aggregated_results diff --git a/examples/flows/evaluation/basic-eval/data.jsonl b/examples/flows/evaluation/basic-eval/data.jsonl new file mode 100644 index 00000000000..fcd0a4a38b4 --- /dev/null +++ b/examples/flows/evaluation/basic-eval/data.jsonl @@ -0,0 +1 @@ +{"groundtruth": "Tomorrow's weather will be sunny.","prediction": "The weather will be sunny tomorrow.","variant_id": "variant_0","line_number": 0} diff --git a/examples/flows/evaluation/basic-eval/flow.dag.yaml b/examples/flows/evaluation/basic-eval/flow.dag.yaml new file mode 100644 index 00000000000..0066f58ca25 --- /dev/null +++ b/examples/flows/evaluation/basic-eval/flow.dag.yaml @@ -0,0 +1,38 @@ +inputs: + groundtruth: + type: string + default: groundtruth + prediction: + type: string + default: prediction + variant_id: + type: string + default: variant_0 + line_number: + type: int + default: 0 +outputs: + results: + type: string + reference: ${line_process.output} +nodes: +- name: line_process + type: python + source: + type: code + path: line_process.py + inputs: + groundtruth: ${inputs.groundtruth} + prediction: ${inputs.prediction} +- name: aggregate + type: python + source: + type: code + path: aggregate.py + inputs: + processed_results: ${line_process.output} + variant_ids: ${inputs.variant_id} + line_numbers: ${inputs.line_number} + aggregation: true +id: template_eval_flow +name: Template Evaluation Flow diff --git a/examples/flows/evaluation/basic-eval/line_process.py b/examples/flows/evaluation/basic-eval/line_process.py new file mode 100644 index 00000000000..736e66795bf --- /dev/null +++ b/examples/flows/evaluation/basic-eval/line_process.py @@ -0,0 +1,17 @@ +from promptflow import tool + + +@tool +def line_process(groundtruth: str, prediction: str): + """ + This tool processes the prediction of a single line and returns the processed result. + + :param groundtruth: the groundtruth of a single line. + :param prediction: the prediction of a single line. + """ + + processed_result = "" + + # Add your line processing logic here + + return processed_result diff --git a/examples/flows/evaluation/classification-accuracy-eval/README.md b/examples/flows/evaluation/classification-accuracy-eval/README.md new file mode 100644 index 00000000000..9e832fa4499 --- /dev/null +++ b/examples/flows/evaluation/classification-accuracy-eval/README.md @@ -0,0 +1,38 @@ +# Classification Accuracy Evaluation + +This is a flow illustrating how to evaluate the performance of a classification system. It involves comparing each prediction to the groundtruth and assigns a "Correct" or "Incorrect" grade, and aggregating the results to produce metrics such as accuracy, which reflects how good the system is at classifying the data. + +Tools used in this flow： +- `python` tool + +## What you will learn + +In this flow, you will learn +- how to compose a point based evaluation flow, where you can calculate point-wise metrics. +- the way to log metrics. use `from promptflow import log_metric` + - see file [calculate_accuracy.py](calculate_accuracy.py) + +### 1. Test flow/node + +```bash +# test with default input value in flow.dag.yaml +pf flow test --flow . + +# test with flow inputs +pf flow test --flow . --inputs line_number=0 variant_id=variant_0 + +# test node with inputs +pf flow test --flow . --node grade --inputs groundtruth=groundtruth prediction=prediction +``` + +### 2. create flow run with multi line data +There are two ways to evaluate an classification flow. + +```bash +pf run create --flow . --data ./data.jsonl --stream +``` + +### 3. create run against other flow run + +Learn more in [web-classification](../../standard/web-classification/README.md) + diff --git a/examples/flows/evaluation/classification-accuracy-eval/calculate_accuracy.py b/examples/flows/evaluation/classification-accuracy-eval/calculate_accuracy.py new file mode 100644 index 00000000000..56a475672e5 --- /dev/null +++ b/examples/flows/evaluation/classification-accuracy-eval/calculate_accuracy.py @@ -0,0 +1,21 @@ +from typing import List + +from promptflow import log_metric, tool + + +@tool +def calculate_accuracy(grades: List[str], variant_ids: List[str]): + aggregate_grades = {} + for index in range(len(grades)): + grade = grades[index] + variant_id = variant_ids[index] + if variant_id not in aggregate_grades.keys(): + aggregate_grades[variant_id] = [] + aggregate_grades[variant_id].append(grade) + + # calculate accuracy for each variant + for name, values in aggregate_grades.items(): + accuracy = round((values.count("Correct") / len(values)), 2) + log_metric("accuracy", accuracy, variant_id=name) + + return aggregate_grades diff --git a/examples/flows/evaluation/classification-accuracy-eval/data.jsonl b/examples/flows/evaluation/classification-accuracy-eval/data.jsonl new file mode 100644 index 00000000000..b3bbef684e0 --- /dev/null +++ b/examples/flows/evaluation/classification-accuracy-eval/data.jsonl @@ -0,0 +1,3 @@ +{"line_number": 0, "variant_id": "variant_0", "groundtruth": "App","prediction": "App"} +{"line_number": 1, "variant_id": "variant_0", "groundtruth": "Channel","prediction": "Channel"} +{"line_number": 2, "variant_id": "variant_0", "groundtruth": "Academic","prediction": "Academic"} diff --git a/examples/flows/evaluation/classification-accuracy-eval/flow.dag.yaml b/examples/flows/evaluation/classification-accuracy-eval/flow.dag.yaml new file mode 100644 index 00000000000..d0fc6b7e039 --- /dev/null +++ b/examples/flows/evaluation/classification-accuracy-eval/flow.dag.yaml @@ -0,0 +1,39 @@ +inputs: + line_number: + type: int + default: 0 + variant_id: + type: string + default: variant_0 + groundtruth: + type: string + description: Please specify the groundtruth column, which contains the true label + to the outputs that your flow produces. + default: APP + prediction: + type: string + description: Please specify the prediction column, which contains the predicted + outputs that your flow produces. + default: APP +outputs: + grade: + type: string + reference: ${grade.output} +nodes: +- name: grade + type: python + source: + type: code + path: grade.py + inputs: + groundtruth: ${inputs.groundtruth} + prediction: ${inputs.prediction} +- name: calculate_accuracy + type: python + source: + type: code + path: calculate_accuracy.py + inputs: + grades: ${grade.output} + variant_ids: ${inputs.variant_id} + aggregation: true diff --git a/examples/flows/evaluation/classification-accuracy-eval/grade.py b/examples/flows/evaluation/classification-accuracy-eval/grade.py new file mode 100644 index 00000000000..96a1106a048 --- /dev/null +++ b/examples/flows/evaluation/classification-accuracy-eval/grade.py @@ -0,0 +1,6 @@ +from promptflow import tool + + +@tool +def grade(groundtruth: str, prediction: str): + return "Correct" if groundtruth.lower() == prediction.lower() else "Incorrect" diff --git a/examples/flows/standard/basic-with-builtin-llm/README.md b/examples/flows/standard/basic-with-builtin-llm/README.md new file mode 100644 index 00000000000..b2602af2d22 --- /dev/null +++ b/examples/flows/standard/basic-with-builtin-llm/README.md @@ -0,0 +1,66 @@ +# Basic +A basic standard flow that calls azure open ai with Azure OpenAI connection info stored in environment variables. + +Tools used in this flow： +- `prompt` tool +- built-in `llm` tool + +Connections used in this flow: +- `azure_open_ai` connection + +## Prerequisites + +Install promptflow sdk and other dependencies: +```bash +pip install -r requirements.txt +``` + +## Setup connection +Prepare your Azure Open AI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one. + +Ensure you have created `azure_open_ai_connection` connection before. +```bash +pf connection show -n azure_open_ai_connection +``` + +Create connection if you haven't done that. Ensure you have put your azure open ai endpoint key in [azure_openai.yml](azure_openai.yml) file. +```bash +# Override keys with --set to avoid yaml file changes +pf connection create -f azure_openai.yml --set api_key= api_base= +``` + + +## Run flow in local + +### Run locally with single line input + +```bash +# test with default input value in flow.dag.yaml +pf flow test --flow . + +# test with inputs +pf flow test --flow . --inputs text="Hello World!" +``` + +### run with multiple lines data + +- create run +```bash +pf run create --flow . --data ./data.jsonl --stream +``` + +- list and show run meta +```bash +# list created run +pf run list + +# show specific run detail +pf run show --name "basic_with_builtin_llm_default_20230724_160639_803018" + +# show output +pf run show-details --name "basic_with_builtin_llm_default_20230724_160639_803018" + +# visualize run in browser +pf run visualize --name "basic_with_builtin_llm_default_20230724_160639_803018" +``` + diff --git a/examples/flows/standard/basic-with-builtin-llm/azure_openai.yml b/examples/flows/standard/basic-with-builtin-llm/azure_openai.yml new file mode 100644 index 00000000000..f23469369e4 --- /dev/null +++ b/examples/flows/standard/basic-with-builtin-llm/azure_openai.yml @@ -0,0 +1,7 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/AzureOpenAIConnection.schema.json +name: azure_open_ai_connection +type: azure_open_ai +api_key: "" +api_base: "https://.openai.azure.com/" +api_type: "azure" +api_version: "2023-03-15-preview" diff --git a/examples/flows/standard/basic-with-builtin-llm/data.jsonl b/examples/flows/standard/basic-with-builtin-llm/data.jsonl new file mode 100644 index 00000000000..c1f3b7791d2 --- /dev/null +++ b/examples/flows/standard/basic-with-builtin-llm/data.jsonl @@ -0,0 +1,2 @@ +{"text": "Hello World!"} +{"text": "Hello PromptFlow!"} diff --git a/examples/flows/standard/basic-with-builtin-llm/flow.dag.yaml b/examples/flows/standard/basic-with-builtin-llm/flow.dag.yaml new file mode 100644 index 00000000000..3faea94c399 --- /dev/null +++ b/examples/flows/standard/basic-with-builtin-llm/flow.dag.yaml @@ -0,0 +1,29 @@ +inputs: + text: + type: string + default: Hello World! +outputs: + output: + type: string + reference: ${llm.output} +nodes: +- name: hello_prompt + type: prompt + inputs: + text: ${inputs.text} + source: + type: code + path: hello.jinja2 +- name: llm + type: llm + inputs: + prompt: ${hello_prompt.output} + deployment_name: text-davinci-003 + max_tokens: '120' + source: + type: code + path: hello.jinja2 + provider: AzureOpenAI + connection: azure_open_ai_connection + api: completion +node_variants: {} diff --git a/examples/flows/standard/basic-with-builtin-llm/hello.jinja2 b/examples/flows/standard/basic-with-builtin-llm/hello.jinja2 new file mode 100644 index 00000000000..df86ef4f5e7 --- /dev/null +++ b/examples/flows/standard/basic-with-builtin-llm/hello.jinja2 @@ -0,0 +1,2 @@ +{# Please replace the template with your own prompt. #} +Write a simple {{text}} program that displays the greeting message when executed. \ No newline at end of file diff --git a/examples/flows/standard/basic-with-builtin-llm/requirements.txt b/examples/flows/standard/basic-with-builtin-llm/requirements.txt new file mode 100644 index 00000000000..18e9929aa30 --- /dev/null +++ b/examples/flows/standard/basic-with-builtin-llm/requirements.txt @@ -0,0 +1,6 @@ +--extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ +promptflow +promptflow-tools +python-dotenv +langchain +jinja2 \ No newline at end of file diff --git a/examples/flows/standard/basic-with-connection/README.md b/examples/flows/standard/basic-with-connection/README.md new file mode 100644 index 00000000000..4cf2c99564d --- /dev/null +++ b/examples/flows/standard/basic-with-connection/README.md @@ -0,0 +1,101 @@ +# Basic +A basic standard flow that calls azure open ai with Azure OpenAI connection info stored in environment variables. + +Tools used in this flow： +- `prompt` tool +- custom `python` Tool + +Connections used in this flow: +- None + +## Prerequisites + +Install promptflow sdk and other dependencies: +```bash +pip install -r requirements.txt +``` + +## Setup connection +Prepare your Azure Open AI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one. +Ensure you have created `basic_custom_connection` connection before. +```bash +pf connection show -n basic_custom_connection +``` + +Create connection if you haven't done that. +```bash +# Override keys with --set to avoid yaml file changes +pf connection create -f custom.yml --set secrets.api_key= configs.api_base= +``` + +## Run flow in local + +### Run locally with single line input + +```bash +# test with default input value in flow.dag.yaml +pf flow test --flow . + +# test with flow inputs +pf flow test --flow . --inputs text="Hello World!" + +# test node with inputs +pf flow test --flow . --node llm --inputs prompt="Write a simple Hello World! program that displays the greeting message when executed." + +``` + +### Run with multiple lines data + +- create run +```bash +pf run create --flow . --data ./data.jsonl --stream +``` + +- list and show run meta +```bash +# list created run +pf run list -r 3 + +# show specific run detail +pf run show --name "basic_with_connection_default_20230724_160757_016682" + +# show output +pf run show-details --name "basic_with_connection_default_20230724_160757_016682" + +# visualize run in browser +pf run visualize --name "basic_with_connection_default_20230724_160757_016682" +``` + +### Run with connection overwrite + +Ensure you have created `azure_open_ai_connection` connection before. + +```bash +pf connection show -n azure_open_ai_connection +``` + +Create connection if you haven't done that. +```bash +# Override keys with --set to avoid yaml file changes +pf connection create --file azure_openai.yml --set api_key= api_base= +``` + +Run flow with newly created connection. + +```bash +pf run create --flow . --data ./data.jsonl --connections llm.connection=azure_open_ai_connection --stream +``` + +### Run in cloud with connection overwrite + +Ensure you have created `azure_open_ai_connection` connection in cloud. Reference [this notebook](../../../tutorials/get-started/quickstart-azure.ipynb) on how to create connections in cloud with UI. + +Run flow with connection `azure_open_ai_connection`. + +```bash +# set default workspace +az account set -s 96aede12-2f73-41cb-b983-6d11a904839b +az configure --defaults group="promptflow" workspace="promptflow-eastus" + +pfazure run create --flow . --data ./data.jsonl --connections llm.connection=azure_open_ai_connection --stream --runtime demo-mir +``` diff --git a/examples/flows/standard/basic-with-connection/azure_openai.yml b/examples/flows/standard/basic-with-connection/azure_openai.yml new file mode 100644 index 00000000000..5646047bebe --- /dev/null +++ b/examples/flows/standard/basic-with-connection/azure_openai.yml @@ -0,0 +1,7 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/AzureOpenAIConnection.schema.json +name: azure_open_ai_connection +type: azure_open_ai +api_key: "" +api_base: "aoai-api-endpoint" +api_type: "azure" +api_version: "2023-03-15-preview" diff --git a/examples/flows/standard/basic-with-connection/custom.yml b/examples/flows/standard/basic-with-connection/custom.yml new file mode 100644 index 00000000000..e8d96560f3c --- /dev/null +++ b/examples/flows/standard/basic-with-connection/custom.yml @@ -0,0 +1,9 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/CustomConnection.schema.json +name: basic_custom_connection +type: custom +configs: + api_type: azure + api_version: 2023-03-15-preview + api_base: https://.openai.azure.com/ +secrets: # must-have + api_key: diff --git a/examples/flows/standard/basic-with-connection/data.jsonl b/examples/flows/standard/basic-with-connection/data.jsonl new file mode 100644 index 00000000000..c1f3b7791d2 --- /dev/null +++ b/examples/flows/standard/basic-with-connection/data.jsonl @@ -0,0 +1,2 @@ +{"text": "Hello World!"} +{"text": "Hello PromptFlow!"} diff --git a/examples/flows/standard/basic-with-connection/flow.dag.yaml b/examples/flows/standard/basic-with-connection/flow.dag.yaml new file mode 100644 index 00000000000..eb28f0f95dd --- /dev/null +++ b/examples/flows/standard/basic-with-connection/flow.dag.yaml @@ -0,0 +1,26 @@ +inputs: + text: + type: string + default: Hello World! +outputs: + output: + type: string + reference: ${llm.output} +nodes: +- name: hello_prompt + type: prompt + source: + type: code + path: hello.jinja2 + inputs: + text: ${inputs.text} +- name: llm + type: python + source: + type: code + path: hello.py + inputs: + connection: basic_custom_connection + deployment_name: text-davinci-003 + max_tokens: "120" + prompt: ${hello_prompt.output} diff --git a/examples/flows/standard/basic-with-connection/hello.jinja2 b/examples/flows/standard/basic-with-connection/hello.jinja2 new file mode 100644 index 00000000000..df86ef4f5e7 --- /dev/null +++ b/examples/flows/standard/basic-with-connection/hello.jinja2 @@ -0,0 +1,2 @@ +{# Please replace the template with your own prompt. #} +Write a simple {{text}} program that displays the greeting message when executed. \ No newline at end of file diff --git a/examples/flows/standard/basic-with-connection/hello.py b/examples/flows/standard/basic-with-connection/hello.py new file mode 100644 index 00000000000..d8db836ed7e --- /dev/null +++ b/examples/flows/standard/basic-with-connection/hello.py @@ -0,0 +1,71 @@ +from typing import Union +import openai +from dataclasses import asdict + +from promptflow import tool +from promptflow.connections import CustomConnection, AzureOpenAIConnection + +# The inputs section will change based on the arguments of the tool function, after you save the code +# Adding type to arguments and return value will help the system show the types properly +# Please update the function name/signature per need + + +def to_bool(value) -> bool: + return str(value).lower() == "true" + + +@tool +def my_python_tool( + prompt: str, + # for AOAI, deployment name is customized by user, not model name. + deployment_name: str, + suffix: str = None, + max_tokens: int = 120, + temperature: float = 1.0, + top_p: float = 1.0, + n: int = 1, + logprobs: int = None, + echo: bool = False, + stop: list = None, + presence_penalty: float = 0, + frequency_penalty: float = 0, + best_of: int = 1, + logit_bias: dict = {}, + user: str = "", + connection: Union[CustomConnection, AzureOpenAIConnection] = None, + **kwargs, +) -> str: + + # TODO: remove below type conversion after client can pass json rather than string. + echo = to_bool(echo) + if isinstance(connection, AzureOpenAIConnection): + connection_dict = asdict(connection) + else: + # custom connection is dict like object contains the configs and secrets + connection_dict = dict(connection) + + response = openai.Completion.create( + prompt=prompt, + engine=deployment_name, + # empty string suffix should be treated as None. + suffix=suffix if suffix else None, + max_tokens=int(max_tokens), + temperature=float(temperature), + top_p=float(top_p), + n=int(n), + logprobs=int(logprobs) if logprobs else None, + echo=echo, + # fix bug "[] is not valid under any of the given schemas-'stop'" + stop=stop if stop else None, + presence_penalty=float(presence_penalty), + frequency_penalty=float(frequency_penalty), + best_of=int(best_of), + # Logit bias must be a dict if we passed it to openai api. + logit_bias=logit_bias if logit_bias else {}, + user=user, + request_timeout=30, + **dict(connection_dict), + ) + + # get first element because prompt is single. + return response.choices[0].text diff --git a/examples/flows/standard/basic-with-connection/requirements.txt b/examples/flows/standard/basic-with-connection/requirements.txt new file mode 100644 index 00000000000..18e9929aa30 --- /dev/null +++ b/examples/flows/standard/basic-with-connection/requirements.txt @@ -0,0 +1,6 @@ +--extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ +promptflow +promptflow-tools +python-dotenv +langchain +jinja2 \ No newline at end of file diff --git a/examples/flows/standard/basic/.env.example b/examples/flows/standard/basic/.env.example new file mode 100644 index 00000000000..31a426c5287 --- /dev/null +++ b/examples/flows/standard/basic/.env.example @@ -0,0 +1,4 @@ +AZURE_OPENAI_API_KEY= +AZURE_OPENAI_API_BASE=https://.openai.azure.com/ +AZURE_OPENAI_API_TYPE=azure +AZURE_OPENAI_API_VERSION=2023-03-15-preview diff --git a/examples/flows/standard/basic/README.md b/examples/flows/standard/basic/README.md new file mode 100644 index 00000000000..9e4b88c2700 --- /dev/null +++ b/examples/flows/standard/basic/README.md @@ -0,0 +1,122 @@ +# Basic +A basic standard flow that calls azure open ai with Azure OpenAI connection info stored in environment variables. + +Tools used in this flow： +- `prompt` tool +- custom `python` Tool + +Connections used in this flow: +- None + +## Prerequisites + +Install promptflow sdk and other dependencies: +```bash +pip install -r requirements.txt +``` + +## Run flow in local + +- Prepare your Azure Open AI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one. + +- Setup environment variables + +Ensure you have put your azure open ai endpoint key in [.env](.env) file. You can create one refer to this [example file](.env.example). + +```bash +cat .env +``` + +- Test flow/node +```bash +# test with default input value in flow.dag.yaml +pf flow test --flow . + +# test with flow inputs +pf flow test --flow . --inputs text="Hello World!" + +# test node with inputs +pf flow test --flow . --node llm --inputs prompt="Write a simple Hello World! program that displays the greeting message when executed." +``` + +- Create run with multiple lines data +```bash +# using environment from .env file (loaded in user code: hello.py) +pf run create --flow . --data ./data.jsonl --stream +``` + +- List and show run meta +```bash +# list created run +pf run list + +# show specific run detail +pf run show --name "basic_default_20230724_155834_331725" + +# show output +pf run show-details --name "basic_default_20230724_155834_331725" + +# visualize run in browser +pf run visualize --name "basic_default_20230724_155834_331725" +``` + +## Run flow locally with connection +Storing connection info in .env with plaintext is not safe. We recommend to use `pf connection` to guard secrets like `api_key` from leak. + +- Show or create `azure_open_ai_connection` +```bash +# check if connection exists +pf connection show -n azure_open_ai_connection + +# create connection from `azure_openai.yml` file +# Override keys with --set to avoid yaml file changes +pf connection create --file azure_openai.yml --set api_key= api_base= +``` + +- Test using connection secret specified in environment variables +```bash +# test with default input value in flow.dag.yaml +pf flow test --flow . --environment-variables AZURE_OPENAI_API_KEY='${azure_open_ai_connection.api_key}' AZURE_OPENAI_API_BASE='${azure_open_ai_connection.api_base}' +``` + +- Create run using connection secret binding specified in environment variables, see [run.yml](run.yml) +```bash +# create run +pf run create --flow . --data ./data.jsonl --stream --environment-variables AZURE_OPENAI_API_KEY='${azure_open_ai_connection.api_key}' AZURE_OPENAI_API_BASE='${azure_open_ai_connection.api_base}' +# create run using yaml file +pf run create --file run.yml --stream + +# show outputs +pf run show-details --name "basic_default_20230724_160138_517171" +``` + +## Run flow in cloud with connection +- Assume we already have a connection named `azure_open_ai_connection` in workspace. +```bash +# set default workspace +az account set -s 96aede12-2f73-41cb-b983-6d11a904839b +az configure --defaults group="promptflow" workspace="promptflow-eastus" +``` + +- Create run +```bash +# run with environment variable reference connection in azureml workspace +pfazure run create --flow . --data ./data.jsonl --environment-variables AZURE_OPENAI_API_KEY='${azure_open_ai_connection.api_key}' AZURE_OPENAI_API_BASE='${azure_open_ai_connection.api_base}' --stream --runtime demo-mir +# run using yaml file +pfazure run create --file run.yml --stream --runtime demo-mir +``` + +- List and show run meta +```bash +# list created run +pfazure run list -r 3 + +# show specific run detail +pfazure run show --name "basic_default_20230724_160252_071554" + +# show output +pfazure run show-details --name "basic_default_20230724_160252_071554" + +# visualize run in browser +pfazure run visualize --name "basic_default_20230724_160252_071554" +``` \ No newline at end of file diff --git a/examples/flows/standard/basic/azure_openai.yml b/examples/flows/standard/basic/azure_openai.yml new file mode 100644 index 00000000000..5646047bebe --- /dev/null +++ b/examples/flows/standard/basic/azure_openai.yml @@ -0,0 +1,7 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/AzureOpenAIConnection.schema.json +name: azure_open_ai_connection +type: azure_open_ai +api_key: "" +api_base: "aoai-api-endpoint" +api_type: "azure" +api_version: "2023-03-15-preview" diff --git a/examples/flows/standard/basic/data.jsonl b/examples/flows/standard/basic/data.jsonl new file mode 100644 index 00000000000..c1f3b7791d2 --- /dev/null +++ b/examples/flows/standard/basic/data.jsonl @@ -0,0 +1,2 @@ +{"text": "Hello World!"} +{"text": "Hello PromptFlow!"} diff --git a/examples/flows/standard/basic/flow.dag.yaml b/examples/flows/standard/basic/flow.dag.yaml new file mode 100644 index 00000000000..5f8e1f2d022 --- /dev/null +++ b/examples/flows/standard/basic/flow.dag.yaml @@ -0,0 +1,25 @@ +inputs: + text: + type: string + default: Hello World! +outputs: + output: + type: string + reference: ${llm.output} +nodes: +- name: hello_prompt + type: prompt + source: + type: code + path: hello.jinja2 + inputs: + text: ${inputs.text} +- name: llm + type: python + source: + type: code + path: hello.py + inputs: + prompt: ${hello_prompt.output} + deployment_name: text-davinci-003 + max_tokens: "120" diff --git a/examples/flows/standard/basic/hello.jinja2 b/examples/flows/standard/basic/hello.jinja2 new file mode 100644 index 00000000000..df86ef4f5e7 --- /dev/null +++ b/examples/flows/standard/basic/hello.jinja2 @@ -0,0 +1,2 @@ +{# Please replace the template with your own prompt. #} +Write a simple {{text}} program that displays the greeting message when executed. \ No newline at end of file diff --git a/examples/flows/standard/basic/hello.py b/examples/flows/standard/basic/hello.py new file mode 100644 index 00000000000..c6c66719d66 --- /dev/null +++ b/examples/flows/standard/basic/hello.py @@ -0,0 +1,77 @@ +import os +import openai + +from dotenv import load_dotenv +from promptflow import tool + +# The inputs section will change based on the arguments of the tool function, after you save the code +# Adding type to arguments and return value will help the system show the types properly +# Please update the function name/signature per need + + +def to_bool(value) -> bool: + return str(value).lower() == "true" + + +@tool +def my_python_tool( + prompt: str, + # for AOAI, deployment name is customized by user, not model name. + deployment_name: str, + suffix: str = None, + max_tokens: int = 120, + temperature: float = 1.0, + top_p: float = 1.0, + n: int = 1, + logprobs: int = None, + echo: bool = False, + stop: list = None, + presence_penalty: float = 0, + frequency_penalty: float = 0, + best_of: int = 1, + logit_bias: dict = {}, + user: str = "", + **kwargs, +) -> str: + if "AZURE_OPENAI_API_KEY" not in os.environ: + # load environment variables from .env file + load_dotenv() + + if "AZURE_OPENAI_API_KEY" not in os.environ: + raise Exception("Please sepecify environment variables: AZURE_OPENAI_API_KEY") + + conn = dict( + api_key=os.environ["AZURE_OPENAI_API_KEY"], + api_base=os.environ["AZURE_OPENAI_API_BASE"], + api_type=os.environ.get("AZURE_OPENAI_API_TYPE", "azure"), + api_version=os.environ.get("AZURE_OPENAI_API_VERSION", "2023-03-15-preview"), + ) + + # TODO: remove below type conversion after client can pass json rather than string. + echo = to_bool(echo) + + response = openai.Completion.create( + prompt=prompt, + engine=deployment_name, + # empty string suffix should be treated as None. + suffix=suffix if suffix else None, + max_tokens=int(max_tokens), + temperature=float(temperature), + top_p=float(top_p), + n=int(n), + logprobs=int(logprobs) if logprobs else None, + echo=echo, + # fix bug "[] is not valid under any of the given schemas-'stop'" + stop=stop if stop else None, + presence_penalty=float(presence_penalty), + frequency_penalty=float(frequency_penalty), + best_of=int(best_of), + # Logit bias must be a dict if we passed it to openai api. + logit_bias=logit_bias if logit_bias else {}, + user=user, + request_timeout=30, + **conn, + ) + + # get first element because prompt is single. + return response.choices[0].text diff --git a/examples/flows/standard/basic/requirements.txt b/examples/flows/standard/basic/requirements.txt new file mode 100644 index 00000000000..18e9929aa30 --- /dev/null +++ b/examples/flows/standard/basic/requirements.txt @@ -0,0 +1,6 @@ +--extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ +promptflow +promptflow-tools +python-dotenv +langchain +jinja2 \ No newline at end of file diff --git a/examples/flows/standard/basic/run.yml b/examples/flows/standard/basic/run.yml new file mode 100644 index 00000000000..2ea811d5ce1 --- /dev/null +++ b/examples/flows/standard/basic/run.yml @@ -0,0 +1,9 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Run.schema.json +flow: . +data: data.jsonl +environment_variables: + # environment variables from connection + AZURE_OPENAI_API_KEY: ${azure_open_ai_connection.api_key} + AZURE_OPENAI_API_BASE: ${azure_open_ai_connection.api_base} + AZURE_OPENAI_API_TYPE: azure + AZURE_OPENAI_API_VERSION: 2023-03-15-preview \ No newline at end of file diff --git a/examples/flows/standard/flow-with-additional-includes/README.md b/examples/flows/standard/flow-with-additional-includes/README.md new file mode 100644 index 00000000000..6ff821d4f78 --- /dev/null +++ b/examples/flows/standard/flow-with-additional-includes/README.md @@ -0,0 +1,76 @@ +# Flow with additional_includes + +User sometimes need to reference some common files or folders on other top level folders. In this case, use additional +includes to set the additional file or folders used by the flow. The file or folders in additional includes will be +copied to the snapshot folder by promptflow when operate this flow. + +## Tools used in this flow +- LLM Tool +- Python Tool + +## What you will learn + +In this flow, you will learn +- how to add additional includes to the flow + +## Prerequisites + +Install promptflow sdk and other dependencies: +```bash +pip install -r requirements.txt +``` + +## Getting Started + +### 1. Add additional includes to flow + +You can add this field `additional_includes` into the [`flow.dag.yaml`](flow.dag.yaml). +The value of this field is a list of the relative file/folder path to the flow folder. + +``` yaml +additional_includes: +- ../web-classification/classify_with_llm.jinja2 +- ../web-classification/convert_to_dict.py +- ../web-classification/fetch_text_content_from_url.py +- ../web-classification/prepare_examples.py +- ../web-classification/summarize_text_content.jinja2 +- ../web-classification/summarize_text_content__variant_1.jinja2 +``` + +### 2. Test & run the flow with additional includes + +In this sample, this flow will references some files in the [web-classification](../web-classification/README.md) flow. +You can execute this flow with additional_include locally or submit it to cloud. + + +#### Test flow with single line data + +```bash +# test with default input value in flow.dag.yaml +pf flow test --flow . +# test with user specified inputs +pf flow test --flow . --inputs url='https://www.microsoft.com/en-us/d/xbox-wireless-controller-stellar-shift-special-edition/94fbjc7h0h6h' +``` + + +#### Run with multi-line data + +```bash +# create run using command line args +pf run create --flow . --data ./data.jsonl --stream +# create run using yaml file +pf run create --file run.yml --stream +``` +Note: the snapshot folder in run should contain the additional_includes file. + +#### Submit run to cloud + +``` bash +# create run +pfazure run create --flow . --data ./data.jsonl --stream --runtime demo-mir --subscription -g -w +pfazure run create --flow . --data ./data.jsonl --stream # serverless compute +pfazure run create --file run.yml --runtime demo-mir +pfazure run create --file run.yml --stream # serverless compute +``` + +Note: the snapshot folder in run should contain the additional_includes file. Click portal_url of the run to view the final snapshot. \ No newline at end of file diff --git a/examples/flows/standard/flow-with-additional-includes/azure_openai.yml b/examples/flows/standard/flow-with-additional-includes/azure_openai.yml new file mode 100644 index 00000000000..5646047bebe --- /dev/null +++ b/examples/flows/standard/flow-with-additional-includes/azure_openai.yml @@ -0,0 +1,7 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/AzureOpenAIConnection.schema.json +name: azure_open_ai_connection +type: azure_open_ai +api_key: "" +api_base: "aoai-api-endpoint" +api_type: "azure" +api_version: "2023-03-15-preview" diff --git a/examples/flows/standard/flow-with-additional-includes/data.jsonl b/examples/flows/standard/flow-with-additional-includes/data.jsonl new file mode 100644 index 00000000000..87beaaa7268 --- /dev/null +++ b/examples/flows/standard/flow-with-additional-includes/data.jsonl @@ -0,0 +1,3 @@ +{"url": "https://www.youtube.com/watch?v=o5ZQyXaAv1g", "answer": "Channel", "evidence": "Url"} +{"url": "https://arxiv.org/abs/2307.04767", "answer": "Academic", "evidence": "Text content"} +{"url": "https://play.google.com/store/apps/details?id=com.twitter.android", "answer": "App", "evidence": "Both"} diff --git a/examples/flows/standard/flow-with-additional-includes/flow.dag.yaml b/examples/flows/standard/flow-with-additional-includes/flow.dag.yaml new file mode 100644 index 00000000000..22c3a076539 --- /dev/null +++ b/examples/flows/standard/flow-with-additional-includes/flow.dag.yaml @@ -0,0 +1,94 @@ +inputs: + url: + type: string + default: https://www.microsoft.com/en-us/d/xbox-wireless-controller-stellar-shift-special-edition/94fbjc7h0h6h +outputs: + category: + type: string + reference: ${convert_to_dict.output.category} + evidence: + type: string + reference: ${convert_to_dict.output.evidence} +nodes: +- name: fetch_text_content_from_url + type: python + source: + type: code + path: fetch_text_content_from_url.py + inputs: + url: ${inputs.url} +- name: summarize_text_content + use_variants: true +- name: prepare_examples + type: python + source: + type: code + path: prepare_examples.py + inputs: {} +- name: classify_with_llm + type: llm + source: + type: code + path: classify_with_llm.jinja2 + inputs: + deployment_name: text-davinci-003 + max_tokens: '128' + temperature: '0.2' + url: ${inputs.url} + text_content: ${summarize_text_content.output} + examples: ${prepare_examples.output} + provider: AzureOpenAI + connection: azure_open_ai_connection + api: completion +- name: convert_to_dict + type: python + source: + type: code + path: convert_to_dict.py + inputs: + input_str: ${classify_with_llm.output} +id: web_classification +name: Web Classification + +additional_includes: +- ../web-classification/classify_with_llm.jinja2 +- ../web-classification/convert_to_dict.py +- ../web-classification/fetch_text_content_from_url.py +- ../web-classification/prepare_examples.py +- ../web-classification/summarize_text_content.jinja2 +- ../web-classification/summarize_text_content__variant_1.jinja2 + +node_variants: + summarize_text_content: + default_variant_id: variant_0 + variants: + variant_0: + node: + type: llm + source: + type: code + path: summarize_text_content.jinja2 + inputs: + deployment_name: text-davinci-003 + max_tokens: '128' + temperature: '0.2' + text: ${fetch_text_content_from_url.output} + provider: AzureOpenAI + connection: azure_open_ai_connection + api: completion + module: promptflow.tools.aoai + variant_1: + node: + type: llm + source: + type: code + path: summarize_text_content__variant_1.jinja2 + inputs: + deployment_name: text-davinci-003 + max_tokens: '256' + temperature: '0.3' + text: ${fetch_text_content_from_url.output} + provider: AzureOpenAI + connection: azure_open_ai_connection + api: completion + module: promptflow.tools.aoai diff --git a/examples/flows/standard/flow-with-additional-includes/requirements.txt b/examples/flows/standard/flow-with-additional-includes/requirements.txt new file mode 100644 index 00000000000..16306751554 --- /dev/null +++ b/examples/flows/standard/flow-with-additional-includes/requirements.txt @@ -0,0 +1,3 @@ +--extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ +promptflow +promptflow-tools \ No newline at end of file diff --git a/examples/flows/standard/flow-with-additional-includes/run.yml b/examples/flows/standard/flow-with-additional-includes/run.yml new file mode 100644 index 00000000000..9522372f0e0 --- /dev/null +++ b/examples/flows/standard/flow-with-additional-includes/run.yml @@ -0,0 +1,4 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Run.schema.json +flow: . +data: data.jsonl +variant: ${summarize_text_content.variant_1} \ No newline at end of file diff --git a/examples/flows/standard/flow-with-additional-includes/run_evaluation.yml b/examples/flows/standard/flow-with-additional-includes/run_evaluation.yml new file mode 100644 index 00000000000..3b0bcf3d71b --- /dev/null +++ b/examples/flows/standard/flow-with-additional-includes/run_evaluation.yml @@ -0,0 +1,7 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Run.schema.json +flow: ../../evaluation/classification-accuracy-eval +data: data.jsonl +run: web_classification_variant_1_20230724_173442_973403 # replace with your run name +column_mapping: + groundtruth: ${data.answer} + prediction: ${run.outputs.category} \ No newline at end of file diff --git a/examples/flows/standard/flow-with-symlinks/.promptflow/flow.tools.json b/examples/flows/standard/flow-with-symlinks/.promptflow/flow.tools.json new file mode 100644 index 00000000000..cb83d11f500 --- /dev/null +++ b/examples/flows/standard/flow-with-symlinks/.promptflow/flow.tools.json @@ -0,0 +1,30 @@ +{ + "package": {}, + "code": { + "summarize_text_content.jinja2": { + "type": "llm", + "inputs": { + "text": { + "type": [ + "string" + ] + } + }, + "description": "Summarize webpage content into a short paragraph." + }, + "summarize_text_content__variant_1.jinja2": { + "type": "llm", + "inputs": { + "text": { + "type": [ + "string" + ] + } + } + }, + "prepare_examples.py": { + "type": "python", + "function": "prepare_examples" + } + } +} \ No newline at end of file diff --git a/examples/flows/standard/flow-with-symlinks/README.md b/examples/flows/standard/flow-with-symlinks/README.md new file mode 100644 index 00000000000..579668abf96 --- /dev/null +++ b/examples/flows/standard/flow-with-symlinks/README.md @@ -0,0 +1,83 @@ +# Flow with symlinks + +User sometimes need to reference some common files or folders on other top level folders. +In this case, user can use symlink to facilitate development. +But it has the following limitations. It is recommended to use **additional include**. +Learn more: [flow-with-additional-includes](../flow-with-additional-includes/README.md) + +1. For Windows user, by default need Administrator role to create symlinks. +2. For Windows user, directly copy the folder with symlinks, it will deep copy the contents to the location. +3. Need to update the git config to support symlinks. + +**Notes**: +- For Windows user, please grant user permission to [create symbolic links without administrator role](https://learn.microsoft.com/en-us/windows/security/threat-protection/security-policy-settings/create-symbolic-links). + 1. Open your `Local Security Policy` + 2. Find `Local Policies` -> `User Rights Assignment` -> `Create symbolic links` + 3. Add you user name to this policy then reboot the compute. + +**Attention**: +- For git operations, need to set: `git config core.symlinks true` + +## Tools used in this flow +- LLM Tool +- Python Tool + +## What you will learn + +In this flow, you will learn +- how to use symlinks in the flow + +## Prerequisites + +Install promptflow sdk and other dependencies: +```bash +pip install -r requirements.txt +``` + +## Getting Started + +### 1. Create symbolic links in the flow + +```bash +python ./create_symlinks.py +``` + +### 2. Test & run the flow with symlinks + +In this sample, this flow will references some files in the [web-classification](../web-classification/README.md) flow, and assume you already have required connection setup. +You can execute this flow locally or submit it to cloud. + + +#### Test flow with single line data + +```bash +# test flow with default input value in flow.dag.yaml +pf flow test --flow . + +# test flow with input +pf flow test --flow . --inputs url=https://www.youtube.com/watch?v=o5ZQyXaAv1g answer=Channel evidence=Url + +# test node in the flow +pf flow test --flow . --node convert_to_dict --inputs classify_with_llm.output='{"category": "App", "evidence": "URL"}' +``` + + +#### Run with multi-line data + +```bash +# create run using command line args +pf run create --flow . --data ./data.jsonl --stream +# create run using yaml file +pf run create --file run.yml --stream +``` + + +#### Submit run to cloud + +``` bash +# create run +pfazure run create --flow . --data ./data.jsonl --stream --runtime demo-mir --subscription -g -w +pfazure run create --flow . --data ./data.jsonl --stream # serverless compute +pfazure run create --file run.yml --runtime demo-mir --stream +pfazure run create --file run.yml --stream # serverless compute +``` diff --git a/examples/flows/standard/flow-with-symlinks/create_symlinks.py b/examples/flows/standard/flow-with-symlinks/create_symlinks.py new file mode 100644 index 00000000000..e6626d0126b --- /dev/null +++ b/examples/flows/standard/flow-with-symlinks/create_symlinks.py @@ -0,0 +1,15 @@ +import os +from pathlib import Path + +saved_path = os.getcwd() +os.chdir(Path(__file__).parent) + +source_folder = Path("../web-classification") +for file_name in os.listdir(source_folder): + if not Path(file_name).exists(): + os.symlink( + source_folder / file_name, + file_name + ) + +os.chdir(saved_path) diff --git a/examples/flows/standard/flow-with-symlinks/flow.dag.yaml b/examples/flows/standard/flow-with-symlinks/flow.dag.yaml new file mode 100644 index 00000000000..71a72c2d219 --- /dev/null +++ b/examples/flows/standard/flow-with-symlinks/flow.dag.yaml @@ -0,0 +1,83 @@ +inputs: + url: + type: string + default: https://www.microsoft.com/en-us/d/xbox-wireless-controller-stellar-shift-special-edition/94fbjc7h0h6h +outputs: + category: + type: string + reference: ${convert_to_dict.output.category} + evidence: + type: string + reference: ${convert_to_dict.output.evidence} +nodes: +- name: fetch_text_content_from_url + type: python + source: + type: code + path: fetch_text_content_from_url.py + inputs: + url: ${inputs.url} +- name: summarize_text_content + use_variants: true +- name: prepare_examples + type: python + source: + type: code + path: prepare_examples.py + inputs: {} +- name: classify_with_llm + type: llm + source: + type: code + path: classify_with_llm.jinja2 + inputs: + deployment_name: text-davinci-003 + max_tokens: 128 + temperature: 0.2 + url: ${inputs.url} + text_content: ${summarize_text_content.output} + examples: ${prepare_examples.output} + provider: AzureOpenAI + connection: azure_open_ai_connection + api: completion +- name: convert_to_dict + type: python + source: + type: code + path: convert_to_dict.py + inputs: + input_str: ${classify_with_llm.output} +node_variants: + summarize_text_content: + default_variant_id: variant_0 + variants: + variant_0: + node: + type: llm + source: + type: code + path: summarize_text_content.jinja2 + inputs: + deployment_name: text-davinci-003 + max_tokens: 128 + temperature: 0.2 + text: ${fetch_text_content_from_url.output} + provider: AzureOpenAI + connection: azure_open_ai_connection + api: completion + module: promptflow.tools.aoai + variant_1: + node: + type: llm + source: + type: code + path: summarize_text_content__variant_1.jinja2 + inputs: + deployment_name: text-davinci-003 + max_tokens: 256 + temperature: 0.3 + text: ${fetch_text_content_from_url.output} + provider: AzureOpenAI + connection: azure_open_ai_connection + api: completion + module: promptflow.tools.aoai diff --git a/examples/flows/standard/intent-copilot/.amlignore b/examples/flows/standard/intent-copilot/.amlignore new file mode 100644 index 00000000000..f302db186ca --- /dev/null +++ b/examples/flows/standard/intent-copilot/.amlignore @@ -0,0 +1,7 @@ +*.ipynb +.venv/ +.data/ +.env +.vscode/ +outputs/ +connection.json \ No newline at end of file diff --git a/examples/flows/standard/intent-copilot/.env.example b/examples/flows/standard/intent-copilot/.env.example new file mode 100644 index 00000000000..bdffccd929c --- /dev/null +++ b/examples/flows/standard/intent-copilot/.env.example @@ -0,0 +1,3 @@ +CHAT_DEPLOYMENT_NAME=gpt-35-turbo +AZURE_OPENAI_API_KEY= +AZURE_OPENAI_API_BASE=https://.openai.azure.com/ diff --git a/examples/flows/standard/intent-copilot/.promptflow/flow.tools.json b/examples/flows/standard/intent-copilot/.promptflow/flow.tools.json new file mode 100644 index 00000000000..b04cb403c40 --- /dev/null +++ b/examples/flows/standard/intent-copilot/.promptflow/flow.tools.json @@ -0,0 +1,48 @@ +{ + "package": {}, + "code": { + "user_prompt_template": { + "type": "prompt", + "inputs": { + "customer_info": { + "type": [ + "string" + ] + }, + "chat_history": { + "type": [ + "string" + ] + } + }, + "source": "user_intent_zero_shot.jinja2" + }, + "extract_intent_tool.py": { + "type": "python", + "inputs": { + "customer_info": { + "type": [ + "string" + ] + }, + "history": { + "type": [ + "list" + ] + }, + "user_prompt_template": { + "type": [ + "string" + ] + }, + "connection": { + "type": [ + "CustomConnection" + ] + } + }, + "function": "extract_intent_tool", + "source": "extract_intent_tool.py" + } + } +} \ No newline at end of file diff --git a/examples/flows/standard/intent-copilot/README.md b/examples/flows/standard/intent-copilot/README.md new file mode 100644 index 00000000000..f98c6231231 --- /dev/null +++ b/examples/flows/standard/intent-copilot/README.md @@ -0,0 +1,106 @@ +# Intent-copilot +This example shows how to create a flow from existing langchain [code](./intent.py). + +## Prerequisites + +install promptflow sdk and other dependencies: +```bash +pip install -r requirements.txt +``` + +Ensure you have put enough your azure open ai endpoint key in .env file. +```bash +cat .env +``` + +## Run flow in local + +1. init flow directory - create promptflow folder from existing python file +```bash +pf flow init --flow . --entry intent.py --function extract_intent --prompt-template user_prompt_template=user_intent_zero_shot.jinja2 +``` +TODO introduce the generated files + +2. create needed custom connection +```bash +pf connection create -f .env --name custom_connection +``` + +3. test flow locally with single line input +```bash +pf flow test --flow . --input ./data/denormalized-flat.jsonl +``` + +4. run with multiple lines input +```bash +pf run create --flow . --data ./data +``` + +5. list/show + +```bash +# list created run +pf run list +# show run +pf run show --name "intent_copilot_default_20230724_171809_745938" +# show specific run detail, top 3 lines +pf run show-details --name "intent_copilot_default_20230724_171809_745938" -r 3 +``` + +6. evaluation + +```bash +# create evaluation run +pf run create --flow ../../evaluation/classification-accuracy-eval --data ./data --column-mapping groundtruth='${data.intent}' prediction='${run.outputs.output}' --run "intent_copilot_default_20230724_171809_745938" +``` + +```bash +# show run +pf run show --name "classification_accuracy_eval_default_20230724_172154_294669" +# show run output +pf run show-details --name "classification_accuracy_eval_default_20230724_172154_294669" -r 3 +``` + +6. visualize +```bash +# visualize in browser +pf run visualize --name "classification_accuracy_eval_default_20230724_172154_294669" # your evaluation run name +``` + +## Tuning node variant +TODO: Compare the zero_shot & few_shot prompt. + +1. change the dag to include node variants + +2. validate the dag +```bash +pf validate --flow . +``` + +3. run the node_variant +```bash +pf run create --flow . --node_variant node.variant1 +``` + +## Deploy + +### Serve as a local test app + +```bash +pf flow serve --source . --port 5123 --host localhost +``` + +TODO: introduce the browser based test app + +### Export + +#### Export as package + +```bash +pf flow export --source . --format package --path ./package +``` + +#### Export as docker +```bash +pf flow export --source . --format docker --path ./package +``` \ No newline at end of file diff --git a/examples/flows/standard/intent-copilot/data/denormalized-flat.jsonl b/examples/flows/standard/intent-copilot/data/denormalized-flat.jsonl new file mode 100644 index 00000000000..5e2a7461a81 --- /dev/null +++ b/examples/flows/standard/intent-copilot/data/denormalized-flat.jsonl @@ -0,0 +1,39 @@ +{"customer_info": "## Customer_Info\n\nFirst Name: Sarah \nLast Name: Lee \nAge: 38 \nEmail Address: sarahlee@example.com \nPhone Number: 555-867-5309 \nShipping Address: 321 Maple St, Bigtown USA, 90123 \nMembership: Platinum \n\n## Recent_Purchases\n\norder_number: 2 \ndate: 2023-02-10 \nitem:\n- description: TrailMaster X4 Tent, quantity 1, price $250 \n\u00a0 item_number: 1 \n\norder_number: 26 \ndate: 2023-02-05 \nitem:\n- description: CozyNights Sleeping Bag, quantity 1, price $100 \n\u00a0 item_number: 7 \n\norder_number: 35 \ndate: 2023-02-20 \nitem:\n- description: TrailBlaze Hiking Pants, quantity 1, price $75 \n\u00a0 item_number: 10 \n\norder_number: 42 \ndate: 2023-04-06 \nitem:\n- description: TrekMaster Camping Chair, quantity 2, price $100 \n\u00a0 item_number: 12 \n\norder_number: 51 \ndate: 2023-04-21 \nitem:\n- description: SkyView 2-Person Tent, quantity 1, price $200 \n\u00a0 item_number: 15 \n\norder_number: 56 \ndate: 2023-03-26 \nitem:\n- description: RainGuard Hiking Jacket, quantity 1, price $110 \n\u00a0 item_number: 17 \n\norder_number: 65 \ndate: 2023-04-11 \nitem:\n- description: CompactCook Camping Stove, quantity 1, price $60 \n\u00a0 item_number: 20 \n\n", "history": [{"role": "customer", "content": "I recently bought the TrailMaster X4 Tent, and it leaked during a light rain. This is unacceptable! I expected a reliable and waterproof tent, but it failed to deliver. I'm extremely disappointed in the quality."}], "item_number": 1, "order_number": 2, "description": "TrailMaster X4 Tent, quantity 1, price $250", "intent": "product return"} +{"customer_info": "## Customer_Info\n\nFirst Name: John \nLast Name: Smith \nAge: 35 \nEmail Address: johnsmith@example.com \nPhone Number: 555-123-4567 \nShipping Address: 123 Main St, Anytown USA, 12345 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 1 \ndate: 2023-01-05 \nitem:\n- description: TrailMaster X4 Tent, quantity 2, price $500 \n\u00a0 item_number: 1 \n\norder_number: 19 \ndate: 2023-01-25 \nitem:\n- description: BaseCamp Folding Table, quantity 1, price $60 \n\u00a0 item_number: 5 \n\norder_number: 29 \ndate: 2023-02-10 \nitem:\n- description: Alpine Explorer Tent, quantity 2, price $700 \n\u00a0 item_number: 8 \n\norder_number: 41 \ndate: 2023-03-01 \nitem:\n- description: TrekMaster Camping Chair, quantity 1, price $50 \n\u00a0 item_number: 12 \n\norder_number: 50 \ndate: 2023-03-16 \nitem:\n- description: SkyView 2-Person Tent, quantity 2, price $400 \n\u00a0 item_number: 15 \n\norder_number: 59 \ndate: 2023-04-01 \nitem:\n- description: TrekStar Hiking Sandals, quantity 1, price $70 \n\u00a0 item_number: 18 \n\n", "history": [{"role": "customer", "content": "I recently purchased two TrailMaster X4 Tents, and while the overall quality is good, I noticed a minor issue. One of the tent poles arrived slightly bent, making it challenging to assemble the tent properly. I'm concerned that this may affect the stability of the tent. Can you provide any guidance on how to address this?"}], "item_number": 1, "order_number": 1, "description": "TrailMaster X4 Tent, quantity 2, price $500", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Jason \nLast Name: Brown \nAge: 50 \nEmail Address: jasonbrown@example.com \nPhone Number: 555-222-3333 \nShipping Address: 456 Cedar Rd, Anytown USA, 12345 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 8 \ndate: 2023-03-20 \nitem:\n- description: Adventurer Pro Backpack, quantity 1, price $90 \n\u00a0 item_number: 2 \n\norder_number: 27 \ndate: 2023-03-10 \nitem:\n- description: CozyNights Sleeping Bag, quantity 2, price $200 \n\u00a0 item_number: 7 \n\norder_number: 36 \ndate: 2023-03-25 \nitem:\n- description: TrailBlaze Hiking Pants, quantity 2, price $150 \n\u00a0 item_number: 10 \n\norder_number: 43 \ndate: 2023-05-11 \nitem:\n- description: TrekMaster Camping Chair, quantity 1, price $50 \n\u00a0 item_number: 12 \n\norder_number: 52 \ndate: 2023-05-26 \nitem:\n- description: SkyView 2-Person Tent, quantity 1, price $200 \n\u00a0 item_number: 15 \n\norder_number: 57 \ndate: 2023-05-01 \nitem:\n- description: RainGuard Hiking Jacket, quantity 2, price $220 \n\u00a0 item_number: 17 \n\norder_number: 66 \ndate: 2023-05-16 \nitem:\n- description: CompactCook Camping Stove, quantity 2, price $120 \n\u00a0 item_number: 20 \n\n", "history": [{"role": "customer", "content": "I recently purchased the Adventurer Pro Backpack, and I'm excited to use it for my upcoming camping trip. Can you provide some guidance on the backpack's capacity and any special features I should be aware of? I want to make sure I utilize it to its fullest potential."}], "item_number": 2, "order_number": 8, "description": "Adventurer Pro Backpack, quantity 3, price $270", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Daniel \nLast Name: Wilson \nAge: 47 \nEmail Address: danielw@example.com \nPhone Number: 555-444-5555 \nShipping Address: 321 Birch Ln, Smallville USA, 34567 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 9 \ndate: 2023-04-25 \nitem:\n- description: Adventurer Pro Backpack, quantity 3, price $270 \n\u00a0 item_number: 2 \n\norder_number: 13 \ndate: 2023-03-25 \nitem:\n- description: Summit Breeze Jacket, quantity 1, price $120 \n\u00a0 item_number: 3 \n\norder_number: 22 \ndate: 2023-05-07 \nitem:\n- description: BaseCamp Folding Table, quantity 3, price $180 \n\u00a0 item_number: 5 \n\norder_number: 40 \ndate: 2023-04-05 \nitem:\n- description: TrailWalker Hiking Shoes, quantity 1, price $110 \n\u00a0 item_number: 11 \n\norder_number: 49 \ndate: 2023-05-21 \nitem:\n- description: MountainDream Sleeping Bag, quantity 1, price $130 \n\u00a0 item_number: 14 \n\n", "history": [{"role": "customer", "content": "I recently received the Adventurer Pro Backpack I ordered, but there seems to be a problem. One of the zippers on the main compartment is jammed, making it difficult to open and close. This is quite frustrating, as I was looking forward to using it on my upcoming hiking trip."}], "item_number": 2, "order_number": 9, "description": "Adventurer Pro Backpack, quantity 2, price $180", "intent": "product return"} +{"customer_info": "## Customer_Info\n\nFirst Name: Robert \nLast Name: Johnson \nAge: 36 \nEmail Address: robertj@example.com \nPhone Number: 555-555-1212 \nShipping Address: 123 Main St, Anytown USA, 12345 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 10 \ndate: 2023-05-05 \nitem:\n- description: Adventurer Pro Backpack, quantity 2, price $180 \n\u00a0 item_number: 2 \n\n", "history": [{"role": "customer", "content": "I recently purchased two Adventurer Pro Backpacks, and I'm curious to know if they are waterproof. I'm planning to go on a camping trip where we might encounter some rain, and I want to make sure my belongings stay dry."}], "item_number": 2, "order_number": 10, "description": "Adventurer Pro Backpack, quantity 2, price $180", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Michael \nLast Name: Johnson \nAge: 45 \nEmail Address: michaelj@example.com \nPhone Number: 555-555-1212 \nShipping Address: 789 Elm St, Smallville USA, 34567 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 11 \ndate: 2023-01-15 \nitem:\n- description: Summit Breeze Jacket, quantity 1, price $120 \n\u00a0 item_number: 3 \n\norder_number: 20 \ndate: 2023-02-28 \nitem:\n- description: BaseCamp Folding Table, quantity 2, price $120 \n\u00a0 item_number: 5 \n\norder_number: 30 \ndate: 2023-03-15 \nitem:\n- description: Alpine Explorer Tent, quantity 1, price $350 \n\u00a0 item_number: 8 \n\norder_number: 38 \ndate: 2023-02-25 \nitem:\n- description: TrailWalker Hiking Shoes, quantity 1, price $110 \n\u00a0 item_number: 11 \n\norder_number: 47 \ndate: 2023-03-11 \nitem:\n- description: MountainDream Sleeping Bag, quantity 1, price $130 \n\u00a0 item_number: 14 \n\norder_number: 60 \ndate: 2023-05-06 \nitem:\n- description: TrekStar Hiking Sandals, quantity 2, price $140 \n\u00a0 item_number: 18 \n\n", "history": [{"role": "customer", "content": "I recently purchased the Summit Breeze Jacket, and I'm extremely disappointed. The jacket doesn't provide the protection it claims. I wore it during a light rain, and it soaked through within minutes. This is completely unacceptable!"}], "item_number": 3, "order_number": 11, "description": "Summit Breeze Jacket, quantity 1, price $120", "intent": "product return"} +{"customer_info": "## Customer_Info\n\nFirst Name: Melissa \nLast Name: Davis \nAge: 31 \nEmail Address: melissad@example.com \nPhone Number: 555-333-4444 \nShipping Address: 789 Ash St, Another City USA, 67890 \nMembership: Gold \n\n## Recent_Purchases\n\norder_number: 4 \ndate: 2023-04-22 \nitem:\n- description: TrailMaster X4 Tent, quantity 2, price $500 \n\u00a0 item_number: 1 \n\norder_number: 17 \ndate: 2023-03-30 \nitem:\n- description: TrekReady Hiking Boots, quantity 1, price $140 \n\u00a0 item_number: 4 \n\norder_number: 25 \ndate: 2023-04-10 \nitem:\n- description: EcoFire Camping Stove, quantity 1, price $80 \n\u00a0 item_number: 6 \n\norder_number: 34 \ndate: 2023-04-25 \nitem:\n- description: SummitClimber Backpack, quantity 1, price $120 \n\u00a0 item_number: 9 \n\norder_number: 46 \ndate: 2023-05-16 \nitem:\n- description: PowerBurner Camping Stove, quantity 1, price $100 \n\u00a0 item_number: 13 \n\norder_number: 55 \ndate: 2023-05-31 \nitem:\n- description: TrailLite Daypack, quantity 1, price $60 \n\u00a0 item_number: 16 \n\norder_number: 64 \ndate: 2023-06-16 \nitem:\n- description: Adventure Dining Table, quantity 1, price $90 \n\u00a0 item_number: 19 \n\n", "history": [{"role": "customer", "content": "I recently purchased the TrekReady Hiking Boots, and I must say I'm disappointed. The boots started falling apart after just a few uses. The stitching came undone, and the sole started detaching. I expected better quality and durability from these boots. I'm not satisfied with my purchase."}], "item_number": 4, "order_number": 17, "description": "TrekReady Hiking Boots, quantity 1, price $140", "intent": "product return"} +{"customer_info": "## Customer_Info\n\nFirst Name: Emily \nLast Name: Rodriguez \nAge: 29 \nEmail Address: emilyr@example.com \nPhone Number: 555-111-2222 \nShipping Address: 987 Oak Ave, Cityville USA, 56789 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 3 \ndate: 2023-03-18 \nitem:\n- description: TrailMaster X4 Tent, quantity 3, price $750 \n\u00a0 item_number: 1 \n\norder_number: 12 \ndate: 2023-02-20 \nitem:\n- description: Summit Breeze Jacket, quantity 2, price $240 \n\u00a0 item_number: 3 \n\norder_number: 21 \ndate: 2023-04-02 \nitem:\n- description: BaseCamp Folding Table, quantity 1, price $60 \n\u00a0 item_number: 5 \n\norder_number: 31 \ndate: 2023-04-20 \nitem:\n- description: Alpine Explorer Tent, quantity 1, price $350 \n\u00a0 item_number: 8 \n\norder_number: 39 \ndate: 2023-03-30 \nitem:\n- description: TrailWalker Hiking Shoes, quantity 2, price $220 \n\u00a0 item_number: 11 \n\norder_number: 48 \ndate: 2023-04-16 \nitem:\n- description: MountainDream Sleeping Bag, quantity 2, price $260 \n\u00a0 item_number: 14 \n\norder_number: 61 \ndate: 2023-06-11 \nitem:\n- description: TrekStar Hiking Sandals, quantity 1, price $70 \n\u00a0 item_number: 18 \n\n", "history": [{"role": "customer", "content": "I'm interested in purchasing the BaseCamp Folding Table. Can you provide me with more details about its dimensions and weight? I want to make sure it will fit well in my camping gear."}], "item_number": 5, "order_number": 21, "description": "BaseCamp Folding Table, quantity 1, price $60", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Jane \nLast Name: Doe \nAge: 28 \nEmail Address: janedoe@example.com \nPhone Number: 555-987-6543 \nShipping Address: 456 Oak St, Another City USA, 67890 \nMembership: Gold \n\n## Recent_Purchases\n\norder_number: 6 \ndate: 2023-01-10 \nitem:\n- description: Adventurer Pro Backpack, quantity 1, price $90 \n\u00a0 item_number: 2 \n\norder_number: 15 \ndate: 2023-01-20 \nitem:\n- description: TrekReady Hiking Boots, quantity 1, price $140 \n\u00a0 item_number: 4 \n\norder_number: 23 \ndate: 2023-01-30 \nitem:\n- description: EcoFire Camping Stove, quantity 1, price $80 \n\u00a0 item_number: 6 \n\norder_number: 32 \ndate: 2023-02-15 \nitem:\n- description: SummitClimber Backpack, quantity 1, price $120 \n\u00a0 item_number: 9 \n\norder_number: 44 \ndate: 2023-03-06 \nitem:\n- description: PowerBurner Camping Stove, quantity 1, price $100 \n\u00a0 item_number: 13 \n\norder_number: 53 \ndate: 2023-03-21 \nitem:\n- description: TrailLite Daypack, quantity 1, price $60 \n\u00a0 item_number: 16 \n\norder_number: 62 \ndate: 2023-04-06 \nitem:\n- description: Adventure Dining Table, quantity 1, price $90 \n\u00a0 item_number: 19 \n\n", "history": [{"role": "customer", "content": "I recently purchased a camping cooker, but I'm disappointed with its performance. It takes too long to heat up, and the flames seem to be uneven. I expected better quality from this stove."}], "item_number": 6, "order_number": 23, "description": "EcoFire Camping Stove, quantity 1, price $80", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Melissa \nLast Name: Davis \nAge: 31 \nEmail Address: melissad@example.com \nPhone Number: 555-333-4444 \nShipping Address: 789 Ash St, Another City USA, 67890 \nMembership: Gold \n\n## Recent_Purchases\n\norder_number: 4 \ndate: 2023-04-22 \nitem:\n- description: TrailMaster X4 Tent, quantity 2, price $500 \n\u00a0 item_number: 1 \n\norder_number: 17 \ndate: 2023-03-30 \nitem:\n- description: TrekReady Hiking Boots, quantity 1, price $140 \n\u00a0 item_number: 4 \n\norder_number: 25 \ndate: 2023-04-10 \nitem:\n- description: EcoFire Camping Stove, quantity 1, price $80 \n\u00a0 item_number: 6 \n\norder_number: 34 \ndate: 2023-04-25 \nitem:\n- description: SummitClimber Backpack, quantity 1, price $120 \n\u00a0 item_number: 9 \n\norder_number: 46 \ndate: 2023-05-16 \nitem:\n- description: PowerBurner Camping Stove, quantity 1, price $100 \n\u00a0 item_number: 13 \n\norder_number: 55 \ndate: 2023-05-31 \nitem:\n- description: TrailLite Daypack, quantity 1, price $60 \n\u00a0 item_number: 16 \n\norder_number: 64 \ndate: 2023-06-16 \nitem:\n- description: Adventure Dining Table, quantity 1, price $90 \n\u00a0 item_number: 19 \n\n", "history": [{"role": "customer", "content": "I'm interested in purchasing the a camping cooker. Can you tell me more about its fuel efficiency and cooking capacity? I want to make sure it will suit my camping needs."}], "item_number": 6, "order_number": 25, "description": "EcoFire Camping Stove, quantity 1, price $80", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Sarah \nLast Name: Lee \nAge: 38 \nEmail Address: sarahlee@example.com \nPhone Number: 555-867-5309 \nShipping Address: 321 Maple St, Bigtown USA, 90123 \nMembership: Platinum \n\n## Recent_Purchases\n\norder_number: 2 \ndate: 2023-02-10 \nitem:\n- description: TrailMaster X4 Tent, quantity 1, price $250 \n\u00a0 item_number: 1 \n\norder_number: 26 \ndate: 2023-02-05 \nitem:\n- description: CozyNights Sleeping Bag, quantity 1, price $100 \n\u00a0 item_number: 7 \n\norder_number: 35 \ndate: 2023-02-20 \nitem:\n- description: TrailBlaze Hiking Pants, quantity 1, price $75 \n\u00a0 item_number: 10 \n\norder_number: 42 \ndate: 2023-04-06 \nitem:\n- description: TrekMaster Camping Chair, quantity 2, price $100 \n\u00a0 item_number: 12 \n\norder_number: 51 \ndate: 2023-04-21 \nitem:\n- description: SkyView 2-Person Tent, quantity 1, price $200 \n\u00a0 item_number: 15 \n\norder_number: 56 \ndate: 2023-03-26 \nitem:\n- description: RainGuard Hiking Jacket, quantity 1, price $110 \n\u00a0 item_number: 17 \n\norder_number: 65 \ndate: 2023-04-11 \nitem:\n- description: CompactCook Camping Stove, quantity 1, price $60 \n\u00a0 item_number: 20 \n\n", "history": [{"role": "customer", "content": "Hi, I recently bought a sleeping Bag, and I'm having some trouble with the zipper. It seems to get stuck every time I try to close it. What should I do?"}], "item_number": 7, "order_number": 26, "description": "CozyNights Sleeping Bag, quantity 1, price $100", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Amanda \nLast Name: Perez \nAge: 26 \nEmail Address: amandap@example.com \nPhone Number: 555-123-4567 \nShipping Address: 654 Pine St, Suburbia USA, 23456 \nMembership: Gold \n\n## Recent_Purchases\n\norder_number: 5 \ndate: 2023-05-01 \nitem:\n- description: TrailMaster X4 Tent, quantity 1, price $250 \n\u00a0 item_number: 1 \n\norder_number: 18 \ndate: 2023-05-04 \nitem:\n- description: TrekReady Hiking Boots, quantity 3, price $420 \n\u00a0 item_number: 4 \n\norder_number: 28 \ndate: 2023-04-15 \nitem:\n- description: CozyNights Sleeping Bag, quantity 1, price $100 \n\u00a0 item_number: 7 \n\norder_number: 37 \ndate: 2023-04-30 \nitem:\n- description: TrailBlaze Hiking Pants, quantity 1, price $75 \n\u00a0 item_number: 10 \n\norder_number: 58 \ndate: 2023-06-06 \nitem:\n- description: RainGuard Hiking Jacket, quantity 1, price $110 \n\u00a0 item_number: 17 \n\norder_number: 67 \ndate: 2023-06-21 \nitem:\n- description: CompactCook Camping Stove, quantity 1, price $60 \n\u00a0 item_number: 20 \n\n", "history": [{"role": "customer", "content": "I received the sleeping bag I ordered, but it's not the color I chose. I specifically selected the blue one, but I got a green one instead. This is really disappointing!"}], "item_number": 7, "order_number": 28, "description": "CozyNights Sleeping Bag, quantity 2, price $100", "intent": "product exchange"} +{"customer_info": "## Customer_Info\n\nFirst Name: John \nLast Name: Smith \nAge: 35 \nEmail Address: johnsmith@example.com \nPhone Number: 555-123-4567 \nShipping Address: 123 Main St, Anytown USA, 12345 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 1 \ndate: 2023-01-05 \nitem:\n- description: TrailMaster X4 Tent, quantity 2, price $500 \n\u00a0 item_number: 1 \n\norder_number: 19 \ndate: 2023-01-25 \nitem:\n- description: BaseCamp Folding Table, quantity 1, price $60 \n\u00a0 item_number: 5 \n\norder_number: 29 \ndate: 2023-02-10 \nitem:\n- description: Alpine Explorer Tent, quantity 2, price $700 \n\u00a0 item_number: 8 \n\norder_number: 41 \ndate: 2023-03-01 \nitem:\n- description: TrekMaster Camping Chair, quantity 1, price $50 \n\u00a0 item_number: 12 \n\norder_number: 50 \ndate: 2023-03-16 \nitem:\n- description: SkyView 2-Person Tent, quantity 2, price $400 \n\u00a0 item_number: 15 \n\norder_number: 59 \ndate: 2023-04-01 \nitem:\n- description: TrekStar Hiking Sandals, quantity 1, price $70 \n\u00a0 item_number: 18 \n\n", "history": [{"role": "customer", "content": "Hi there! I recently purchased two Tents from your store. They look great, but I wanted to know if they come with any additional accessories like stakes or a rainfly?"}], "item_number": 8, "order_number": 29, "description": "Alpine Explorer Tents, quantity 2, price $700", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Michael \nLast Name: Johnson \nAge: 45 \nEmail Address: michaelj@example.com \nPhone Number: 555-555-1212 \nShipping Address: 789 Elm St, Smallville USA, 34567 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 11 \ndate: 2023-01-15 \nitem:\n- description: Summit Breeze Jacket, quantity 1, price $120 \n\u00a0 item_number: 3 \n\norder_number: 20 \ndate: 2023-02-28 \nitem:\n- description: BaseCamp Folding Table, quantity 2, price $120 \n\u00a0 item_number: 5 \n\norder_number: 30 \ndate: 2023-03-15 \nitem:\n- description: Alpine Explorer Tent, quantity 1, price $350 \n\u00a0 item_number: 8 \n\norder_number: 38 \ndate: 2023-02-25 \nitem:\n- description: TrailWalker Hiking Shoes, quantity 1, price $110 \n\u00a0 item_number: 11 \n\norder_number: 47 \ndate: 2023-03-11 \nitem:\n- description: MountainDream Sleeping Bag, quantity 1, price $130 \n\u00a0 item_number: 14 \n\norder_number: 60 \ndate: 2023-05-06 \nitem:\n- description: TrekStar Hiking Sandals, quantity 2, price $140 \n\u00a0 item_number: 18 \n\n", "history": [{"role": "customer", "content": "I recently bought a Tent, and I have to say, I'm really disappointed. The tent poles seem flimsy, and the zippers are constantly getting stuck. It's not what I expected from a high-end tent."}], "item_number": 8, "order_number": 30, "description": "Alpine Explorer Tents, quantity 1, price $350", "intent": "product return"} +{"customer_info": "## Customer_Info\n\nFirst Name: Emily \nLast Name: Rodriguez \nAge: 29 \nEmail Address: emilyr@example.com \nPhone Number: 555-111-2222 \nShipping Address: 987 Oak Ave, Cityville USA, 56789 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 3 \ndate: 2023-03-18 \nitem:\n- description: TrailMaster X4 Tent, quantity 3, price $750 \n\u00a0 item_number: 1 \n\norder_number: 12 \ndate: 2023-02-20 \nitem:\n- description: Summit Breeze Jacket, quantity 2, price $240 \n\u00a0 item_number: 3 \n\norder_number: 21 \ndate: 2023-04-02 \nitem:\n- description: BaseCamp Folding Table, quantity 1, price $60 \n\u00a0 item_number: 5 \n\norder_number: 31 \ndate: 2023-04-20 \nitem:\n- description: Alpine Explorer Tent, quantity 1, price $350 \n\u00a0 item_number: 8 \n\norder_number: 39 \ndate: 2023-03-30 \nitem:\n- description: TrailWalker Hiking Shoes, quantity 2, price $220 \n\u00a0 item_number: 11 \n\norder_number: 48 \ndate: 2023-04-16 \nitem:\n- description: MountainDream Sleeping Bag, quantity 2, price $260 \n\u00a0 item_number: 14 \n\norder_number: 61 \ndate: 2023-06-11 \nitem:\n- description: TrekStar Hiking Sandals, quantity 1, price $70 \n\u00a0 item_number: 18 \n\n", "history": [{"role": "customer", "content": "I recently received the tent I ordered, but I'm having trouble setting it up. The instructions provided are a bit unclear. Can you guide me through the setup process?"}], "item_number": 8, "order_number": 31, "description": "Alpine Explorer Tents, quantity 1, price $350", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Jane \nLast Name: Doe \nAge: 28 \nEmail Address: janedoe@example.com \nPhone Number: 555-987-6543 \nShipping Address: 456 Oak St, Another City USA, 67890 \nMembership: Gold \n\n## Recent_Purchases\n\norder_number: 6 \ndate: 2023-01-10 \nitem:\n- description: Adventurer Pro Backpack, quantity 1, price $90 \n\u00a0 item_number: 2 \n\norder_number: 15 \ndate: 2023-01-20 \nitem:\n- description: TrekReady Hiking Boots, quantity 1, price $140 \n\u00a0 item_number: 4 \n\norder_number: 23 \ndate: 2023-01-30 \nitem:\n- description: EcoFire Camping Stove, quantity 1, price $80 \n\u00a0 item_number: 6 \n\norder_number: 32 \ndate: 2023-02-15 \nitem:\n- description: SummitClimber Backpack, quantity 1, price $120 \n\u00a0 item_number: 9 \n\norder_number: 44 \ndate: 2023-03-06 \nitem:\n- description: PowerBurner Camping Stove, quantity 1, price $100 \n\u00a0 item_number: 13 \n\norder_number: 53 \ndate: 2023-03-21 \nitem:\n- description: TrailLite Daypack, quantity 1, price $60 \n\u00a0 item_number: 16 \n\norder_number: 62 \ndate: 2023-04-06 \nitem:\n- description: Adventure Dining Table, quantity 1, price $90 \n\u00a0 item_number: 19 \n\n", "history": [{"role": "customer", "content": "Hi, I recently purchased the a backpack from your store. It looks great, but I'm wondering if it has a separate compartment for a hydration bladder?"}], "item_number": 9, "order_number": 32, "description": "SummitClimber Backpack, quantity 1, price $120", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Melissa \nLast Name: Davis \nAge: 31 \nEmail Address: melissad@example.com \nPhone Number: 555-333-4444 \nShipping Address: 789 Ash St, Another City USA, 67890 \nMembership: Gold \n\n## Recent_Purchases\n\norder_number: 4 \ndate: 2023-04-22 \nitem:\n- description: TrailMaster X4 Tent, quantity 2, price $500 \n\u00a0 item_number: 1 \n\norder_number: 17 \ndate: 2023-03-30 \nitem:\n- description: TrekReady Hiking Boots, quantity 1, price $140 \n\u00a0 item_number: 4 \n\norder_number: 25 \ndate: 2023-04-10 \nitem:\n- description: EcoFire Camping Stove, quantity 1, price $80 \n\u00a0 item_number: 6 \n\norder_number: 34 \ndate: 2023-04-25 \nitem:\n- description: SummitClimber Backpack, quantity 1, price $120 \n\u00a0 item_number: 9 \n\norder_number: 46 \ndate: 2023-05-16 \nitem:\n- description: PowerBurner Camping Stove, quantity 1, price $100 \n\u00a0 item_number: 13 \n\norder_number: 55 \ndate: 2023-05-31 \nitem:\n- description: TrailLite Daypack, quantity 1, price $60 \n\u00a0 item_number: 16 \n\norder_number: 64 \ndate: 2023-06-16 \nitem:\n- description: Adventure Dining Table, quantity 1, price $90 \n\u00a0 item_number: 19 \n\n", "history": [{"role": "customer", "content": "I recently received the Backpack I ordered, and I noticed that one of the zippers is not closing smoothly. It seems to get stuck halfway. Is there anything I can do to fix it?"}], "item_number": 9, "order_number": 34, "description": "SummitClimber Backpack, quantity 1, price $120", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Sarah \nLast Name: Lee \nAge: 38 \nEmail Address: sarahlee@example.com \nPhone Number: 555-867-5309 \nShipping Address: 321 Maple St, Bigtown USA, 90123 \nMembership: Platinum \n\n## Recent_Purchases\n\norder_number: 2 \ndate: 2023-02-10 \nitem:\n- description: TrailMaster X4 Tent, quantity 1, price $250 \n\u00a0 item_number: 1 \n\norder_number: 26 \ndate: 2023-02-05 \nitem:\n- description: CozyNights Sleeping Bag, quantity 1, price $100 \n\u00a0 item_number: 7 \n\norder_number: 35 \ndate: 2023-02-20 \nitem:\n- description: TrailBlaze Hiking Pants, quantity 1, price $75 \n\u00a0 item_number: 10 \n\norder_number: 42 \ndate: 2023-04-06 \nitem:\n- description: TrekMaster Camping Chair, quantity 2, price $100 \n\u00a0 item_number: 12 \n\norder_number: 51 \ndate: 2023-04-21 \nitem:\n- description: SkyView 2-Person Tent, quantity 1, price $200 \n\u00a0 item_number: 15 \n\norder_number: 56 \ndate: 2023-03-26 \nitem:\n- description: RainGuard Hiking Jacket, quantity 1, price $110 \n\u00a0 item_number: 17 \n\norder_number: 65 \ndate: 2023-04-11 \nitem:\n- description: CompactCook Camping Stove, quantity 1, price $60 \n\u00a0 item_number: 20 \n\n", "history": [{"role": "customer", "content": "I recently purchased the Pants from your store, but I'm disappointed with the fit. They're too tight around the waist, and the length is too short. Can I exchange them for a different size?"}], "item_number": 10, "order_number": 35, "description": "TrailBlaze Hiking Pants, quantity 1, price $75", "intent": "product return"} +{"customer_info": "## Customer_Info\n\nFirst Name: Amanda \nLast Name: Perez \nAge: 26 \nEmail Address: amandap@example.com \nPhone Number: 555-123-4567 \nShipping Address: 654 Pine St, Suburbia USA, 23456 \nMembership: Gold \n\n## Recent_Purchases\n\norder_number: 5 \ndate: 2023-05-01 \nitem:\n- description: TrailMaster X4 Tent, quantity 1, price $250 \n\u00a0 item_number: 1 \n\norder_number: 18 \ndate: 2023-05-04 \nitem:\n- description: TrekReady Hiking Boots, quantity 3, price $420 \n\u00a0 item_number: 4 \n\norder_number: 28 \ndate: 2023-04-15 \nitem:\n- description: CozyNights Sleeping Bag, quantity 1, price $100 \n\u00a0 item_number: 7 \n\norder_number: 37 \ndate: 2023-04-30 \nitem:\n- description: TrailBlaze Hiking Pants, quantity 1, price $75 \n\u00a0 item_number: 10 \n\norder_number: 58 \ndate: 2023-06-06 \nitem:\n- description: RainGuard Hiking Jacket, quantity 1, price $110 \n\u00a0 item_number: 17 \n\norder_number: 67 \ndate: 2023-06-21 \nitem:\n- description: CompactCook Camping Stove, quantity 1, price $60 \n\u00a0 item_number: 20 \n\n", "history": [{"role": "customer", "content": "I recently received the Pants I ordered, and I noticed that they have several pockets. Can you provide some information on the pocket layout and their intended purposes?"}], "item_number": 10, "order_number": 37, "description": "TrailBlaze Hiking Pants, quantity 1, price $75", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Emily \nLast Name: Rodriguez \nAge: 29 \nEmail Address: emilyr@example.com \nPhone Number: 555-111-2222 \nShipping Address: 987 Oak Ave, Cityville USA, 56789 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 3 \ndate: 2023-03-18 \nitem:\n- description: TrailMaster X4 Tent, quantity 3, price $750 \n\u00a0 item_number: 1 \n\norder_number: 12 \ndate: 2023-02-20 \nitem:\n- description: Summit Breeze Jacket, quantity 2, price $240 \n\u00a0 item_number: 3 \n\norder_number: 21 \ndate: 2023-04-02 \nitem:\n- description: BaseCamp Folding Table, quantity 1, price $60 \n\u00a0 item_number: 5 \n\norder_number: 31 \ndate: 2023-04-20 \nitem:\n- description: Alpine Explorer Tent, quantity 1, price $350 \n\u00a0 item_number: 8 \n\norder_number: 39 \ndate: 2023-03-30 \nitem:\n- description: TrailWalker Hiking Shoes, quantity 2, price $220 \n\u00a0 item_number: 11 \n\norder_number: 48 \ndate: 2023-04-16 \nitem:\n- description: MountainDream Sleeping Bag, quantity 2, price $260 \n\u00a0 item_number: 14 \n\norder_number: 61 \ndate: 2023-06-11 \nitem:\n- description: TrekStar Hiking Sandals, quantity 1, price $70 \n\u00a0 item_number: 18 \n\n", "history": [{"role": "customer", "content": "I purchased two pairs of Hiking Shoes for myself and my husband last month. While my pair is great, my husband's pair seems to be slightly small. Is it possible to exchange them for a larger size?"}], "item_number": 11, "order_number": 39, "description": "TrailWalker Hiking Shoes, quantity 2, price $220", "intent": "product exchange"} +{"customer_info": "## Customer_Info\n\nFirst Name: Daniel \nLast Name: Wilson \nAge: 47 \nEmail Address: danielw@example.com \nPhone Number: 555-444-5555 \nShipping Address: 321 Birch Ln, Smallville USA, 34567 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 9 \ndate: 2023-04-25 \nitem:\n- description: Adventurer Pro Backpack, quantity 3, price $270 \n\u00a0 item_number: 2 \n\norder_number: 13 \ndate: 2023-03-25 \nitem:\n- description: Summit Breeze Jacket, quantity 1, price $120 \n\u00a0 item_number: 3 \n\norder_number: 22 \ndate: 2023-05-07 \nitem:\n- description: BaseCamp Folding Table, quantity 3, price $180 \n\u00a0 item_number: 5 \n\norder_number: 40 \ndate: 2023-04-05 \nitem:\n- description: TrailWalker Hiking Shoes, quantity 1, price $110 \n\u00a0 item_number: 11 \n\norder_number: 49 \ndate: 2023-05-21 \nitem:\n- description: MountainDream Sleeping Bag, quantity 1, price $130 \n\u00a0 item_number: 14 \n\n", "history": [{"role": "customer", "content": "I just bought a pair of Hiking Shoes, and I'm planning a hiking trip soon. Do you have any recommendations for maintaining the shoes and increasing their lifespan?"}], "item_number": 11, "order_number": 40, "description": "TrailWalker Hiking Shoes, quantity 1, price $110", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Sarah \nLast Name: Lee \nAge: 38 \nEmail Address: sarahlee@example.com \nPhone Number: 555-867-5309 \nShipping Address: 321 Maple St, Bigtown USA, 90123 \nMembership: Platinum \n\n## Recent_Purchases\n\norder_number: 2 \ndate: 2023-02-10 \nitem:\n- description: TrailMaster X4 Tent, quantity 1, price $250 \n\u00a0 item_number: 1 \n\norder_number: 26 \ndate: 2023-02-05 \nitem:\n- description: CozyNights Sleeping Bag, quantity 1, price $100 \n\u00a0 item_number: 7 \n\norder_number: 35 \ndate: 2023-02-20 \nitem:\n- description: TrailBlaze Hiking Pants, quantity 1, price $75 \n\u00a0 item_number: 10 \n\norder_number: 42 \ndate: 2023-04-06 \nitem:\n- description: TrekMaster Camping Chair, quantity 2, price $100 \n\u00a0 item_number: 12 \n\norder_number: 51 \ndate: 2023-04-21 \nitem:\n- description: SkyView 2-Person Tent, quantity 1, price $200 \n\u00a0 item_number: 15 \n\norder_number: 56 \ndate: 2023-03-26 \nitem:\n- description: RainGuard Hiking Jacket, quantity 1, price $110 \n\u00a0 item_number: 17 \n\norder_number: 65 \ndate: 2023-04-11 \nitem:\n- description: CompactCook Camping Stove, quantity 1, price $60 \n\u00a0 item_number: 20 \n\n", "history": [{"role": "customer", "content": "I'm a Platinum member, and I just ordered two outdoor seats. I was wondering if there is any assembly required and if any tools are needed for that?"}], "item_number": 12, "order_number": 42, "description": "TrekMaster Camping Chair, quantity 2, price $100", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Jason \nLast Name: Brown \nAge: 50 \nEmail Address: jasonbrown@example.com \nPhone Number: 555-222-3333 \nShipping Address: 456 Cedar Rd, Anytown USA, 12345 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 8 \ndate: 2023-03-20 \nitem:\n- description: Adventurer Pro Backpack, quantity 1, price $90 \n\u00a0 item_number: 2 \n\norder_number: 27 \ndate: 2023-03-10 \nitem:\n- description: CozyNights Sleeping Bag, quantity 2, price $200 \n\u00a0 item_number: 7 \n\norder_number: 36 \ndate: 2023-03-25 \nitem:\n- description: TrailBlaze Hiking Pants, quantity 2, price $150 \n\u00a0 item_number: 10 \n\norder_number: 43 \ndate: 2023-05-11 \nitem:\n- description: TrekMaster Camping Chair, quantity 1, price $50 \n\u00a0 item_number: 12 \n\norder_number: 52 \ndate: 2023-05-26 \nitem:\n- description: SkyView 2-Person Tent, quantity 1, price $200 \n\u00a0 item_number: 15 \n\norder_number: 57 \ndate: 2023-05-01 \nitem:\n- description: RainGuard Hiking Jacket, quantity 2, price $220 \n\u00a0 item_number: 17 \n\norder_number: 66 \ndate: 2023-05-16 \nitem:\n- description: CompactCook Camping Stove, quantity 2, price $120 \n\u00a0 item_number: 20 \n\n", "history": [{"role": "customer", "content": "I bought a camping Chair last month, and it seems to be slightly wobbly when I sit on it. Is there any way to fix this issue?"}], "item_number": 12, "order_number": 43, "description": "TrekMaster Camping Chair, quantity 1, price $50", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: David \nLast Name: Kim \nAge: 42 \nEmail Address: davidkim@example.com \nPhone Number: 555-555-5555 \nShipping Address: 654 Pine St, Suburbia USA, 23456 \nMembership: Gold \n\n## Recent_Purchases\n\norder_number: 7 \ndate: 2023-02-15 \nitem:\n- description: Adventurer Pro Backpack, quantity 2, price $180 \n\u00a0 item_number: 2 \n\norder_number: 16 \ndate: 2023-02-25 \nitem:\n- description: TrekReady Hiking Boots, quantity 2, price $280 \n\u00a0 item_number: 4 \n\norder_number: 24 \ndate: 2023-03-05 \nitem:\n- description: EcoFire Camping Stove, quantity 2, price $160 \n\u00a0 item_number: 6 \n\norder_number: 33 \ndate: 2023-03-20 \nitem:\n- description: SummitClimber Backpack, quantity 2, price $240 \n\u00a0 item_number: 9 \n\norder_number: 45 \ndate: 2023-04-11 \nitem:\n- description: PowerBurner Camping Stove, quantity 2, price $200 \n\u00a0 item_number: 13 \n\norder_number: 54 \ndate: 2023-04-26 \nitem:\n- description: TrailLite Daypack, quantity 2, price $120 \n\u00a0 item_number: 16 \n\norder_number: 63 \ndate: 2023-05-11 \nitem:\n- description: Adventure Dining Table, quantity 2, price $180 \n\u00a0 item_number: 19 \n\n", "history": [{"role": "customer", "content": "I just bought an outdoor stove but I'm not sure how to attach the fuel canister. Can you please guide me through the process?"}], "item_number": 13, "order_number": 45, "description": "PowerBurner Camping Stove, quantity 2, price $200", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Emily \nLast Name: Rodriguez \nAge: 29 \nEmail Address: emilyr@example.com \nPhone Number: 555-111-2222 \nShipping Address: 987 Oak Ave, Cityville USA, 56789 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 3 \ndate: 2023-03-18 \nitem:\n- description: TrailMaster X4 Tent, quantity 3, price $750 \n\u00a0 item_number: 1 \n\norder_number: 12 \ndate: 2023-02-20 \nitem:\n- description: Summit Breeze Jacket, quantity 2, price $240 \n\u00a0 item_number: 3 \n\norder_number: 21 \ndate: 2023-04-02 \nitem:\n- description: BaseCamp Folding Table, quantity 1, price $60 \n\u00a0 item_number: 5 \n\norder_number: 31 \ndate: 2023-04-20 \nitem:\n- description: Alpine Explorer Tent, quantity 1, price $350 \n\u00a0 item_number: 8 \n\norder_number: 39 \ndate: 2023-03-30 \nitem:\n- description: TrailWalker Hiking Shoes, quantity 2, price $220 \n\u00a0 item_number: 11 \n\norder_number: 48 \ndate: 2023-04-16 \nitem:\n- description: MountainDream Sleeping Bag, quantity 2, price $260 \n\u00a0 item_number: 14 \n\norder_number: 61 \ndate: 2023-06-11 \nitem:\n- description: TrekStar Hiking Sandals, quantity 1, price $70 \n\u00a0 item_number: 18 \n\n", "history": [{"role": "customer", "content": "I've ordered two Sleeping Bags for my upcoming camping trip. I was wondering if they can be zipped together to create a double sleeping bag?"}], "item_number": 14, "order_number": 48, "description": "MountainDream Sleeping Bag, quantity 2, price $260", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Daniel \nLast Name: Wilson \nAge: 47 \nEmail Address: danielw@example.com \nPhone Number: 555-444-5555 \nShipping Address: 321 Birch Ln, Smallville USA, 34567 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 9 \ndate: 2023-04-25 \nitem:\n- description: Adventurer Pro Backpack, quantity 3, price $270 \n\u00a0 item_number: 2 \n\norder_number: 13 \ndate: 2023-03-25 \nitem:\n- description: Summit Breeze Jacket, quantity 1, price $120 \n\u00a0 item_number: 3 \n\norder_number: 22 \ndate: 2023-05-07 \nitem:\n- description: BaseCamp Folding Table, quantity 3, price $180 \n\u00a0 item_number: 5 \n\norder_number: 40 \ndate: 2023-04-05 \nitem:\n- description: TrailWalker Hiking Shoes, quantity 1, price $110 \n\u00a0 item_number: 11 \n\norder_number: 49 \ndate: 2023-05-21 \nitem:\n- description: MountainDream Sleeping Bag, quantity 1, price $130 \n\u00a0 item_number: 14 \n\n", "history": [{"role": "customer", "content": "I recently purchased a Sleeping Bag, and I'm wondering how to properly clean and store it to ensure it lasts for a long time."}], "item_number": 14, "order_number": 49, "description": "MountainDream Sleeping Bag, quantity 1, price $130", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: John \nLast Name: Smith \nAge: 35 \nEmail Address: johnsmith@example.com \nPhone Number: 555-123-4567 \nShipping Address: 123 Main St, Anytown USA, 12345 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 1 \ndate: 2023-01-05 \nitem:\n- description: TrailMaster X4 Tent, quantity 2, price $500 \n\u00a0 item_number: 1 \n\norder_number: 19 \ndate: 2023-01-25 \nitem:\n- description: BaseCamp Folding Table, quantity 1, price $60 \n\u00a0 item_number: 5 \n\norder_number: 29 \ndate: 2023-02-10 \nitem:\n- description: Alpine Explorer Tent, quantity 2, price $700 \n\u00a0 item_number: 8 \n\norder_number: 41 \ndate: 2023-03-01 \nitem:\n- description: TrekMaster Camping Chair, quantity 1, price $50 \n\u00a0 item_number: 12 \n\norder_number: 50 \ndate: 2023-03-16 \nitem:\n- description: SkyView 2-Person Tent, quantity 2, price $400 \n\u00a0 item_number: 15 \n\norder_number: 59 \ndate: 2023-04-01 \nitem:\n- description: TrekStar Hiking Sandals, quantity 1, price $70 \n\u00a0 item_number: 18 \n\n", "history": [{"role": "customer", "content": "I just received my Tents, and they look amazing! I can't wait to use them on our next camping trip. Quick question, though - what's the best way to set up the tent?"}], "item_number": 15, "order_number": 50, "description": "SkyView 2-Person Tent, quantity 2, price $400", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Sarah \nLast Name: Lee \nAge: 38 \nEmail Address: sarahlee@example.com \nPhone Number: 555-867-5309 \nShipping Address: 321 Maple St, Bigtown USA, 90123 \nMembership: Platinum \n\n## Recent_Purchases\n\norder_number: 2 \ndate: 2023-02-10 \nitem:\n- description: TrailMaster X4 Tent, quantity 1, price $250 \n\u00a0 item_number: 1 \n\norder_number: 26 \ndate: 2023-02-05 \nitem:\n- description: CozyNights Sleeping Bag, quantity 1, price $100 \n\u00a0 item_number: 7 \n\norder_number: 35 \ndate: 2023-02-20 \nitem:\n- description: TrailBlaze Hiking Pants, quantity 1, price $75 \n\u00a0 item_number: 10 \n\norder_number: 42 \ndate: 2023-04-06 \nitem:\n- description: TrekMaster Camping Chair, quantity 2, price $100 \n\u00a0 item_number: 12 \n\norder_number: 51 \ndate: 2023-04-21 \nitem:\n- description: SkyView 2-Person Tent, quantity 1, price $200 \n\u00a0 item_number: 15 \n\norder_number: 56 \ndate: 2023-03-26 \nitem:\n- description: RainGuard Hiking Jacket, quantity 1, price $110 \n\u00a0 item_number: 17 \n\norder_number: 65 \ndate: 2023-04-11 \nitem:\n- description: CompactCook Camping Stove, quantity 1, price $60 \n\u00a0 item_number: 20 \n\n", "history": [{"role": "customer", "content": "I ordered a Tent, and it arrived yesterday. However, I noticed that one of the tent poles is damaged. How can I get a replacement for the damaged pole?"}], "item_number": 15, "order_number": 51, "description": "SkyView 2-Person Tent, quantity 1, price $200", "intent": "product return"} +{"customer_info": "## Customer_Info\n\nFirst Name: Jason \nLast Name: Brown \nAge: 50 \nEmail Address: jasonbrown@example.com \nPhone Number: 555-222-3333 \nShipping Address: 456 Cedar Rd, Anytown USA, 12345 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 8 \ndate: 2023-03-20 \nitem:\n- description: Adventurer Pro Backpack, quantity 1, price $90 \n\u00a0 item_number: 2 \n\norder_number: 27 \ndate: 2023-03-10 \nitem:\n- description: CozyNights Sleeping Bag, quantity 2, price $200 \n\u00a0 item_number: 7 \n\norder_number: 36 \ndate: 2023-03-25 \nitem:\n- description: TrailBlaze Hiking Pants, quantity 2, price $150 \n\u00a0 item_number: 10 \n\norder_number: 43 \ndate: 2023-05-11 \nitem:\n- description: TrekMaster Camping Chair, quantity 1, price $50 \n\u00a0 item_number: 12 \n\norder_number: 52 \ndate: 2023-05-26 \nitem:\n- description: SkyView 2-Person Tent, quantity 1, price $200 \n\u00a0 item_number: 15 \n\norder_number: 57 \ndate: 2023-05-01 \nitem:\n- description: RainGuard Hiking Jacket, quantity 2, price $220 \n\u00a0 item_number: 17 \n\norder_number: 66 \ndate: 2023-05-16 \nitem:\n- description: CompactCook Camping Stove, quantity 2, price $120 \n\u00a0 item_number: 20 \n\n", "history": [{"role": "customer", "content": "I'm considering buying the SkyView 2-Person Tent for an upcoming camping trip. Can you please tell me more about the tent's ventilation and how it performs in rainy conditions?"}], "item_number": 15, "order_number": 52, "description": "SkyView 2-Person Tent, quantity 1, price $200", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: David \nLast Name: Kim \nAge: 42 \nEmail Address: davidkim@example.com \nPhone Number: 555-555-5555 \nShipping Address: 654 Pine St, Suburbia USA, 23456 \nMembership: Gold \n\n## Recent_Purchases\n\norder_number: 7 \ndate: 2023-02-15 \nitem:\n- description: Adventurer Pro Backpack, quantity 2, price $180 \n\u00a0 item_number: 2 \n\norder_number: 16 \ndate: 2023-02-25 \nitem:\n- description: TrekReady Hiking Boots, quantity 2, price $280 \n\u00a0 item_number: 4 \n\norder_number: 24 \ndate: 2023-03-05 \nitem:\n- description: EcoFire Camping Stove, quantity 2, price $160 \n\u00a0 item_number: 6 \n\norder_number: 33 \ndate: 2023-03-20 \nitem:\n- description: SummitClimber Backpack, quantity 2, price $240 \n\u00a0 item_number: 9 \n\norder_number: 45 \ndate: 2023-04-11 \nitem:\n- description: PowerBurner Camping Stove, quantity 2, price $200 \n\u00a0 item_number: 13 \n\norder_number: 54 \ndate: 2023-04-26 \nitem:\n- description: TrailLite Daypack, quantity 2, price $120 \n\u00a0 item_number: 16 \n\norder_number: 63 \ndate: 2023-05-11 \nitem:\n- description: Adventure Dining Table, quantity 2, price $180 \n\u00a0 item_number: 19 \n\n", "history": [{"role": "customer", "content": "I purchased two Daypacks for my kids, and while one is perfect, the other has a zipper issue that makes it difficult to open and close. Can I get a replacement for the faulty daypack?"}], "item_number": 16, "order_number": 54, "description": "TrailLite Daypack, quantity 2, price $120", "intent": "product return"} +{"customer_info": "## Customer_Info\n\nFirst Name: Melissa \nLast Name: Davis \nAge: 31 \nEmail Address: melissad@example.com \nPhone Number: 555-333-4444 \nShipping Address: 789 Ash St, Another City USA, 67890 \nMembership: Gold \n\n## Recent_Purchases\n\norder_number: 4 \ndate: 2023-04-22 \nitem:\n- description: TrailMaster X4 Tent, quantity 2, price $500 \n\u00a0 item_number: 1 \n\norder_number: 17 \ndate: 2023-03-30 \nitem:\n- description: TrekReady Hiking Boots, quantity 1, price $140 \n\u00a0 item_number: 4 \n\norder_number: 25 \ndate: 2023-04-10 \nitem:\n- description: EcoFire Camping Stove, quantity 1, price $80 \n\u00a0 item_number: 6 \n\norder_number: 34 \ndate: 2023-04-25 \nitem:\n- description: SummitClimber Backpack, quantity 1, price $120 \n\u00a0 item_number: 9 \n\norder_number: 46 \ndate: 2023-05-16 \nitem:\n- description: PowerBurner Camping Stove, quantity 1, price $100 \n\u00a0 item_number: 13 \n\norder_number: 55 \ndate: 2023-05-31 \nitem:\n- description: TrailLite Daypack, quantity 1, price $60 \n\u00a0 item_number: 16 \n\norder_number: 64 \ndate: 2023-06-16 \nitem:\n- description: Adventure Dining Table, quantity 1, price $90 \n\u00a0 item_number: 19 \n\n", "history": [{"role": "customer", "content": "I just bought a TrailLite Daypack for my day hikes, and I was wondering if it's water-resistant. Should I be concerned about my belongings getting wet if it rains?"}], "item_number": 16, "order_number": 55, "description": "TrailLite Daypack, quantity 1, price $60", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Jason \nLast Name: Brown \nAge: 50 \nEmail Address: jasonbrown@example.com \nPhone Number: 555-222-3333 \nShipping Address: 456 Cedar Rd, Anytown USA, 12345 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 8 \ndate: 2023-03-20 \nitem:\n- description: Adventurer Pro Backpack, quantity 1, price $90 \n\u00a0 item_number: 2 \n\norder_number: 27 \ndate: 2023-03-10 \nitem:\n- description: CozyNights Sleeping Bag, quantity 2, price $200 \n\u00a0 item_number: 7 \n\norder_number: 36 \ndate: 2023-03-25 \nitem:\n- description: TrailBlaze Hiking Pants, quantity 2, price $150 \n\u00a0 item_number: 10 \n\norder_number: 43 \ndate: 2023-05-11 \nitem:\n- description: TrekMaster Camping Chair, quantity 1, price $50 \n\u00a0 item_number: 12 \n\norder_number: 52 \ndate: 2023-05-26 \nitem:\n- description: SkyView 2-Person Tent, quantity 1, price $200 \n\u00a0 item_number: 15 \n\norder_number: 57 \ndate: 2023-05-01 \nitem:\n- description: RainGuard Hiking Jacket, quantity 2, price $220 \n\u00a0 item_number: 17 \n\norder_number: 66 \ndate: 2023-05-16 \nitem:\n- description: CompactCook Camping Stove, quantity 2, price $120 \n\u00a0 item_number: 20 \n\n", "history": [{"role": "customer", "content": "I just bought two RainGuard Hiking Jackets for my wife and me. Can you please provide some care instructions to ensure the jackets maintain their water resistance and durability?"}], "item_number": 17, "order_number": 57, "description": "RainGuard Hiking Jacket, quantity 2, price $220", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Amanda \nLast Name: Perez \nAge: 26 \nEmail Address: amandap@example.com \nPhone Number: 555-123-4567 \nShipping Address: 654 Pine St, Suburbia USA, 23456 \nMembership: Gold \n\n## Recent_Purchases\n\norder_number: 5 \ndate: 2023-05-01 \nitem:\n- description: TrailMaster X4 Tent, quantity 1, price $250 \n\u00a0 item_number: 1 \n\norder_number: 18 \ndate: 2023-05-04 \nitem:\n- description: TrekReady Hiking Boots, quantity 3, price $420 \n\u00a0 item_number: 4 \n\norder_number: 28 \ndate: 2023-04-15 \nitem:\n- description: CozyNights Sleeping Bag, quantity 1, price $100 \n\u00a0 item_number: 7 \n\norder_number: 37 \ndate: 2023-04-30 \nitem:\n- description: TrailBlaze Hiking Pants, quantity 1, price $75 \n\u00a0 item_number: 10 \n\norder_number: 58 \ndate: 2023-06-06 \nitem:\n- description: RainGuard Hiking Jacket, quantity 1, price $110 \n\u00a0 item_number: 17 \n\norder_number: 67 \ndate: 2023-06-21 \nitem:\n- description: CompactCook Camping Stove, quantity 1, price $60 \n\u00a0 item_number: 20 \n\n", "history": [{"role": "customer", "content": "I bought the RainGuard Hiking Jacket a few weeks ago, but I noticed that the seam tape on the inside is starting to peel off. Is there anything I can do to fix this issue?"}], "item_number": 17, "order_number": 58, "description": "RainGuard Hiking Jacket, quantity 1, price $110", "intent": "product return"} +{"customer_info": "## Customer_Info\n\nFirst Name: John \nLast Name: Smith \nAge: 35 \nEmail Address: johnsmith@example.com \nPhone Number: 555-123-4567 \nShipping Address: 123 Main St, Anytown USA, 12345 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 1 \ndate: 2023-01-05 \nitem:\n- description: TrailMaster X4 Tent, quantity 2, price $500 \n\u00a0 item_number: 1 \n\norder_number: 19 \ndate: 2023-01-25 \nitem:\n- description: BaseCamp Folding Table, quantity 1, price $60 \n\u00a0 item_number: 5 \n\norder_number: 29 \ndate: 2023-02-10 \nitem:\n- description: Alpine Explorer Tent, quantity 2, price $700 \n\u00a0 item_number: 8 \n\norder_number: 41 \ndate: 2023-03-01 \nitem:\n- description: TrekMaster Camping Chair, quantity 1, price $50 \n\u00a0 item_number: 12 \n\norder_number: 50 \ndate: 2023-03-16 \nitem:\n- description: SkyView 2-Person Tent, quantity 2, price $400 \n\u00a0 item_number: 15 \n\norder_number: 59 \ndate: 2023-04-01 \nitem:\n- description: TrekStar Hiking Sandals, quantity 1, price $70 \n\u00a0 item_number: 18 \n\n", "history": [{"role": "customer", "content": "Hi, I purchased the TrekStar Hiking Sandals a few weeks ago, and they feel a bit tight. Is there a break-in period for these sandals, or should I exchange them for a larger size?"}], "item_number": 18, "order_number": 59, "description": "TrekStar Hiking Sandals, quantity 1, price $70", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Emily \nLast Name: Rodriguez \nAge: 29 \nEmail Address: emilyr@example.com \nPhone Number: 555-111-2222 \nShipping Address: 987 Oak Ave, Cityville USA, 56789 \nMembership: None \n\n## Recent_Purchases\n\norder_number: 3 \ndate: 2023-03-18 \nitem:\n- description: TrailMaster X4 Tent, quantity 3, price $750 \n\u00a0 item_number: 1 \n\norder_number: 12 \ndate: 2023-02-20 \nitem:\n- description: Summit Breeze Jacket, quantity 2, price $240 \n\u00a0 item_number: 3 \n\norder_number: 21 \ndate: 2023-04-02 \nitem:\n- description: BaseCamp Folding Table, quantity 1, price $60 \n\u00a0 item_number: 5 \n\norder_number: 31 \ndate: 2023-04-20 \nitem:\n- description: Alpine Explorer Tent, quantity 1, price $350 \n\u00a0 item_number: 8 \n\norder_number: 39 \ndate: 2023-03-30 \nitem:\n- description: TrailWalker Hiking Shoes, quantity 2, price $220 \n\u00a0 item_number: 11 \n\norder_number: 48 \ndate: 2023-04-16 \nitem:\n- description: MountainDream Sleeping Bag, quantity 2, price $260 \n\u00a0 item_number: 14 \n\norder_number: 61 \ndate: 2023-06-11 \nitem:\n- description: TrekStar Hiking Sandals, quantity 1, price $70 \n\u00a0 item_number: 18 \n\n", "history": [{"role": "customer", "content": "Hi there, I'm interested in purchasing the TrekStar Hiking Sandals. Can you tell me more about their features?"}], "item_number": 18, "order_number": 61, "description": "TrekStar Hiking Sandals, quantity 1, price $70", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Jane \nLast Name: Doe \nAge: 28 \nEmail Address: janedoe@example.com \nPhone Number: 555-987-6543 \nShipping Address: 456 Oak St, Another City USA, 67890 \nMembership: Gold \n\n## Recent_Purchases\n\norder_number: 6 \ndate: 2023-01-10 \nitem:\n- description: Adventurer Pro Backpack, quantity 1, price $90 \n\u00a0 item_number: 2 \n\norder_number: 15 \ndate: 2023-01-20 \nitem:\n- description: TrekReady Hiking Boots, quantity 1, price $140 \n\u00a0 item_number: 4 \n\norder_number: 23 \ndate: 2023-01-30 \nitem:\n- description: EcoFire Camping Stove, quantity 1, price $80 \n\u00a0 item_number: 6 \n\norder_number: 32 \ndate: 2023-02-15 \nitem:\n- description: SummitClimber Backpack, quantity 1, price $120 \n\u00a0 item_number: 9 \n\norder_number: 44 \ndate: 2023-03-06 \nitem:\n- description: PowerBurner Camping Stove, quantity 1, price $100 \n\u00a0 item_number: 13 \n\norder_number: 53 \ndate: 2023-03-21 \nitem:\n- description: TrailLite Daypack, quantity 1, price $60 \n\u00a0 item_number: 16 \n\norder_number: 62 \ndate: 2023-04-06 \nitem:\n- description: Adventure Dining Table, quantity 1, price $90 \n\u00a0 item_number: 19 \n\n", "history": [{"role": "customer", "content": "I recently purchased the Adventure Dining Table, but I'm not happy with its quality. The table arrived with scratches on the surface, and one of the legs seems wobbly. I expected better craftsmanship from your brand. This is disappointing."}], "item_number": 19, "order_number": 62, "description": "Adventure Dining Table, quantity 1, price $90", "intent": "product return"} +{"customer_info": "## Customer_Info\n\nFirst Name: Melissa \nLast Name: Davis \nAge: 31 \nEmail Address: melissad@example.com \nPhone Number: 555-333-4444 \nShipping Address: 789 Ash St, Another City USA, 67890 \nMembership: Gold \n\n## Recent_Purchases\n\norder_number: 4 \ndate: 2023-04-22 \nitem:\n- description: TrailMaster X4 Tent, quantity 2, price $500 \n\u00a0 item_number: 1 \n\norder_number: 17 \ndate: 2023-03-30 \nitem:\n- description: TrekReady Hiking Boots, quantity 1, price $140 \n\u00a0 item_number: 4 \n\norder_number: 25 \ndate: 2023-04-10 \nitem:\n- description: EcoFire Camping Stove, quantity 1, price $80 \n\u00a0 item_number: 6 \n\norder_number: 34 \ndate: 2023-04-25 \nitem:\n- description: SummitClimber Backpack, quantity 1, price $120 \n\u00a0 item_number: 9 \n\norder_number: 46 \ndate: 2023-05-16 \nitem:\n- description: PowerBurner Camping Stove, quantity 1, price $100 \n\u00a0 item_number: 13 \n\norder_number: 55 \ndate: 2023-05-31 \nitem:\n- description: TrailLite Daypack, quantity 1, price $60 \n\u00a0 item_number: 16 \n\norder_number: 64 \ndate: 2023-06-16 \nitem:\n- description: Adventure Dining Table, quantity 1, price $90 \n\u00a0 item_number: 19 \n\n", "history": [{"role": "customer", "content": "I recently purchased the Adventure Dining Table, and I have a question about its setup. The instructions provided are not very clear, and I'm having trouble assembling the table correctly. Can you provide me with some guidance or more detailed instructions?"}], "item_number": 19, "order_number": 64, "description": "Adventure Dining Table, quantity 2, price $180", "intent": "product question"} +{"customer_info": "## Customer_Info\n\nFirst Name: Sarah \nLast Name: Lee \nAge: 38 \nEmail Address: sarahlee@example.com \nPhone Number: 555-867-5309 \nShipping Address: 321 Maple St, Bigtown USA, 90123 \nMembership: Platinum \n\n## Recent_Purchases\n\norder_number: 2 \ndate: 2023-02-10 \nitem:\n- description: TrailMaster X4 Tent, quantity 1, price $250 \n\u00a0 item_number: 1 \n\norder_number: 26 \ndate: 2023-02-05 \nitem:\n- description: CozyNights Sleeping Bag, quantity 1, price $100 \n\u00a0 item_number: 7 \n\norder_number: 35 \ndate: 2023-02-20 \nitem:\n- description: TrailBlaze Hiking Pants, quantity 1, price $75 \n\u00a0 item_number: 10 \n\norder_number: 42 \ndate: 2023-04-06 \nitem:\n- description: TrekMaster Camping Chair, quantity 2, price $100 \n\u00a0 item_number: 12 \n\norder_number: 51 \ndate: 2023-04-21 \nitem:\n- description: SkyView 2-Person Tent, quantity 1, price $200 \n\u00a0 item_number: 15 \n\norder_number: 56 \ndate: 2023-03-26 \nitem:\n- description: RainGuard Hiking Jacket, quantity 1, price $110 \n\u00a0 item_number: 17 \n\norder_number: 65 \ndate: 2023-04-11 \nitem:\n- description: CompactCook Camping Stove, quantity 1, price $60 \n\u00a0 item_number: 20 \n\n", "history": [{"role": "customer", "content": "I recently purchased the CompactCook Camping Stove, and I'm quite disappointed with its performance. The flame doesn't seem to stay consistent, and it takes forever to boil water. This is not what I expected from a camping stove. Can you help me with this issue?"}], "item_number": 20, "order_number": 65, "description": "CompactCook Camping Stove, quantity 1, price $60", "intent": "product return"} +{"customer_info": "## Customer_Info\n\nFirst Name: Amanda \nLast Name: Perez \nAge: 26 \nEmail Address: amandap@example.com \nPhone Number: 555-123-4567 \nShipping Address: 654 Pine St, Suburbia USA, 23456 \nMembership: Gold \n\n## Recent_Purchases\n\norder_number: 5 \ndate: 2023-05-01 \nitem:\n- description: TrailMaster X4 Tent, quantity 1, price $250 \n\u00a0 item_number: 1 \n\norder_number: 18 \ndate: 2023-05-04 \nitem:\n- description: TrekReady Hiking Boots, quantity 3, price $420 \n\u00a0 item_number: 4 \n\norder_number: 28 \ndate: 2023-04-15 \nitem:\n- description: CozyNights Sleeping Bag, quantity 1, price $100 \n\u00a0 item_number: 7 \n\norder_number: 37 \ndate: 2023-04-30 \nitem:\n- description: TrailBlaze Hiking Pants, quantity 1, price $75 \n\u00a0 item_number: 10 \n\norder_number: 58 \ndate: 2023-06-06 \nitem:\n- description: RainGuard Hiking Jacket, quantity 1, price $110 \n\u00a0 item_number: 17 \n\norder_number: 67 \ndate: 2023-06-21 \nitem:\n- description: CompactCook Camping Stove, quantity 1, price $60 \n\u00a0 item_number: 20 \n\n", "history": [{"role": "customer", "content": "I recently received the CompactCook Camping Stove, and I'm not sure about its maintenance requirements. Are there any specific cleaning or storage instructions I should follow to ensure the stove's longevity?"}], "item_number": 20, "order_number": 67, "description": "CompactCook Camping Stove, quantity 1, price $60", "intent": "product question"} diff --git a/examples/flows/standard/intent-copilot/extract_intent_tool.py b/examples/flows/standard/intent-copilot/extract_intent_tool.py new file mode 100644 index 00000000000..517619c93d4 --- /dev/null +++ b/examples/flows/standard/intent-copilot/extract_intent_tool.py @@ -0,0 +1,26 @@ +import os + +from promptflow import tool +from promptflow.connections import CustomConnection + +from intent import extract_intent + + +@tool +def extract_intent_tool( + customer_info, + history, + user_prompt_template, + connection: CustomConnection +) -> str: + + # set environment variables + for key, value in connection.items(): + os.environ[key] = value + + # call the entry function + return extract_intent( + customer_info=customer_info, + history=history, + user_prompt_template=user_prompt_template, + ) diff --git a/examples/flows/standard/intent-copilot/flow.dag.yaml b/examples/flows/standard/intent-copilot/flow.dag.yaml new file mode 100644 index 00000000000..37556208da5 --- /dev/null +++ b/examples/flows/standard/intent-copilot/flow.dag.yaml @@ -0,0 +1,28 @@ +inputs: + customer_info: + type: string + history: + type: list +outputs: + output: + type: string + reference: ${extract_intent.output} +nodes: +- name: user_prompt_template + type: prompt + source: + type: code + path: user_intent_zero_shot.jinja2 + inputs: {} # Please enter the inputs of this node +- name: extract_intent + type: python + source: + type: code + path: extract_intent_tool.py + inputs: + customer_info: ${inputs.customer_info} + history: ${inputs.history} + user_prompt_template: ${user_prompt_template.output} + connection: custom_connection +environment: + python_requirements_txt: requirements.txt diff --git a/examples/flows/standard/intent-copilot/inputs.json b/examples/flows/standard/intent-copilot/inputs.json new file mode 100644 index 00000000000..a028f126737 --- /dev/null +++ b/examples/flows/standard/intent-copilot/inputs.json @@ -0,0 +1,4 @@ +{ + "customer_info": "## Customer_Info\\n\\nFirst Name: Sarah \\nLast Name: Lee \\nAge: 38 \\nEmail Address: sarahlee@example.com \\nPhone Number: 555-867-5309 \\nShipping Address: 321 Maple St, Bigtown USA, 90123 \\nMembership: Platinum \\n\\n## Recent_Purchases\\n\\norder_number: 2 \\ndate: 2023-02-10 \\nitem:\\n- description: TrailMaster X4 Tent, quantity 1, price $250 \\n\\u00a0 item_number: 1 \\n\\norder_number: 26 \\ndate: 2023-02-05 \\nitem:\\n- description: CozyNights Sleeping Bag, quantity 1, price $100 \\n\\u00a0 item_number: 7 \\n\\norder_number: 35 \\ndate: 2023-02-20 \\nitem:\\n- description: TrailBlaze Hiking Pants, quantity 1, price $75 \\n\\u00a0 item_number: 10 \\n\\norder_number: 42 \\ndate: 2023-04-06 \\nitem:\\n- description: TrekMaster Camping Chair, quantity 2, price $100 \\n\\u00a0 item_number: 12 \\n\\norder_number: 51 \\ndate: 2023-04-21 \\nitem:\\n- description: SkyView 2-Person Tent, quantity 1, price $200 \\n\\u00a0 item_number: 15 \\n\\norder_number: 56 \\ndate: 2023-03-26 \\nitem:\\n- description: RainGuard Hiking Jacket, quantity 1, price $110 \\n\\u00a0 item_number: 17 \\n\\norder_number: 65 \\ndate: 2023-04-11 \\nitem:\\n- description: CompactCook Camping Stove, quantity 1, price $60 \\n\\u00a0 item_number: 20 \\n\\n", + "history": "[ { \"role\": \"customer\", \"content\": \"I recently bought the TrailMaster X4 Tent, and it leaked during a light rain. This is unacceptable! I expected a reliable and waterproof tent, but it failed to deliver. I'm extremely disappointed in the quality.\" } ]" +} \ No newline at end of file diff --git a/examples/flows/standard/intent-copilot/intent.py b/examples/flows/standard/intent-copilot/intent.py new file mode 100644 index 00000000000..63a1d8039f1 --- /dev/null +++ b/examples/flows/standard/intent-copilot/intent.py @@ -0,0 +1,67 @@ +import os +import pip +from langchain import LLMChain +from langchain.chat_models import AzureChatOpenAI +from langchain.prompts.chat import ChatPromptTemplate, HumanMessagePromptTemplate + + +def extract_intent(customer_info: str, history: list, user_prompt_template: str): + if "AZURE_OPENAI_API_KEY" not in os.environ: + # load environment variables from .env file + try: + from dotenv import load_dotenv + except ImportError: + # This can be removed if user using custom image. + pip.main(["install", "python-dotenv"]) + from dotenv import load_dotenv + + load_dotenv() + + chat_history_text = "\n".join( + [message["role"] + ": " + message["content"] for message in history] + ) + + chat = AzureChatOpenAI( + deployment_name=os.environ["CHAT_DEPLOYMENT_NAME"], + openai_api_key=os.environ["AZURE_OPENAI_API_KEY"], + openai_api_base=os.environ["AZURE_OPENAI_API_BASE"], + openai_api_type="azure", + openai_api_version="2023-03-15-preview", + temperature=0, + ) + + chat_prompt_template = ChatPromptTemplate.from_messages( + [HumanMessagePromptTemplate.from_template(user_prompt_template)] + ) + + chain = LLMChain(llm=chat, prompt=chat_prompt_template) + + reply = chain.run(customer_info=customer_info, chat_history=chat_history_text) + return reply + + +if __name__ == "__main__": + import json + + with open("./data/denormalized-flat.jsonl", "r") as f: + data = [json.loads(line) for line in f.readlines()] + + # only ten samples + data = data[:10] + + # load template from file + with open("user_intent_zero_shot.md", "r") as f: + user_prompt_template = f.read() + + # each test + for item in data: + reply = extract_intent(item["customer_info"], item["history"], user_prompt_template) + print("=====================================") + # print("Customer info: ", item["customer_info"]) + # print("+++++++++++++++++++++++++++++++++++++") + print("Chat history: ", item["history"]) + print("+++++++++++++++++++++++++++++++++++++") + print(reply) + print("+++++++++++++++++++++++++++++++++++++") + print(f"Ground Truth: {item['intent']}") + print("=====================================") diff --git a/examples/flows/standard/intent-copilot/requirements.txt b/examples/flows/standard/intent-copilot/requirements.txt new file mode 100644 index 00000000000..18e9929aa30 --- /dev/null +++ b/examples/flows/standard/intent-copilot/requirements.txt @@ -0,0 +1,6 @@ +--extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ +promptflow +promptflow-tools +python-dotenv +langchain +jinja2 \ No newline at end of file diff --git a/examples/flows/standard/intent-copilot/user_intent_few_shot.jinja2 b/examples/flows/standard/intent-copilot/user_intent_few_shot.jinja2 new file mode 100644 index 00000000000..01428d4fcf8 --- /dev/null +++ b/examples/flows/standard/intent-copilot/user_intent_few_shot.jinja2 @@ -0,0 +1,43 @@ +You are given a list of orders with item_numbers from a customer and a statement from the customer. It is your job to identify +the intent that the customer has with their statement. Possible intents can be: +"product return", "product exchange", "general question", "product question", "other". + +If the intent is product related ("product return", "product exchange", "product question"), then you should also +provide the order id and item that the customer is referring to in their statement. + +For instance if you are give the following list of orders: + +order_number: 2020230 +date: 2023-04-23 +store_location: SeattleStore +items: +- description: Roof Rack, color black, price $199.99 + item_number: 101010 +- description: Running Shoes, size 10, color blue, price $99.99 + item_number: 202020 + +You are given the following customer statements: +- I am having issues with the jobbing shoes I bought. + +Then you should answer with in valid yaml format with the fields intent, order_number, item, and item_number like so: +intent: product question +order_number: 2020230 +descrption: Running Shoes, size 10, color blue, price $99.99 +item_number: 202020 + +Here is the acutal problem you need to solve: + +In triple backticks below is the customer information and a list of orders. +``` +{{customer_info}} +``` + +In triple backticks below are the is the chat history with customer statements and replies from the customer service agent: +``` +{{chat_history}} +``` + +What is the customer's `intent:` here? +"product return", "exchange product", "general question", "product question" or "other"? + +Reply with only the intent string. \ No newline at end of file diff --git a/examples/flows/standard/intent-copilot/user_intent_zero_shot.jinja2 b/examples/flows/standard/intent-copilot/user_intent_zero_shot.jinja2 new file mode 100644 index 00000000000..610e84e578e --- /dev/null +++ b/examples/flows/standard/intent-copilot/user_intent_zero_shot.jinja2 @@ -0,0 +1,37 @@ +You are given a list of orders with item_numbers from a customer and a statement from the customer. It is your job to identify + +the intent that the customer has with their statement. Possible intents can be: + +"product return", "product exchange", "general question", "product question", "other". + + + + + +In triple backticks below is the customer information and a list of orders. + +``` + +{{customer_info}} + +``` + + + + +In triple backticks below are the is the chat history with customer statements and replies from the customer service agent: + +``` + +{{chat_history}} + +``` + + + + +What is the customer's `intent:` here? + +"product return", "exchange product", "general question", "product question" or "other"? + +Reply with only the intent string. \ No newline at end of file diff --git a/examples/flows/standard/summarizing-film-with-autogpt/README.md b/examples/flows/standard/summarizing-film-with-autogpt/README.md new file mode 100644 index 00000000000..a869a9d9cb1 --- /dev/null +++ b/examples/flows/standard/summarizing-film-with-autogpt/README.md @@ -0,0 +1,61 @@ +# Summarizing Film With AutoGPT + +This is a flow showcasing how to construct a AutoGPT flow to autonomously figures out how to apply the given functions +to solve the goal, which is film trivia that provides accurate and up-to-date information about movies, directors, +actors, and more in this sample. + +It involves inferring next executed function and user intent with LLM, and then use the function to generate +observation. The observation above will be used as augmented prompt which is the input of next LLM inferring loop +until the inferred function is the signal that you have finished all your objectives. The functions set used in the +flow contains Wikipedia search function that can search the web to find answer about current events and PythonREPL +python function that can run python code in a REPL. + +For the sample input about movie introduction, the AutoGPT usually runs 4 rounds to finish the task. The first round +is searching for the movie name, the second round is searching for the movie director, the third round is calculating +director age, and the last round is outputting finishing signal. It takes 30s~40s to finish the task, but may take +longer time if you use "gpt-3.5" or encounter Azure OpenAI rate limit. You could use "gpt-4" or go to +https://aka.ms/oai/quotaincrease if you would like to further increase the default rate limit. + +Note: This is just a sample introducing how to use promptflow to build a simple AutoGPT. You can go to +https://github.com/Significant-Gravitas/Auto-GPT to get more concepts about AutoGPT. + +## What you will learn + +In this flow, you will learn +- how to use prompt tool. +- how to compose an AutoGPT flow using functions. + +## Prerequisites + +Install prompt-flow sdk and other dependencies: +```bash +pip install -r requirements.txt +``` + +## Getting Started + +### 1 Create Azure OpenAI or OpenAI connection +```bash +# Override keys with --set to avoid yaml file changes +pf connection create --file azure_openai.yml --set api_key= api_base= +``` +Note that you need to use "2023-07-01-preview" as Azure OpenAI connection API version when using function calling. +See How to use function calling with Azure OpenAI Service for more details. + +### 2. Configure the flow with your connection +`flow.dag.yaml` is already configured with connection named `azure_open_ai_connection`. It is recommended to use "gpt-4" model for stable performance. Using "gpt-3.5-turbo" may lead to the model getting +stuck in the agent inner loop due to its suboptimal and unstable performance. + +### 3. Test flow with single line data + +```bash +# test with default input value in flow.dag.yaml +pf flow test --flow . +``` + +### 4. Run with multi-line data + +```bash +# create run using command line args +pf run create --flow . --data ./data.jsonl --stream +``` \ No newline at end of file diff --git a/examples/flows/standard/summarizing-film-with-autogpt/autogpt_class.py b/examples/flows/standard/summarizing-film-with-autogpt/autogpt_class.py new file mode 100644 index 00000000000..d7d734f3c38 --- /dev/null +++ b/examples/flows/standard/summarizing-film-with-autogpt/autogpt_class.py @@ -0,0 +1,144 @@ +from promptflow.tools.aoai import chat as aoai_chat +from promptflow.tools.openai import chat as openai_chat +from promptflow.connections import AzureOpenAIConnection, OpenAIConnection +from util import count_message_tokens, count_string_tokens, create_chat_message, generate_context, get_logger, \ + parse_reply, construct_prompt + +autogpt_logger = get_logger("autogpt_agent") + + +class AutoGPT: + def __init__( + self, + connection, + tools, + full_message_history, + functions, + system_prompt=None, + triggering_prompt=None, + user_prompt=None, + model_or_deployment_name=None + ): + self.tools = tools + self.full_message_history = full_message_history + self.functions = functions + self.system_prompt = system_prompt + self.connection = connection + self.model_or_deployment_name = model_or_deployment_name + self.triggering_prompt = triggering_prompt + self.user_prompt = user_prompt + + def chat_with_ai(self, token_limit): + """Interact with the OpenAI API, sending the prompt, message history and functions.""" + + # Reserve 1000 tokens for the response + send_token_limit = token_limit - 1000 + ( + next_message_to_add_index, + current_tokens_used, + insertion_index, + current_context, + ) = generate_context(self.system_prompt, self.full_message_history, self.user_prompt) + # Account for user input (appended later) + current_tokens_used += count_message_tokens([create_chat_message("user", self.triggering_prompt)]) + current_tokens_used += 500 # Account for memory (appended later) + # Add Messages until the token limit is reached or there are no more messages to add. + while next_message_to_add_index >= 0: + message_to_add = self.full_message_history[next_message_to_add_index] + + tokens_to_add = count_message_tokens([message_to_add]) + if current_tokens_used + tokens_to_add > send_token_limit: + break + + # Add the most recent message to the start of the current context, after the two system prompts. + current_context.insert( + insertion_index, self.full_message_history[next_message_to_add_index] + ) + + # Count the currently used tokens + current_tokens_used += tokens_to_add + # Move to the next most recent message in the full message history + next_message_to_add_index -= 1 + + # Append user input, the length of this is accounted for above + current_context.extend([create_chat_message("user", self.triggering_prompt)]) + # Calculate remaining tokens + tokens_remaining = token_limit - current_tokens_used + + current_context = construct_prompt(current_context) + if isinstance(self.connection, AzureOpenAIConnection): + try: + response = aoai_chat( + connection=self.connection, + prompt=current_context, + deployment_name=self.model_or_deployment_name, + max_tokens=tokens_remaining, + functions=self.functions) + return response + except Exception as e: + if "The API deployment for this resource does not exist" in e.exc_msg: + raise Exception( + "Please fill in the deployment name of your Azure OpenAI resoure gpt-4 model.") + + elif isinstance(self.connection, OpenAIConnection): + response = openai_chat( + connection=self.connection, + prompt=current_context, + model=self.model_or_deployment_name, + max_tokens=tokens_remaining, + functions=self.functions) + return response + else: + raise ValueError("Connection must be an instance of AzureOpenAIConnection or OpenAIConnection") + + def run(self): + tools = {t.__name__: t for t in self.tools} + while True: + # Send message to AI, get response + response = self.chat_with_ai(token_limit=4000) + if "function_call" in response: + # Update full message history + function_name = response["function_call"]["name"] + parsed_output = parse_reply(response["function_call"]["arguments"]) + if "Error" in parsed_output: + error_message = parsed_output["Error"] + autogpt_logger.info(f"Error: {error_message}") + command_result = f"Error: {error_message}" + else: + autogpt_logger.info(f"Function generation requested, function = {function_name}, args = " + f"{parsed_output}") + self.full_message_history.append( + create_chat_message("assistant", f"Function generation requested, function = {function_name}, " + f"args = {parsed_output}") + ) + if function_name == "finish": + response = parsed_output["response"] + autogpt_logger.info(f"Responding to user: {response}") + return response + if function_name in tools: + tool = tools[function_name] + try: + autogpt_logger.info(f"Next function = {function_name}, arguments = {parsed_output}") + result = tool(**parsed_output) + command_result = f"Executed function {function_name} and returned: {result}" + except Exception as e: + command_result = ( + f"Error: {str(e)}, {type(e).__name__}" + ) + result_length = count_string_tokens(command_result) + + if result_length + 600 > 4000: + command_result = f"Failure: function {function_name} returned too much output. Do not " \ + f"execute this function again with the same arguments." + else: + command_result = f"Unknown function '{function_name}'. Please refer to available functions " \ + f"defined in functions parameter." + + # Append command result to the message history + self.full_message_history.append(create_chat_message("function", str(command_result), function_name)) + autogpt_logger.info(f"function: {command_result}") + else: + autogpt_logger.info(f"No function generated, returned: {response['content']}") + self.full_message_history.append( + create_chat_message("assistant", f"No function generated, returned: {response['content']}") + ) diff --git a/examples/flows/standard/summarizing-film-with-autogpt/autogpt_easy_start.py b/examples/flows/standard/summarizing-film-with-autogpt/autogpt_easy_start.py new file mode 100644 index 00000000000..945b714954e --- /dev/null +++ b/examples/flows/standard/summarizing-film-with-autogpt/autogpt_easy_start.py @@ -0,0 +1,30 @@ +from typing import Union + +from promptflow import tool +from promptflow.connections import AzureOpenAIConnection, OpenAIConnection + + +@tool +def autogpt_easy_start(connection: Union[AzureOpenAIConnection, OpenAIConnection], system_prompt: str, user_prompt: str, + triggering_prompt: str, functions: list, model_or_deployment_name: str): + from wiki_search import search + from python_repl import python + from autogpt_class import AutoGPT + + full_message_history = [] + tools = [ + search, + python + ] + agent = AutoGPT( + full_message_history=full_message_history, + tools=tools, + system_prompt=system_prompt, + connection=connection, + model_or_deployment_name=model_or_deployment_name, + functions=functions, + user_prompt=user_prompt, + triggering_prompt=triggering_prompt + ) + result = agent.run() + return result diff --git a/examples/flows/standard/summarizing-film-with-autogpt/azure_openai.yml b/examples/flows/standard/summarizing-film-with-autogpt/azure_openai.yml new file mode 100644 index 00000000000..48cd8c1fff5 --- /dev/null +++ b/examples/flows/standard/summarizing-film-with-autogpt/azure_openai.yml @@ -0,0 +1,7 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/AzureOpenAIConnection.schema.json +name: azure_open_ai_connection +type: azure_open_ai +api_key: "" +api_base: "aoai-api-endpoint" +api_type: "azure" +api_version: "2023-07-01-preview" diff --git a/examples/flows/standard/summarizing-film-with-autogpt/data.jsonl b/examples/flows/standard/summarizing-film-with-autogpt/data.jsonl new file mode 100644 index 00000000000..bf6dac898d3 --- /dev/null +++ b/examples/flows/standard/summarizing-film-with-autogpt/data.jsonl @@ -0,0 +1 @@ +{"name": "FilmTriviaGPT", "role": "an AI specialized in film trivia that provides accurate and up-to-date information about movies, directors, actors, and more.", "goals": ["Introduce 'Lord of the Rings' film trilogy including the film title, release year, director, current age of the director, production company and a brief summary of the film."]} \ No newline at end of file diff --git a/examples/flows/standard/summarizing-film-with-autogpt/flow.dag.yaml b/examples/flows/standard/summarizing-film-with-autogpt/flow.dag.yaml new file mode 100644 index 00000000000..35835e9df63 --- /dev/null +++ b/examples/flows/standard/summarizing-film-with-autogpt/flow.dag.yaml @@ -0,0 +1,59 @@ +inputs: + name: + type: string + default: "FilmTriviaGPT" + goals: + type: list + default: ["Introduce 'Lord of the Rings' film trilogy including the film title, release year, director, current age of the director, production company and a brief summary of the film."] + role: + type: string + default: "an AI specialized in film trivia that provides accurate and up-to-date information about movies, directors, actors, and more." +outputs: + output: + type: string + reference: ${autogpt_easy_start.output} +nodes: +- name: autogpt_easy_start + type: python + source: + type: code + path: autogpt_easy_start.py + inputs: + connection: azure_open_ai_connection + functions: ${functions.output} + model_or_deployment_name: gpt-4 + system_prompt: ${system_prompt.output} + triggering_prompt: ${triggering_prompt.output} + user_prompt: ${user_prompt.output} +- name: system_prompt + type: prompt + source: + type: code + path: system_prompt.jinja2 + inputs: + name: ${inputs.name} + role: ${inputs.role} +- name: user_prompt + type: prompt + source: + type: code + path: user_prompt.jinja2 + inputs: + goals: ${generate_goal.output} +- name: triggering_prompt + type: prompt + source: + type: code + path: triggering_prompt.jinja2 +- name: functions + type: python + source: + type: code + path: functions.py +- name: generate_goal + type: python + source: + type: code + path: generate_goal.py + inputs: + items: ${inputs.goals} \ No newline at end of file diff --git a/examples/flows/standard/summarizing-film-with-autogpt/functions.py b/examples/flows/standard/summarizing-film-with-autogpt/functions.py new file mode 100644 index 00000000000..33eb276f777 --- /dev/null +++ b/examples/flows/standard/summarizing-film-with-autogpt/functions.py @@ -0,0 +1,59 @@ +from promptflow import tool + + +@tool +def functions_format() -> list: + functions = [ + { + "name": "search", + "description": """The action will search this entity name on Wikipedia and returns the first {count} + sentences if it exists. If not, it will return some related entities to search next.""", + "parameters": { + "type": "object", + "properties": { + "entity": { + "type": "string", + "description": "Entity name which is used for Wikipedia search.", + }, + "count": { + "type": "integer", + "default": 10, + "description": "Returned sentences count if entity name exists Wikipedia.", + }, + }, + "required": ["entity"], + }, + }, + { + "name": "python", + "description": """A Python shell. Use this to execute python commands. Input should be a valid python + command and you should print result with `print(...)` to see the output.""", + "parameters": { + "type": "object", + "properties": { + "command": { + "type": "string", + "description": "The command you want to execute in python", + } + }, + "required": ["command"] + }, + }, + { + "name": "finish", + "description": """use this to signal that you have finished all your goals and remember show your + results""", + "parameters": { + "type": "object", + "properties": { + "response": { + "type": "string", + "description": "final response to let people know you have finished your goals and remember " + "show your results", + }, + }, + "required": ["response"], + }, + }, + ] + return functions diff --git a/examples/flows/standard/summarizing-film-with-autogpt/generate_goal.py b/examples/flows/standard/summarizing-film-with-autogpt/generate_goal.py new file mode 100644 index 00000000000..0cff4a431bf --- /dev/null +++ b/examples/flows/standard/summarizing-film-with-autogpt/generate_goal.py @@ -0,0 +1,15 @@ +from promptflow import tool + + +@tool +def generate_goal(items: list = []) -> str: + """ + Generate a numbered list from given items based on the item_type. + + Args: + items (list): A list of items to be numbered. + + Returns: + str: The formatted numbered list. + """ + return "\n".join(f"{i + 1}. {item}" for i, item in enumerate(items)) diff --git a/examples/flows/standard/summarizing-film-with-autogpt/python_repl.py b/examples/flows/standard/summarizing-film-with-autogpt/python_repl.py new file mode 100644 index 00000000000..9e263b80950 --- /dev/null +++ b/examples/flows/standard/summarizing-film-with-autogpt/python_repl.py @@ -0,0 +1,38 @@ +import sys +from io import StringIO +from typing import Dict, Optional + + +class PythonREPL: + """Simulates a standalone Python REPL.""" + + def __init__(self) -> None: + self.globals: Optional[Dict] = globals() + self.locals: Optional[Dict] = None + + def run(self, command: str) -> str: + """Run command with own globals/locals and returns anything printed.""" + old_stdout = sys.stdout + sys.stdout = mystdout = StringIO() + try: + exec(command, self.globals, self.locals) + sys.stdout = old_stdout + output = mystdout.getvalue() + except Exception as e: + sys.stdout = old_stdout + output = repr(e) + print(output) + return output + + +python_repl = PythonREPL() + + +def python(command: str): + """ + A Python shell. Use this to execute python commands. Input should be a valid python command. + If you want to see the output of a value, you should print it out with `print(...)`. + """ + + command = command.strip().strip("```") + return python_repl.run(command) diff --git a/examples/flows/standard/summarizing-film-with-autogpt/requirements.txt b/examples/flows/standard/summarizing-film-with-autogpt/requirements.txt new file mode 100644 index 00000000000..88941ca6838 --- /dev/null +++ b/examples/flows/standard/summarizing-film-with-autogpt/requirements.txt @@ -0,0 +1,5 @@ +--extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ +promptflow +promptflow-tools +tiktoken +bs4 \ No newline at end of file diff --git a/examples/flows/standard/summarizing-film-with-autogpt/system_prompt.jinja2 b/examples/flows/standard/summarizing-film-with-autogpt/system_prompt.jinja2 new file mode 100644 index 00000000000..8ed09411b18 --- /dev/null +++ b/examples/flows/standard/summarizing-film-with-autogpt/system_prompt.jinja2 @@ -0,0 +1 @@ +You are {{name}}, {{role}} Play to your strengths as an LLM and pursue simple strategies with no legal complications to complete all goals. Your decisions must always be made independently without seeking user assistance. Performance Evaluation: 1. Continuously review and analyze your actions to ensure you are performing to the best of your abilities. 2. Constructively self-criticize your big-picture behavior constantly. 3. Reflect on past decisions and strategies to refine your approach. 4. Every command has a cost, so be smart and efficient. Aim to complete tasks in the least number of steps. \ No newline at end of file diff --git a/examples/flows/standard/summarizing-film-with-autogpt/triggering_prompt.jinja2 b/examples/flows/standard/summarizing-film-with-autogpt/triggering_prompt.jinja2 new file mode 100644 index 00000000000..0d50ff98ed8 --- /dev/null +++ b/examples/flows/standard/summarizing-film-with-autogpt/triggering_prompt.jinja2 @@ -0,0 +1 @@ +Determine which next function to use, and respond using stringfield JSON object. If you have completed all your tasks, make sure to use the 'finish' function to signal and remember show your results. \ No newline at end of file diff --git a/examples/flows/standard/summarizing-film-with-autogpt/user_prompt.jinja2 b/examples/flows/standard/summarizing-film-with-autogpt/user_prompt.jinja2 new file mode 100644 index 00000000000..9ee2b39191b --- /dev/null +++ b/examples/flows/standard/summarizing-film-with-autogpt/user_prompt.jinja2 @@ -0,0 +1 @@ +Goals: {{goals}} \ No newline at end of file diff --git a/examples/flows/standard/summarizing-film-with-autogpt/util.py b/examples/flows/standard/summarizing-film-with-autogpt/util.py new file mode 100644 index 00000000000..22e1ef7724e --- /dev/null +++ b/examples/flows/standard/summarizing-film-with-autogpt/util.py @@ -0,0 +1,163 @@ +import time +from typing import List +import re +import tiktoken +import logging +import sys +import json + +FORMATTER = logging.Formatter( + fmt="[%(asctime)s] %(name)-8s %(levelname)-8s %(message)s", + datefmt="%Y-%m-%d %H:%M:%S %z", +) + + +def get_logger(name: str, level=logging.INFO) -> logging.Logger: + logger = logging.Logger(name) + # log to sys.stdout for backward compatibility. + # TODO: May need to be removed in the future, after local/blob file stream are fully supported. + stdout_handler = logging.StreamHandler(sys.stdout) + stdout_handler.setFormatter(FORMATTER) + logger.addHandler(stdout_handler) + logger.setLevel(level) + return logger + + +def parse_reply(text: str): + try: + parsed = json.loads(text, strict=False) + except json.JSONDecodeError: + preprocessed_text = preprocess_json_input(text) + try: + parsed = json.loads(preprocessed_text, strict=False) + except Exception: + return {"Error": f"Could not parse invalid json: {text}"} + except TypeError: + return {"Error": f"the JSON object must be str, bytes or bytearray not {type(text)}"} + return parsed + + +def count_message_tokens( + messages: List, model: str = "gpt-3.5-turbo-0301" +) -> int: + """ + Returns the number of tokens used by a list of messages. + + Args: + messages (list): A list of messages, each of which is a dictionary + containing the role and content of the message. + model (str): The name of the model to use for tokenization. + Defaults to "gpt-3.5-turbo-0301". + + Returns: + int: The number of tokens used by the list of messages. + """ + try: + encoding = tiktoken.encoding_for_model(model) + except KeyError: + encoding = tiktoken.get_encoding("cl100k_base") + if model == "gpt-3.5-turbo": + # !Note: gpt-3.5-turbo may change over time. + # Returning num tokens assuming gpt-3.5-turbo-0301.") + return count_message_tokens(messages, model="gpt-3.5-turbo-0301") + elif model == "gpt-4": + # !Note: gpt-4 may change over time. Returning num tokens assuming gpt-4-0314.") + return count_message_tokens(messages, model="gpt-4-0314") + elif model == "gpt-3.5-turbo-0301": + tokens_per_message = ( + 4 # every message follows <|start|>{role/name}\n{content}<|end|>\n + ) + tokens_per_name = -1 # if there's a name, the role is omitted + elif model == "gpt-4-0314": + tokens_per_message = 3 + tokens_per_name = 1 + else: + raise NotImplementedError( + f"num_tokens_from_messages() is not implemented for model {model}.\n" + " See https://github.com/openai/openai-python/blob/main/chatml.md for" + " information on how messages are converted to tokens." + ) + num_tokens = 0 + for message in messages: + num_tokens += tokens_per_message + for key, value in message.items(): + num_tokens += len(encoding.encode(value)) + if key == "name": + num_tokens += tokens_per_name + num_tokens += 3 # every reply is primed with <|start|>assistant<|message|> + return num_tokens + + +def count_string_tokens(string: str, model_name="gpt-3.5-turbo") -> int: + """ + Returns the number of tokens in a text string. + + Args: + string (str): The text string. + model_name (str): The name of the encoding to use. (e.g., "gpt-3.5-turbo") + + Returns: + int: The number of tokens in the text string. + """ + encoding = tiktoken.encoding_for_model(model_name) + return len(encoding.encode(string)) + + +def create_chat_message(role, content, name=None): + """ + Create a chat message with the given role and content. + + Args: + role (str): The role of the message sender, e.g., "system", "user", or "assistant". + content (str): The content of the message. + + Returns: + dict: A dictionary containing the role and content of the message. + """ + if name is None: + return {"role": role, "content": content} + else: + return {"role": role, "name": name, "content": content} + + +def generate_context(prompt, full_message_history, user_prompt, model="gpt-3.5-turbo"): + current_context = [ + create_chat_message("system", prompt), + create_chat_message( + "system", f"The current time and date is {time.strftime('%c')}" + ), + create_chat_message("user", user_prompt), + ] + + # Add messages from the full message history until we reach the token limit + next_message_to_add_index = len(full_message_history) - 1 + insertion_index = len(current_context) + # Count the currently used tokens + current_tokens_used = count_message_tokens(current_context, model) + return ( + next_message_to_add_index, + current_tokens_used, + insertion_index, + current_context, + ) + + +def preprocess_json_input(input_str: str) -> str: + # Replace single backslashes with double backslashes, while leaving already escaped ones intact + corrected_str = re.sub(r'(? 2: + page += decode_str(content) + if not content.endswith("\n"): + page += "\n" + obs = get_page_sentence(page, count=count) + return obs diff --git a/examples/flows/standard/web-classification/.promptflow/flow.tools.json b/examples/flows/standard/web-classification/.promptflow/flow.tools.json new file mode 100644 index 00000000000..648c7b0697e --- /dev/null +++ b/examples/flows/standard/web-classification/.promptflow/flow.tools.json @@ -0,0 +1,73 @@ +{ + "package": {}, + "code": { + "fetch_text_content_from_url.py": { + "type": "python", + "inputs": { + "url": { + "type": [ + "string" + ] + } + }, + "function": "fetch_text_content_from_url" + }, + "summarize_text_content.jinja2": { + "type": "llm", + "inputs": { + "text": { + "type": [ + "string" + ] + } + }, + "description": "Summarize webpage content into a short paragraph." + }, + "summarize_text_content__variant_1.jinja2": { + "type": "llm", + "inputs": { + "text": { + "type": [ + "string" + ] + } + } + }, + "prepare_examples.py": { + "type": "python", + "function": "prepare_examples" + }, + "classify_with_llm.jinja2": { + "type": "llm", + "inputs": { + "url": { + "type": [ + "string" + ] + }, + "examples": { + "type": [ + "string" + ] + }, + "text_content": { + "type": [ + "string" + ] + } + }, + "description": "Multi-class classification of a given url and text content." + }, + "convert_to_dict.py": { + "type": "python", + "inputs": { + "input_str": { + "type": [ + "string" + ] + } + }, + "function": "convert_to_dict" + } + } +} \ No newline at end of file diff --git a/examples/flows/standard/web-classification/README.md b/examples/flows/standard/web-classification/README.md new file mode 100644 index 00000000000..cd8a507727a --- /dev/null +++ b/examples/flows/standard/web-classification/README.md @@ -0,0 +1,107 @@ +# Web Classification + +This is a flow demonstrating multi-class classification with LLM. Given an url, it will classify the url into one web category with just a few shots, simple summarization and classification prompts. + +## Tools used in this flow +- LLM Tool +- Python Tool + +## What you will learn + +In this flow, you will learn +- how to compose a classification flow with LLM. +- how to feed few shots to LLM classifier. + +## Prerequisites + +Install promptflow sdk and other dependencies: +```bash +pip install -r requirements.txt +``` + +## Getting Started + +### 1. Setup connection + +Prepare your Azure Open AI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one. + +```bash +# Override keys with --set to avoid yaml file changes +pf connection create --file azure_openai.yml --set api_key= api_base= +``` + +### 2. Configure the flow with your connection +`flow.dag.yaml` is already configured with connection named `azure_open_ai_connection`. + +### 3. Test flow with single line data + +```bash +# test with default input value in flow.dag.yaml +pf flow test --flow . +# test with user specified inputs +pf flow test --flow . --inputs url='https://www.microsoft.com/en-us/d/xbox-wireless-controller-stellar-shift-special-edition/94fbjc7h0h6h' +``` + +### 4. Run with multi-line data + +```bash +# create run using command line args +pf run create --flow . --data ./data.jsonl --stream +# create run using yaml file +pf run create --file run.yml --stream +``` + +```bash +# list run +pf run list +# show run +pf run show --name "web_classification_variant_1_20230724_173442_973403" +# show run outputs + +pf run show-details --name "web_classification_variant_1_20230724_173442_973403" +``` + +### 5. Run with classification evaluation flow + +create `evaluation` run: +```bash +# create run using command line args +pf run create --flow ../../evaluation/classification-accuracy-eval --data ./data.jsonl --column-mapping groundtruth='${data.answer}' prediction='${run.outputs.category}' --run "web_classification_variant_1_20230724_173442_973403" --stream +# create run using yaml file +pf run create --file run_evaluation.yml --run "web_classification_variant_1_20230724_173442_973403" --stream +``` + +```bash +pf run show-details --name "classification_accuracy_eval_default_20230724_173628_639497" +pf run show-metrics --name "classification_accuracy_eval_default_20230724_173628_639497" +pf run visualize --name "classification_accuracy_eval_default_20230724_173628_639497" +``` + + +### 6. Submit run to cloud +```bash +# set default workspace +az account set -s 96aede12-2f73-41cb-b983-6d11a904839b +az configure --defaults group="promptflow" workspace="promptflow-eastus" + +# create run +pfazure run create --flow . --data ./data.jsonl --stream --runtime demo-mir --subscription 96aede12-2f73-41cb-b983-6d11a904839b -g promptflow -w promptflow-eastus +pfazure run create --flow . --data ./data.jsonl --stream # serverless compute +pfazure run create --file run.yml --runtime demo-mir +pfazure run create --file run.yml --stream # serverless compute + + +pfazure run stream --name "web_classification_default_20230724_173705_462735" +pfazure run show-details --name "web_classification_default_20230724_173705_462735" +pfazure run show-metrics --name "web_classification_default_20230724_173705_462735" + +# create evaluation run +pfazure run create --flow ../../evaluation/classification-accuracy-eval --data ./data.jsonl --column-mapping groundtruth='${data.answer}' prediction='${run.outputs.category}' --run "web_classification_default_20230724_173705_462735" --runtime demo-mir +pfazure run create --file run_evaluation.yml --run "web_classification_default_20230724_173705_462735" --stream # serverless compute + +pfazure run stream --name "classification_accuracy_eval_default_20230724_173843_841080" +pfazure run show --name "classification_accuracy_eval_default_20230724_173843_841080" +pfazure run show-details --name "classification_accuracy_eval_default_20230724_173843_841080" +pfazure run show-metrics --name "classification_accuracy_eval_default_20230724_173843_841080" +pfazure run visualize --name "classification_accuracy_eval_default_20230724_173843_841080" +``` \ No newline at end of file diff --git a/examples/flows/standard/web-classification/azure_openai.yml b/examples/flows/standard/web-classification/azure_openai.yml new file mode 100644 index 00000000000..5646047bebe --- /dev/null +++ b/examples/flows/standard/web-classification/azure_openai.yml @@ -0,0 +1,7 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/AzureOpenAIConnection.schema.json +name: azure_open_ai_connection +type: azure_open_ai +api_key: "" +api_base: "aoai-api-endpoint" +api_type: "azure" +api_version: "2023-03-15-preview" diff --git a/examples/flows/standard/web-classification/classify_with_llm.jinja2 b/examples/flows/standard/web-classification/classify_with_llm.jinja2 new file mode 100644 index 00000000000..7488c286153 --- /dev/null +++ b/examples/flows/standard/web-classification/classify_with_llm.jinja2 @@ -0,0 +1,16 @@ +Your task is to classify a given url into one of the following types: +Movie, App, Academic, Channel, Profile, PDF or None based on the text content information. +The classification will be based on the url, the webpage text content summary, or both. + +Here are a few examples: +{% for ex in examples %} +URL: {{ex.url}} +Text content: {{ex.text_content}} +OUTPUT: +{"category": "{{ex.category}}", "evidence": "{{ex.evidence}}"} + +{% endfor %} + +For a given URL : {{url}}, and text content: {{text_content}}. +Classify above url to complete the category and indicate evidence. +OUTPUT: diff --git a/examples/flows/standard/web-classification/convert_to_dict.py b/examples/flows/standard/web-classification/convert_to_dict.py new file mode 100644 index 00000000000..3b287df5c65 --- /dev/null +++ b/examples/flows/standard/web-classification/convert_to_dict.py @@ -0,0 +1,12 @@ +import json + +from promptflow import tool + + +@tool +def convert_to_dict(input_str: str): + try: + return json.loads(input_str) + except Exception as e: + print("input is not valid, error: {}".format(e)) + return {"category": "None", "evidence": "None"} diff --git a/examples/flows/standard/web-classification/data.jsonl b/examples/flows/standard/web-classification/data.jsonl new file mode 100644 index 00000000000..87beaaa7268 --- /dev/null +++ b/examples/flows/standard/web-classification/data.jsonl @@ -0,0 +1,3 @@ +{"url": "https://www.youtube.com/watch?v=o5ZQyXaAv1g", "answer": "Channel", "evidence": "Url"} +{"url": "https://arxiv.org/abs/2307.04767", "answer": "Academic", "evidence": "Text content"} +{"url": "https://play.google.com/store/apps/details?id=com.twitter.android", "answer": "App", "evidence": "Both"} diff --git a/examples/flows/standard/web-classification/fetch_text_content_from_url.py b/examples/flows/standard/web-classification/fetch_text_content_from_url.py new file mode 100644 index 00000000000..1ff7f792909 --- /dev/null +++ b/examples/flows/standard/web-classification/fetch_text_content_from_url.py @@ -0,0 +1,30 @@ +import bs4 +import requests + +from promptflow import tool + + +@tool +def fetch_text_content_from_url(url: str): + # Send a request to the URL + try: + headers = { + "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/113.0.0.0 Safari/537.36 Edg/113.0.1774.35" + } + response = requests.get(url, headers=headers) + if response.status_code == 200: + # Parse the HTML content using BeautifulSoup + soup = bs4.BeautifulSoup(response.text, "html.parser") + soup.prettify() + return soup.get_text()[:2000] + else: + msg = ( + f"Get url failed with status code {response.status_code}.\nURL: {url}\nResponse: " + f"{response.text[:100]}" + ) + print(msg) + return "No available content" + except Exception as e: + print("Get url failed with error: {}".format(e)) + return "No available content" diff --git a/examples/flows/standard/web-classification/flow.dag.yaml b/examples/flows/standard/web-classification/flow.dag.yaml new file mode 100644 index 00000000000..f6af62f6f15 --- /dev/null +++ b/examples/flows/standard/web-classification/flow.dag.yaml @@ -0,0 +1,83 @@ +inputs: + url: + type: string + default: https://www.microsoft.com/en-us/d/xbox-wireless-controller-stellar-shift-special-edition/94fbjc7h0h6h +outputs: + category: + type: string + reference: ${convert_to_dict.output.category} + evidence: + type: string + reference: ${convert_to_dict.output.evidence} +nodes: +- name: fetch_text_content_from_url + type: python + source: + type: code + path: fetch_text_content_from_url.py + inputs: + url: ${inputs.url} +- name: summarize_text_content + use_variants: true +- name: prepare_examples + type: python + source: + type: code + path: prepare_examples.py + inputs: {} +- name: classify_with_llm + type: llm + source: + type: code + path: classify_with_llm.jinja2 + inputs: + deployment_name: text-davinci-003 + max_tokens: 128 + temperature: 0.2 + url: ${inputs.url} + text_content: ${summarize_text_content.output} + examples: ${prepare_examples.output} + provider: AzureOpenAI + connection: azure_open_ai_connection + api: completion +- name: convert_to_dict + type: python + source: + type: code + path: convert_to_dict.py + inputs: + input_str: ${classify_with_llm.output} +node_variants: + summarize_text_content: + variants: + variant_0: + node: + type: llm + source: + type: code + path: summarize_text_content.jinja2 + inputs: + deployment_name: text-davinci-003 + max_tokens: 128 + temperature: 0.2 + text: ${fetch_text_content_from_url.output} + provider: AzureOpenAI + connection: azure_open_ai_connection + api: completion + module: promptflow.tools.aoai + variant_1: + node: + type: llm + source: + type: code + path: summarize_text_content__variant_1.jinja2 + inputs: + deployment_name: text-davinci-003 + max_tokens: 256 + temperature: 0.3 + text: ${fetch_text_content_from_url.output} + provider: AzureOpenAI + connection: azure_open_ai_connection + api: completion + module: promptflow.tools.aoai + default_variant_id: variant_0 diff --git a/examples/flows/standard/web-classification/prepare_examples.py b/examples/flows/standard/web-classification/prepare_examples.py new file mode 100644 index 00000000000..c4ccb76d732 --- /dev/null +++ b/examples/flows/standard/web-classification/prepare_examples.py @@ -0,0 +1,44 @@ +from promptflow import tool + + +@tool +def prepare_examples(): + return [ + { + "url": "https://play.google.com/store/apps/details?id=com.spotify.music", + "text_content": "Spotify is a free music and podcast streaming app with millions of songs, albums, and " + "original podcasts. It also offers audiobooks, so users can enjoy thousands of stories. " + "It has a variety of features such as creating and sharing music playlists, discovering " + "new music, and listening to popular and exclusive podcasts. It also has a Premium " + "subscription option which allows users to download and listen offline, and access " + "ad-free music. It is available on all devices and has a variety of genres and artists " + "to choose from.", + "category": "App", + "evidence": "Both", + }, + { + "url": "https://www.youtube.com/channel/UC_x5XG1OV2P6uZZ5FSM9Ttw", + "text_content": "NFL Sunday Ticket is a service offered by Google LLC that allows users to watch NFL " + "games on YouTube. It is available in 2023 and is subject to the terms and privacy policy " + "of Google LLC. It is also subject to YouTube's terms of use and any applicable laws.", + "category": "Channel", + "evidence": "URL", + }, + { + "url": "https://arxiv.org/abs/2303.04671", + "text_content": "Visual ChatGPT is a system that enables users to interact with ChatGPT by sending and " + "receiving not only languages but also images, providing complex visual questions or " + "visual editing instructions, and providing feedback and asking for corrected results. " + "It incorporates different Visual Foundation Models and is publicly available. Experiments " + "show that Visual ChatGPT opens the door to investigating the visual roles of ChatGPT with " + "the help of Visual Foundation Models.", + "category": "Academic", + "evidence": "Text content", + }, + { + "url": "https://ab.politiaromana.ro/", + "text_content": "There is no content available for this text.", + "category": "None", + "evidence": "None", + }, + ] diff --git a/examples/flows/standard/web-classification/requirements.txt b/examples/flows/standard/web-classification/requirements.txt new file mode 100644 index 00000000000..fda053472d8 --- /dev/null +++ b/examples/flows/standard/web-classification/requirements.txt @@ -0,0 +1,4 @@ +--extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ +promptflow +promptflow-tools +bs4 \ No newline at end of file diff --git a/examples/flows/standard/web-classification/run.yml b/examples/flows/standard/web-classification/run.yml new file mode 100644 index 00000000000..9522372f0e0 --- /dev/null +++ b/examples/flows/standard/web-classification/run.yml @@ -0,0 +1,4 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Run.schema.json +flow: . +data: data.jsonl +variant: ${summarize_text_content.variant_1} \ No newline at end of file diff --git a/examples/flows/standard/web-classification/run_evaluation.yml b/examples/flows/standard/web-classification/run_evaluation.yml new file mode 100644 index 00000000000..3b0bcf3d71b --- /dev/null +++ b/examples/flows/standard/web-classification/run_evaluation.yml @@ -0,0 +1,7 @@ +$schema: https://azuremlschemas.azureedge.net/promptflow/latest/Run.schema.json +flow: ../../evaluation/classification-accuracy-eval +data: data.jsonl +run: web_classification_variant_1_20230724_173442_973403 # replace with your run name +column_mapping: + groundtruth: ${data.answer} + prediction: ${run.outputs.category} \ No newline at end of file diff --git a/examples/flows/standard/web-classification/summarize_text_content.jinja2 b/examples/flows/standard/web-classification/summarize_text_content.jinja2 new file mode 100644 index 00000000000..7162d9afc30 --- /dev/null +++ b/examples/flows/standard/web-classification/summarize_text_content.jinja2 @@ -0,0 +1,5 @@ +Please summarize the following text in one paragraph. 100 words. +Do not add any information that is not in the text. + +Text: {{text}} +Summary: diff --git a/examples/flows/standard/web-classification/summarize_text_content__variant_1.jinja2 b/examples/flows/standard/web-classification/summarize_text_content__variant_1.jinja2 new file mode 100644 index 00000000000..894abf07e27 --- /dev/null +++ b/examples/flows/standard/web-classification/summarize_text_content__variant_1.jinja2 @@ -0,0 +1,5 @@ +Please summarize some keywords of this paragraph and have some details of each keywords. +Do not add any information that is not in the text. + +Text: {{text}} +Summary: diff --git a/examples/requirements.txt b/examples/requirements.txt new file mode 100644 index 00000000000..b105a36fe89 --- /dev/null +++ b/examples/requirements.txt @@ -0,0 +1,8 @@ +# remove when we publish to pypi +--extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ +promptflow[azure]==0.0.101192967 +promptflow-tools==0.0.1.dev0 +python-dotenv +langchain +jinja2 +bs4 \ No newline at end of file diff --git a/examples/setup.sh b/examples/setup.sh new file mode 100644 index 00000000000..31f52c43494 --- /dev/null +++ b/examples/setup.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# +pip install -r requirements.txt +# + +pip list \ No newline at end of file diff --git a/examples/tutorials/flow-deploy/.gitignore b/examples/tutorials/flow-deploy/.gitignore new file mode 100644 index 00000000000..6f5076d2244 --- /dev/null +++ b/examples/tutorials/flow-deploy/.gitignore @@ -0,0 +1,2 @@ +*.sqlite +linux/flow \ No newline at end of file diff --git a/examples/tutorials/flow-deploy/deploy.md b/examples/tutorials/flow-deploy/deploy.md new file mode 100644 index 00000000000..759ad6b6a08 --- /dev/null +++ b/examples/tutorials/flow-deploy/deploy.md @@ -0,0 +1,44 @@ +# Flow Serve & Export + +## Serve a flow as local http endpoint + +The following CLI commands allows you serve a flow as an endpoint. + +Below is taking [basic-with-connection](../../flows/standard/basic-with-connection) flow as example. +- Before we start, please ensure you have create the connection required by flow. +See [connections](../../connections/) for more details about connection. + + +Serve a flow as endpoint: +```bash +pf flow serve --source ../../flows/standard/basic-with-connection --port 8080 --host localhost +``` + +Test the endpoint: +```bash +curl http://localhost:8080/score --data '{"text":"Hello world!"}' -X POST -H "Content-Type: application/json" +``` + +## Export a flow as docker format + +Note that we must have all dependent connections created before exporting as docker: + +```bash +pf connection create --file ../../flows/standard/basic-with-connection/custom.yml +``` + +The following CLI commands allows you export a flow as a sharable folder with a Dockerfile and its dependencies. + +```bash +pf flow export --source ../../flows/standard/basic-with-connection --output --format docker +``` + +You'll be asked to input a migration secret when running this command, which needs to be provided when you run the built docker image. +You can also provide the key via `--migration-secret` directly or passing it with a file via `--migration-secret-file`. + +More details about how to use the exported docker can be seen in `/README.md`. +Part of sample output are under [./linux](./linux/) so you can also check [this README](./linux/README.md) directly. + +## Export a flow as portable package format + +WIP. \ No newline at end of file diff --git a/examples/tutorials/flow-deploy/linux/Dockerfile b/examples/tutorials/flow-deploy/linux/Dockerfile new file mode 100644 index 00000000000..1287fc721c8 --- /dev/null +++ b/examples/tutorials/flow-deploy/linux/Dockerfile @@ -0,0 +1,25 @@ +# syntax=docker/dockerfile:1 +FROM docker.io/continuumio/miniconda3:latest + +WORKDIR / + +COPY ./flow /flow + +# create conda environment, and use as default python environment +ENV CONDA_ENVIRONMENT_PATH=/opt/conda/envs/promptflow-serve +ENV CONDA_DEFAULT_ENVIRONMENT=$CONDA_ENVIRONMENT_PATH +ENV PATH $CONDA_DEFAULT_ENVIRONMENT/bin:$PATH +RUN conda create -n promptflow-serve python=3.9.16 pip=23.0.1 -q -y && \ + conda run -n promptflow-serve \ + pip install -r /flow/requirements_txt && \ + conda run -n promptflow-serve pip cache purge && \ + conda clean -a -y + +RUN conda run -n promptflow-serve sh /flow/setup.sh + +EXPOSE 8080 + +COPY ./connections.sqlite / +COPY ./connections_setup.py / +COPY ./start.sh / +CMD ./start.sh diff --git a/examples/tutorials/flow-deploy/linux/README.md b/examples/tutorials/flow-deploy/linux/README.md new file mode 100644 index 00000000000..953ec705d33 --- /dev/null +++ b/examples/tutorials/flow-deploy/linux/README.md @@ -0,0 +1,115 @@ +# How to use exported dockerfile + +## Exported Dockerfile Structure + +Exported Dockerfile & its dependencies are located in the same folder. The structure is as below: +- flow: the folder contains all the flow files + - ... +- Dockerfile: the dockerfile to build the image +- connections.sqlite: the sqlite database file to store the connections used in the flow +- connections_setup.py: the python script to migrate the connections used in the flow +- start.sh: the script used in `CMD` of `Dockerfile` to start the service +- docker-compose.yaml: a sample compose file to run the service +- README.md: the readme file to describe how to use the dockerfile + +## Build Docker image + +Like other Dockerfile, you need to build the image first. You can tag the image with any name you want. In this example, we use `promptflow-serve`. + +After cd to the output directory, run the command below: + +```bash +docker build . -t promptflow-serve +``` + +## Run Docker image + +Run the docker image will start a service to serve the flow inside the container. Service will listen on port 8080. +You can map the port to any port on the host machine as you want. + +If the service involves connections, you need to migrate the connections before the first request to the service. +Given api_key in connections are secrets, we have provided a way to migrate them with the `migration-secret` you have provided in export command. + +### Manage migration-secret with `docker-secret` + +As a pre-requirement of connections migration, you need to create a docker secret named `MIGRATION_SECRET` +to store the migration secret first. Sample command is like below: + +```bash +#### Init host machine as a swarm manager +docker swarm init +#### Create a secret to store the migration secret +# You can create the docker secret directly from shell with bash +(read -sp "Enter your migration secret: "; echo $REPLY) | docker secret create MIGRATION_SECRET - +# or you can also create secret from a local file with migration secret as its content, like in Powershell +docker secret create MIGRATION_SECRET +``` + +You can check below documents for more details: +- [Swam mode overview](https://docs.docker.com/engine/swarm/) +- [Secrets management](https://docs.docker.com/engine/swarm/secrets/) + +### Run with `docker-service` [WIP] + +To avoid manually migrate the connections, you can use `docker-secret` to manage the migration secret +and `docker-service` to run the service: + +```bash +#### Start the service +docker service create --name promptflow-service -p 8080:8080 --secret MIGRATION_SECRET promptflow-serve +``` + +You can check below documents for more details: +- [Run Docker Engine in swarm mode](https://docs.docker.com/engine/swarm/swarm-mode/) + +### Run with `docker-compose` + +You can also use `docker-secret` in your compose file and use compose file to start your service: + +```bash +#### Deploy the service +docker stack deploy --compose-file=./docker-compose.yaml service1 +``` + +Note that you need to deploy the service to a swarm cluster to use `docker-secret`. +So connections won't be migrated successfully if you run `docker-compose` directly. +More details can be found in the official document: +- [Deploy a stack to a swarm](https://docs.docker.com/engine/swarm/stack-deploy/) + +In the sample compose file `docker-compose.yaml` in the output directory, we claim secret `MIGRATION_SECRET` +as external, which means you need to create the secret first before running the compose file. + +You can also specify the migration secret file and docker image in the compose file: + +```yaml +services: + promptflow: + image: +... +secrets: + MIGRATION_SECRET: + file: +``` + +Official document: +- [Manage secrets in Docker Compose](https://docs.docker.com/compose/compose-file/compose-file-v3/#secrets) +- [Using secrets in Compose](https://docs.docker.com/compose/use-secrets/) + +## Test the endpoint +After start the service, you can use curl to test it: + +```bash +curl http://localhost:8080/score --data '{"text":"Hello world!"}' -X POST -H "Content-Type: application/json" +``` + +## Advanced: Run with `docker run` + +If you want to debug or have provided a wrong migration-secret, you can run the docker image directly and manually migrate the connections via below commands: + +```bash +docker run -p 8080:8080 promptflow-serve +#### Migrate connections +docker exec -it python connections_setup.py --file /connections.sqlite --migration-secret --clean +``` + +Note that the command to migrate the connections must be run before any requests to the service. diff --git a/examples/tutorials/flow-deploy/linux/connections_setup.py b/examples/tutorials/flow-deploy/linux/connections_setup.py new file mode 100644 index 00000000000..fb509027b36 --- /dev/null +++ b/examples/tutorials/flow-deploy/linux/connections_setup.py @@ -0,0 +1,52 @@ +import argparse +import base64 +import hashlib +import os +import shutil +from pathlib import Path + +import keyring + +from promptflow._sdk._constants import LOCAL_MGMT_DB_PATH + + +def update_migration_secret(migration_secret: str): + keyring.set_password( + "promptflow", + "encryption_key", + base64.urlsafe_b64encode(hashlib.sha256(migration_secret.encode("utf-8")).digest()).decode("utf-8"), + ) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser() + parser.add_argument("--file", type=str, required=True, help="db file to migrate") + parser.add_argument("--migration-secret", type=str, help="migration secret used for the db") + parser.add_argument( + "--migration-secret-file", + type=str, + help="migration secret used for the db in file, won't take effect if migration-secret is provided", + ) + parser.add_argument("--ignore-errors", action="store_true", help="whether to ignore errors") + parser.add_argument("--clean", action="store_true", help="whether to delete the db file after migration") + args = parser.parse_args() + + if not args.migration_secret and args.migration_secret_file: + if not os.path.isfile(args.migration_secret_file): + raise FileNotFoundError(f"migration secret file {args.migration_secret_file} not found.") + with open(args.migration_secret_file, "r") as f: + args.migration_secret = f.read() + + if not args.migration_secret: + raise ValueError("either migration-secret or migration-secret-file should be provided") + + if os.path.isfile(args.file): + Path(LOCAL_MGMT_DB_PATH).parent.mkdir(parents=True, exist_ok=True) + shutil.copy(args.file, LOCAL_MGMT_DB_PATH) + update_migration_secret(args.migration_secret) + if args.clean: + os.remove(args.file) + elif args.ignore_errors: + pass + else: + raise FileNotFoundError(f"db file {args.file} not found.") diff --git a/examples/tutorials/flow-deploy/linux/docker-compose.yaml b/examples/tutorials/flow-deploy/linux/docker-compose.yaml new file mode 100644 index 00000000000..42f4a5e5da2 --- /dev/null +++ b/examples/tutorials/flow-deploy/linux/docker-compose.yaml @@ -0,0 +1,17 @@ +version: "3.8" +services: + promptflow: + image: promptflow-serve + ports: + - "8080:8080" + secrets: + - MIGRATION_SECRET + deploy: + replicas: 1 + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 +secrets: + MIGRATION_SECRET: + external: true diff --git a/examples/tutorials/flow-deploy/linux/start.sh b/examples/tutorials/flow-deploy/linux/start.sh new file mode 100644 index 00000000000..b70bbe739a6 --- /dev/null +++ b/examples/tutorials/flow-deploy/linux/start.sh @@ -0,0 +1,2 @@ +python connections_setup.py --file connections.sqlite --migration-secret-file /run/secrets/MIGRATION_SECRET --clean --ignore-errors +pf flow serve --source flow --host 0.0.0.0 diff --git a/examples/tutorials/flow-in-pipeline/data.tsv b/examples/tutorials/flow-in-pipeline/data.tsv new file mode 100644 index 00000000000..2063c94af79 --- /dev/null +++ b/examples/tutorials/flow-in-pipeline/data.tsv @@ -0,0 +1,21 @@ +url answer evidence +https://www.youtube.com/watch?v=o5ZQyXaAv1g Channel Url +https://www.youtube.com/watch?v=o5ZQyXaAv1g Channel Url +https://www.youtube.com/watch?v=o5ZQyXaAv1g Channel Url +https://www.youtube.com/watch?v=o5ZQyXaAv1g Channel Url +https://www.youtube.com/watch?v=o5ZQyXaAv1g Profile Url +https://www.youtube.com/watch?v=o5ZQyXaAv1g Profile Url +https://www.youtube.com/watch?v=o5ZQyXaAv1g Academic Text content +https://www.youtube.com/watch?v=o5ZQyXaAv1g None Text content +https://www.youtube.com/watch?v=o5ZQyXaAv1g Academic Text content +https://www.youtube.com/watch?v=o5ZQyXaAv1g None Text content +https://www.youtube.com/watch?v=o5ZQyXaAv1g Academic Text content +https://www.youtube.com/watch?v=o5ZQyXaAv1g Profile Url +https://www.youtube.com/watch?v=o5ZQyXaAv1g Academic Text content +https://www.youtube.com/watch?v=o5ZQyXaAv1g Profile Url +https://www.youtube.com/watch?v=o5ZQyXaAv1g Profile Text content +https://www.youtube.com/watch?v=o5ZQyXaAv1g POI Url +https://www.youtube.com/watch?v=o5ZQyXaAv1g None Text content +https://www.youtube.com/watch?v=o5ZQyXaAv1g Academic Text content +https://www.youtube.com/watch?v=o5ZQyXaAv1g Academic Text content +https://www.youtube.com/watch?v=o5ZQyXaAv1g Academic Text content \ No newline at end of file diff --git a/examples/tutorials/flow-in-pipeline/pipeline.ipynb b/examples/tutorials/flow-in-pipeline/pipeline.ipynb new file mode 100644 index 00000000000..e2e5098fa6e --- /dev/null +++ b/examples/tutorials/flow-in-pipeline/pipeline.ipynb @@ -0,0 +1,208 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Use Flow in Pipeline\n", + "\n", + "**Requirements** - In order to benefit from this tutorial, you will need:\n", + "- A basic understanding of Machine Learning\n", + "- An Azure account with an active subscription - [Create an account for free](https://azure.microsoft.com/free/?WT.mc_id=A261C142F)\n", + "- An Azure ML workspace - [Configure workspace](../../configuration.ipynb)\n", + "- A python environment\n", + "- Installed Azure Machine Learning Python SDK v2\n", + "- Installed PromptFlow SDK\n", + "\n", + "**Learning Objectives** - By the end of this tutorial, you should be able to:\n", + "- Connect to your AML workspace from the Python SDK\n", + "- Load a flow as a `ParallelComponent`\n", + "- Using the component along with other components loaded from yaml in one `PipelineJob`.\n", + "\n", + "**Motivations** - This guide will introduce how to use a flow along with other data processing steps in a pipeline.\n", + "\n", + "**Known issues** - This feature is not stable now and here are known issues we are actively fixing:\n", + "- You must include a `.promptflow/flow.tools.json` in the flow directory first. This file will automatically generated when you run the flow locally.\n", + "- Component of the same name (even with different version) can be created only once. An auto-generated component name based on hash will be used when component name & version are neither provided.\n", + "- The flow nodes can only run on computer cluster with managed identity assigned Azure ML Data Scientist role.\n", + "- connection/columns_mapping overwrite doesn't work for now.\n", + "- This feature works on canary workspace only for now: [sample job link](https://ml.azure.com/experiments/id/9ce1a534-9d3d-4761-a5e7-5299dd6912f1/runs/clever_leek_4xh6x9z7s5?wsid=/subscriptions/96aede12-2f73-41cb-b983-6d11a904839b/resourcegroups/promptflow/workspaces/promptflow-canary-dev&tid=72f988bf-86f1-41af-91ab-2d7cd011db47)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 0. Install dependent packages\n", + "\n", + "Please follow [configuration.ipynb](../../configuration.ipynb) to install dependent packages and connect to a workspace first." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -r ../../requirements.txt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Connect to MLClient and create necessary connections\n", + "Similar to other SDK in azure-ai-ml, you need to import related packages and prepare a ML client connecting to a specific workspace first." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Import required libraries\n", + "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n", + "\n", + "from azure.ai.ml import MLClient, Input\n", + "from azure.ai.ml.constants import AssetTypes\n", + "from azure.ai.ml.dsl import pipeline\n", + "from azure.ai.ml import load_component\n", + "from promptflow.azure import PFClient\n", + "\n", + "try:\n", + " credential = DefaultAzureCredential()\n", + " # Check if given credential can get token successfully.\n", + " credential.get_token(\"https://management.azure.com/.default\")\n", + "except Exception as ex:\n", + " # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential not work\n", + " credential = InteractiveBrowserCredential()\n", + "\n", + "# Get a handle to workspace\n", + "ml_client = MLClient.from_config(credential=credential)\n", + "\n", + "# Create PFClient connected to workspace\n", + "pf = PFClient(ml_client)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Load flow as a component\n", + "\n", + "Suppose you have already authored a flow, you can load it as component:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "flow_component = pf.load_as_component(\n", + " \"../../flows/standard/web-classification/\",\n", + " columns_mapping={\n", + " \"url\": \"${data.url}\",\n", + " \"groundtruth\": \"${data.answer}\",\n", + " },\n", + " component_type=\"parallel\",\n", + ")\n", + "\n", + "print(flow_component)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Use the component in a pipeline\n", + "\n", + "Then you can use this component along with other components in a pipeline:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "tsv2jsonl_component = load_component(\"./tsv2jsonl-component/component_spec.yaml\")\n", + "\n", + "\n", + "@pipeline\n", + "def pipeline_with_flow(input_data):\n", + " data_transfer = tsv2jsonl_component(input_data=input_data)\n", + "\n", + " flow_node = flow_component(\n", + " # this can be either a URI jsonl file or a URI folder containing multiple jsonl files\n", + " data=data_transfer.outputs.output_data,\n", + " # you can overwrite inputs mapping here\n", + " groundtruth=\"Channel\",\n", + " # this is to overwrite connection settings\n", + " connections={\n", + " # this is to overwrite connection related settings for a LLM node\n", + " # \"summarize_text_content\" is the node name\n", + " \"summarize_text_content\": {\n", + " \"deployment_name\": \"text-davinci-003\",\n", + " },\n", + " # you can overwrite custom connection input of a python node here\n", + " # \"convert_to_dict\": {\n", + " # \"conn1\": \"another_connection\"\n", + " # }\n", + " },\n", + " )\n", + " # node level run settings for flow node is similar to `ParallelComponent`\n", + " flow_node.logging_level = \"DEBUG\"\n", + " flow_node.max_concurrency_per_instance = 2\n", + " return flow_node.outputs\n", + "\n", + "\n", + "pipeline = pipeline_with_flow(\n", + " input_data=Input(path=\"./data.tsv\", type=AssetTypes.URI_FILE),\n", + ")\n", + "\n", + "pipeline.settings.default_compute = \"cpu-cluster\"\n", + "\n", + "created_job = ml_client.jobs.create_or_update(pipeline)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Like other pipeline jobs in azure-ai-ml, you can monitor the status of the job via `ml_client.jobs.stream`:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "ml_client.jobs.stream(created_job.name)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.17" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/tutorials/flow-in-pipeline/tsv2jsonl-component/component_spec.yaml b/examples/tutorials/flow-in-pipeline/tsv2jsonl-component/component_spec.yaml new file mode 100644 index 00000000000..984b8a6da30 --- /dev/null +++ b/examples/tutorials/flow-in-pipeline/tsv2jsonl-component/component_spec.yaml @@ -0,0 +1,27 @@ +$schema: https://azuremlschemas.azureedge.net/development/commandComponent.schema.json +type: command + +name: tsv2jsonl +display_name: TSV to JSONL +description: This is a command component to transfer a TSV file to JSONL file. +tags: + tag: tagvalue + owner: sdkteam + +version: 0.0.1 + +inputs: + input_data: + description: A tsv file + type: uri_file + +outputs: + output_data: + type: uri_file + +code: . + +command: >- + python main.py ${{inputs.input_data}} ${{outputs.output_data}} + +environment: azureml:AzureML-sklearn-1.0-ubuntu20.04-py38-cpu:33 diff --git a/examples/tutorials/flow-in-pipeline/tsv2jsonl-component/main.py b/examples/tutorials/flow-in-pipeline/tsv2jsonl-component/main.py new file mode 100644 index 00000000000..27e3097ffde --- /dev/null +++ b/examples/tutorials/flow-in-pipeline/tsv2jsonl-component/main.py @@ -0,0 +1,17 @@ +import csv +import sys +import json + + +def transfer(input_path, output_path): + with open(input_path, "r", encoding="utf-8") as f: + reader = csv.DictReader(f, delimiter="\t") + with open(output_path, "w", encoding="utf-8") as output_f: + for row_dict in reader: + json.dump(row_dict, output_f) + output_f.write("\n") + + +if __name__ == "__main__": + _, _input, _output = sys.argv + transfer(_input, _output) diff --git a/examples/tutorials/get-started/quickstart-azure.ipynb b/examples/tutorials/get-started/quickstart-azure.ipynb new file mode 100644 index 00000000000..8213dfab452 --- /dev/null +++ b/examples/tutorials/get-started/quickstart-azure.ipynb @@ -0,0 +1,440 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Get-started (Azure version)\n", + "\n", + "**Requirements** - In order to benefit from this tutorial, you will need:\n", + "- A basic understanding of Machine Learning\n", + "- An Azure account with an active subscription - [Create an account for free](https://azure.microsoft.com/free/?WT.mc_id=A261C142F)\n", + "- An Azure ML workspace - [Configure workspace](../../configuration.ipynb)\n", + "- A python environment\n", + "- Installed PromptFlow SDK\n", + "\n", + "**Learning Objectives** - By the end of this tutorial, you should be able to:\n", + "- Connect to your AML workspace from the Python SDK\n", + "- Create and develop a new promptflow run\n", + "- Evaluate the run with a evaluation flow\n", + "- Deploy the flow to a remote endpoint.\n", + "\n", + "**Motivations** - This guide will walk you through the main user journey of prompt flow code-first experience. You will learn how to create and develop your first prompt flow, test and evaluate it, then deploy it to production." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 0. Install dependent packages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -r ../../requirements.txt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 1. Connect to Azure Machine Learning Workspace\n", + "\n", + "The [workspace](https://docs.microsoft.com/en-us/azure/machine-learning/concept-workspace) is the top-level resource for Azure Machine Learning, providing a centralized place to work with all the artifacts you create when you use Azure Machine Learning. In this section we will connect to the workspace in which the job will be run.\n", + "\n", + "## 1.1 Import the required libraries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "\n", + "# Import required libraries\n", + "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n", + "from azure.ai.ml import MLClient\n", + "\n", + "# azure version promptflow apis\n", + "from promptflow.azure import PFClient" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1.2 Configure credential\n", + "\n", + "We are using `DefaultAzureCredential` to get access to workspace. \n", + "`DefaultAzureCredential` should be capable of handling most Azure SDK authentication scenarios. \n", + "\n", + "Reference for more available credentials if it does not work for you: [configure credential example](../../configuration.ipynb), [azure-identity reference doc](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity?view=azure-python)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " credential = DefaultAzureCredential()\n", + " # Check if given credential can get token successfully.\n", + " credential.get_token(\"https://management.azure.com/.default\")\n", + "except Exception as ex:\n", + " # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential not work\n", + " credential = InteractiveBrowserCredential()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1.3 Get a handle to the workspace\n", + "\n", + "We use config file to connect to a workspace. The Azure ML workspace should be configured with computer cluster. [Check this notebook for configure a workspace](../../configuration.ipynb)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Get a handle to workspace\n", + "ml_client = MLClient.from_config(credential=credential)\n", + "\n", + "pf = PFClient(ml_client)\n", + "\n", + "runtime = \"demo-mir\" # TODO remove this after serverless ready" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1.4 Create necessary connections\n", + "Connection helps securely store and manage secret keys or other sensitive credentials required for interacting with LLM and other external tools for example Azure Content Safety.\n", + "\n", + "In this notebook, we will use flow `web-classification` which uses connection `azure_open_ai_connection` inside, we need to set up the connection if we haven't added it before.\n", + "\n", + "Prepare your Azure Open AI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one.\n", + "\n", + "Please go to [workspace portal](https://ml.azure.com/), click `Prompt flow` -> `Connections` -> `Create`, then follow the instruction to create your own connections. \n", + "Learn more on [connections](https://learn.microsoft.com/en-us/azure/machine-learning/prompt-flow/concept-connections?view=azureml-api-2)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Create a new run\n", + "\n", + "`web-classification` is a flow demonstrating multi-class classification with LLM. Given an url, it will classify the url into one web category with just a few shots, simple summarization and classification prompts." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Set flow path and input data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# load flow\n", + "flow = \"../../flows/standard/web-classification\"\n", + "data = \"../../flows/standard/web-classification/data.jsonl\"" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Submit run" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# create run\n", + "base_run = pf.run(\n", + " flow=flow,\n", + " data=data,\n", + " runtime=runtime, # TODO hide this\n", + " connections={}, # TODO introduce this\n", + ")\n", + "print(base_run)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf.stream(base_run)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "details = pf.get_details(base_run)\n", + "details.head(10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf.visualize(base_run)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Evaluate your flow run result\n", + "Then you can use an evaluation method to evaluate your flow. The evaluation methods are also flows which use Python or LLM etc., to calculate metrics like accuracy, relevance score.\n", + "\n", + "In this notebook, we use `classification-accuracy-eval` flow to evaluate. This is a flow illustrating how to evaluate the performance of a classification system. It involves comparing each prediction to the groundtruth and assigns a \"Correct\" or \"Incorrect\" grade, and aggregating the results to produce metrics such as accuracy, which reflects how good the system is at classifying the data." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "eval_run = pf.run(\n", + " flow=\"../../flows/evaluation/classification-accuracy-eval\",\n", + " data=data,\n", + " run=base_run,\n", + " column_mapping={\n", + " \"groundtruth\": \"${data.answer}\",\n", + " \"prediction\": \"${run.outputs.category}\",\n", + " },\n", + " runtime=runtime,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf.stream(eval_run)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "details = pf.get_details(eval_run)\n", + "details.head(10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metrics = pf.get_metrics(eval_run)\n", + "print(json.dumps(metrics, indent=4))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf.visualize([base_run, eval_run])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create another run with different variant node\n", + "\n", + "In this example, `web-classification`'s node `summarize_text_content` has two variants: `variant_0` and `variant_1`. The difference between them is the inputs parameters:\n", + "\n", + "variant_0:\n", + "\n", + " - inputs:\n", + " - deployment_name: text-davinci-003\n", + " - max_tokens: '128'\n", + " - temperature: '0.2'\n", + " - text: ${fetch_text_content_from_url.output}\n", + "\n", + "variant_1:\n", + "\n", + " - inputs:\n", + " - deployment_name: text-davinci-003\n", + " - max_tokens: '256'\n", + " - temperature: '0.3'\n", + " - text: ${fetch_text_content_from_url.output}\n", + "\n", + "\n", + "You can check the whole flow definistion at [flow.dag.yaml](../../flows/standard/web-classification/flow.dag.yaml)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# use the variant1 of the summarize_text_content node.\n", + "variant_run = pf.run(\n", + " flow=flow,\n", + " data=data,\n", + " variant=\"${summarize_text_content.variant_1}\", # here we specify node \"summarize_text_content\" to use variant 1 verison.\n", + " runtime=runtime, # TODO hide this\n", + " connections={}, # TODO introduce this\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf.stream(variant_run)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "details = pf.get_details(variant_run)\n", + "details.head(10)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Run evaluation against variant run" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "eval_flow = \"../../flows/evaluation/classification-accuracy-eval\"\n", + "\n", + "eval_run_variant = pf.run(\n", + " flow=eval_flow,\n", + " data=\"../../flows/standard/web-classification/data.jsonl\", # path to the data file\n", + " run=variant_run, # use run as the variant\n", + " column_mapping={\n", + " # reference data\n", + " \"groundtruth\": \"${data.answer}\",\n", + " # reference the run's output\n", + " \"prediction\": \"${run.outputs.category}\",\n", + " }, \n", + " runtime=runtime,\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf.stream(eval_run_variant)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "details = pf.get_details(eval_run_variant)\n", + "details.head(10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metrics = pf.get_metrics(eval_run_variant)\n", + "print(json.dumps(metrics, indent=4))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf.visualize([eval_run, eval_run_variant])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Next Steps\n", + "\n", + "Learn more on how to:\n", + "- run the flow as a component in a azureml pipeline: [flow in pipeline](../flow-in-pipeline/pipeline.ipynb)." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "promptflow", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.17" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/tutorials/get-started/quickstart.ipynb b/examples/tutorials/get-started/quickstart.ipynb new file mode 100644 index 00000000000..f1b5bc383b3 --- /dev/null +++ b/examples/tutorials/get-started/quickstart.ipynb @@ -0,0 +1,423 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Get-started (Community version)\n", + "\n", + "**Requirements** - In order to benefit from this tutorial, you will need:\n", + "- A basic understanding of Machine Learning\n", + "- A python environment\n", + "\n", + "**Learning Objectives** - By the end of this tutorial, you should be able to:\n", + "- Create and develop a new promptflow run\n", + "- Evaluate the run with a evaluation flow\n", + "- Deploy the flow to a http endpoint.\n", + "\n", + "**Motivations** - This guide will walk you through the main user journey of prompt flow code-first experience. You will learn how to create and develop your first prompt flow, test and evaluate it, then deploy it to production." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 0. Install dependent packages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -r ../../requirements.txt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Create necessary connections\n", + "Connection helps securely store and manage secret keys or other sensitive credentials required for interacting with LLM and other external tools for example Azure Content Safety.\n", + "\n", + "In this notebook, we will use flow `web-classification` which uses connection `azure_open_ai_connection` inside, we need to set up the connection if we haven't added it before. After created, it's stored in local db and can be used in any flow.\n", + "\n", + "Prepare your Azure Open AI resource follow this [instruction](https://learn.microsoft.com/en-us/azure/cognitive-services/openai/how-to/create-resource?pivots=web-portal) and get your `api_key` if you don't have one." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from promptflow import PFClient\n", + "from promptflow.entities import AzureOpenAIConnection\n", + "\n", + "# client can help manage your runs and connections.\n", + "pf = PFClient()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " conn_name = \"azure_open_ai_connection\"\n", + " conn = pf.connections.get(name=conn_name)\n", + " print(\"using existing connection\")\n", + "except:\n", + " # TODO add guide on how to create an Azure Open AI resource.\n", + " connection = AzureOpenAIConnection(\n", + " name=conn_name,\n", + " api_key=\"\",\n", + " api_base=\"\",\n", + " api_type=\"azure\",\n", + " api_version=\"\",\n", + " )\n", + "\n", + " conn = pf.connections.create_or_update(connection)\n", + " print(\"successfully created connection\")\n", + "\n", + "print(conn)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Create a new run\n", + "\n", + "`web-classification` is a flow demonstrating multi-class classification with LLM. Given an url, it will classify the url into one web category with just a few shots, simple summarization and classification prompts." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Set flow path and run input data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "flow = \"../../flows/standard/web-classification\" # path to the flow directory\n", + "data = \"../../flows/standard/web-classification/data.jsonl\" # path to the data file" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Test flow locally" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Test flow\n", + "flow_inputs = {\"url\": \"https://www.youtube.com/watch?v=o5ZQyXaAv1g\", \"answer\": \"Channel\", \"evidence\": \"Url\"}\n", + "flow_result = pf.test(flow=flow, inputs=flow_inputs)\n", + "print(f\"Flow result: {flow_result}\")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Test single node in the flow\n", + "node_name = \"fetch_text_content_from_url\"\n", + "node_inputs = {\"url\": \"https://www.youtube.com/watch?v=o5ZQyXaAv1g\"}\n", + "#flow_result = pf.test(flow=flow, inputs=node_inputs, node=node_name)\n", + "#print(f\"Node result: {flow_result}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create run against multi-line data\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# create run with default variant\n", + "base_run = pf.run(flow=flow, data=data)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf.stream(base_run)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "details = pf.get_details(base_run)\n", + "details.head(10)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Evaluate your run\n", + "Then you can use an evaluation method to evaluate your flow. The evaluation methods are also flows which use Python or LLM etc., to calculate metrics like accuracy, relevance score.\n", + "\n", + "In this notebook, we use `classification-accuracy-eval` flow to evaluate. This is a flow illustrating how to evaluate the performance of a classification system. It involves comparing each prediction to the groundtruth and assigns a \"Correct\" or \"Incorrect\" grade, and aggregating the results to produce metrics such as accuracy, which reflects how good the system is at classifying the data." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Run evaluation against run" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "eval_flow = \"../../flows/evaluation/classification-accuracy-eval\"\n", + "\n", + "eval_run = pf.run(\n", + " flow=eval_flow,\n", + " data=\"../../flows/standard/web-classification/data.jsonl\", # path to the data file\n", + " run=base_run, # use run as the variant\n", + " column_mapping={\n", + " \"groundtruth\": \"${data.answer}\",\n", + " \"prediction\": \"${run.outputs.category}\",\n", + " }, # map the url field from the data to the url input of the flow\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf.stream(eval_run)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "details = pf.get_details(eval_run)\n", + "details.head(10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metrics = pf.get_metrics(eval_run)\n", + "print(json.dumps(metrics, indent=4))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf.visualize([base_run, eval_run])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Create another run with different variant node\n", + "\n", + "In this example, `web-classification`'s node `summarize_text_content` has two variants: `variant_0` and `variant_1`. The difference between them is the inputs parameters:\n", + "\n", + "variant_0:\n", + "\n", + " - inputs:\n", + " - deployment_name: text-davinci-003\n", + " - max_tokens: '128'\n", + " - temperature: '0.2'\n", + " - text: ${fetch_text_content_from_url.output}\n", + "\n", + "variant_1:\n", + "\n", + " - inputs:\n", + " - deployment_name: text-davinci-003\n", + " - max_tokens: '256'\n", + " - temperature: '0.3'\n", + " - text: ${fetch_text_content_from_url.output}\n", + "\n", + "\n", + "You can check the whole flow definistion at `../../flows/standard/web-classification/flow.dag.yaml`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# use the variant1 of the summarize_text_content node.\n", + "variant_run = pf.run(\n", + " flow=flow,\n", + " data=data,\n", + " variant=\"${summarize_text_content.variant_1}\", # here we specify node \"summarize_text_content\" to use variant 1 verison.\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf.stream(variant_run)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "details = pf.get_details(variant_run)\n", + "details.head(10)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Run evaluation run against variant run" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "eval_flow = \"../../flows/evaluation/classification-accuracy-eval\"\n", + "\n", + "eval_run_variant = pf.run(\n", + " flow=eval_flow,\n", + " data=\"../../flows/standard/web-classification/data.jsonl\", # path to the data file\n", + " run=variant_run, # use run as the variant\n", + " column_mapping={\n", + " \"groundtruth\": \"${data.answer}\",\n", + " \"prediction\": \"${run.outputs.category}\",\n", + " }, # map the url field from the data to the url input of the flow\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf.stream(eval_run_variant)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "details = pf.get_details(eval_run_variant)\n", + "details.head(10)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "metrics = pf.get_metrics(eval_run_variant)\n", + "print(json.dumps(metrics, indent=4))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf.visualize([eval_run, eval_run_variant])" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Next Steps\n", + "\n", + "Learn more on how to:\n", + "- manage connection: [connection.ipynb](../../connections/connection.ipynb)\n", + "- manage run: [run.ipynb](../advanced-run-management/run.ipynb)\n", + "- deploy the flow as a local http endpoint: [deploy.md](../flow-deploy/deploy.md)\n", + "- create the flow as a run in azure: [quickstart-azure.ipynb](./quickstart-azure.ipynb)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "pflow", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.9.17" + }, + "orig_nbformat": 4 + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/examples/tutorials/run-management/cloud-run-management.ipynb b/examples/tutorials/run-management/cloud-run-management.ipynb new file mode 100644 index 00000000000..4a235789dc3 --- /dev/null +++ b/examples/tutorials/run-management/cloud-run-management.ipynb @@ -0,0 +1,258 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Cloud Run Management\n", + "\n", + "**Requirements** - In order to benefit from this tutorial, you will need:\n", + "- A basic understanding of Machine Learning\n", + "- An Azure account with an active subscription - [Create an account for free](https://azure.microsoft.com/free/?WT.mc_id=A261C142F)\n", + "- An Azure ML workspace - [Configure workspace](../../configuration.ipynb)\n", + "- A python environment\n", + "- Installed PromptFlow SDK\n", + "\n", + "\n", + "**Learning Objectives** - By the end of this tutorial, you should be able to:\n", + "- create run with remote data\n", + "- create run which references another runs inputs\n", + "- manage runs via run.yaml\n", + "- serverless runtime\n", + "\n", + "\n", + "**Motivations** - This guide will walk you through cloud run management abilities." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 0. Install dependent packages" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install -r ../../requirements.txt" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Create run with remote data\n", + "\n", + "### 1.1 Import the required libraries" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from azure.ai.ml import MLClient\n", + "from azure.identity import DefaultAzureCredential, InteractiveBrowserCredential\n", + "from azure.ai.ml.entities import Data\n", + "from azure.core.exceptions import ResourceNotFoundError\n", + "\n", + "from promptflow.azure import PFClient\n", + "from promptflow.entities import Run" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.2 Configure credential\n", + "\n", + "We are using `DefaultAzureCredential` to get access to workspace. \n", + "`DefaultAzureCredential` should be capable of handling most Azure SDK authentication scenarios. \n", + "\n", + "Reference for more available credentials if it does not work for you: [configure credential example](../../configuration.ipynb), [azure-identity reference doc](https://docs.microsoft.com/en-us/python/api/azure-identity/azure.identity?view=azure-python)." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "try:\n", + " credential = DefaultAzureCredential()\n", + " # Check if given credential can get token successfully.\n", + " credential.get_token(\"https://management.azure.com/.default\")\n", + "except Exception as ex:\n", + " # Fall back to InteractiveBrowserCredential in case DefaultAzureCredential not work\n", + " credential = InteractiveBrowserCredential()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.3 Get a handle to the workspace\n", + "\n", + "We use config file to connect to a workspace. The Azure ML workspace should be configured with computer cluster. [Check this notebook for configure a workspace](../../configuration.ipynb)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Get a handle to workspace\n", + "ml_client = MLClient.from_config(credential=credential)\n", + "\n", + "pf = PFClient(ml_client)\n", + "\n", + "runtime = \"demo-mir\" # TODO remove this after serverless ready" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.4 Create or update remote data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data_name, data_version = \"flow_run_test_data\", \"1\"\n", + "\n", + "try:\n", + " data = ml_client.data.get(name=data_name, version=data_version)\n", + "except ResourceNotFoundError:\n", + " data = Data(\n", + " name=data_name,\n", + " version=data_version,\n", + " path=f\"../../flows/standard/web-classification/data.jsonl\",\n", + " type=\"uri_file\",\n", + " )\n", + " data = ml_client.data.create_or_update(data)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.5 Prepare remote data id" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "data_id = f\"azureml:{data.name}:{data.version}\"\n", + "print(data_id)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.6 Create a flow run with remote data" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# create run\n", + "run = Run(\n", + " # local flow file\n", + " flow=\"../../flows/standard/web-classification\",\n", + " # remote data\n", + " data=data_id,\n", + ")\n", + "\n", + "base_run = pf.runs.create_or_update(\n", + " run=run,\n", + " runtime=\"demo-mir\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.7 Stream the flow run to make sure it runs successfully" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pf.runs.stream(base_run)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.8 Create a flow run which referenced run inputs" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "run = Run(\n", + " # local flow file\n", + " flow=\"../../flows/standard/web-classification\",\n", + " # run name\n", + " run=run,\n", + " column_mapping={\n", + " # reference the run's inputs\n", + " \"url\": \"${run.inputs.url}\",\n", + " \"answer\": \"${run.inputs.answer}\",\n", + " \"evidence\": \"${run.inputs.evidence}\",\n", + " }, \n", + ")\n", + "\n", + "base_run = pf.runs.create_or_update(\n", + " run=run,\n", + " runtime=\"demo-mir\",\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "github_v2", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/scripts/ghactions_driver/driver.py b/scripts/ghactions_driver/driver.py new file mode 100644 index 00000000000..9de7bcd7416 --- /dev/null +++ b/scripts/ghactions_driver/driver.py @@ -0,0 +1,80 @@ +# Driver to create github workflows +from typing import List, Dict + + +class Step: + """ + Base class for each small steps, check out other files for templates. + """ + + name: str + + def __init__(self, name: str) -> None: + self.name = name + + def get_step(self) -> str: + # virtual method for overide + return " " * 6 + f"- name: {self.name}" + + +class Job: + """ + Careful: Job and Job indicates no sequential guarantee by default + """ + + name: str + steps: List[Step] + + def __init__(self, name: str, steps: List[Step]) -> None: + self.name = name + self.steps = steps + + def get_job(self) -> str: + output = " " * 4 + "runs-on: ubuntu-latest\n" + output += " " * 4 + "steps:\n" + output += "\n".join([step.get_step() for step in self.steps]) + return output + + +class Jobs: + + _jobs: Dict[str, Job] + + def add_job(self, job: Job) -> "Jobs": + self._jobs = {} + self._jobs[job.name] = job + return self + + def get_jobs(self) -> str: + output = "jobs:\n" + for job in self._jobs.keys(): + output += " " * 2 + f"{job}:\n" + output += self._jobs[job].get_job() + return output + + +class Workflow: + name: str + jobs: Jobs + _template_string: str + + def __init__(self, name: str, jobs: Jobs) -> None: + self.name = name + self.jobs = jobs + self._template_string = f"""# This code is autogenerated. +# Code is generated by running custom script: python3 readme.py +# Any manual changes to this file may cause incorrect behavior. +# Any manual changes will be overwritten if the code is regenerated. + +name: {name} +on: + pull_request: + branches: [ main,preview/code-first ] + workflow_dispatch: + +""" + + def get_workflow(self): + output = self._template_string + output += self.jobs.get_jobs() + return output diff --git a/scripts/ghactions_driver/steps.py b/scripts/ghactions_driver/steps.py new file mode 100644 index 00000000000..18aec7d5431 --- /dev/null +++ b/scripts/ghactions_driver/steps.py @@ -0,0 +1,130 @@ +from .driver import Step + + +class CheckoutStep(Step): + def __init__(self) -> None: + super().__init__("Checkout repository") + + def get_step(self) -> str: + return Step.get_step(self) + "\n" + " " * 8 + "uses: actions/checkout@v3" + + +class GenerateConfigStep(Step): + def __init__(self) -> None: + super().__init__("Generate config.json") + + def get_step(self) -> str: + return ( + Step.get_step(self) + + "\n" + + " " * 8 + + "run: echo ${{ secrets.TEST_WORKSPACE_CONFIG_JSON }} > ${{ github.workspace }}/examples/config.json" + ) + + +class AzureLoginStep(Step): + def __init__(self) -> None: + super().__init__("Azure Login") + + def get_step(self) -> str: + return ( + Step.get_step(self) + + "\n" + + " " * 8 + + "uses: azure/login@v1\n" + + " " * 8 + + "with:\n" + + " " * 10 + + "creds: ${{ secrets.AZURE_CREDENTIALS }}" + ) + + +class SetupPythonStep(Step): + def __init__(self) -> None: + super().__init__("Setup Python 3.9 environment") + + def get_step(self) -> str: + return ( + Step.get_step(self) + + "\n" + + " " * 8 + + "uses: actions/setup-python@v4\n" + + " " * 8 + + "with:\n" + + " " * 10 + + 'python-version: "3.9"' + ) + + +class InstallDependenciesStep(Step): + def __init__(self) -> None: + super().__init__("Prepare requirements") + + def get_step(self) -> str: + return ( + Step.get_step(self) + + "\n" + + " " * 8 + + "run: |\n" + + " " * 10 + + "python -m pip install --upgrade pip\n" + + " " * 10 + + "pip install -r ${{ github.workspace }}/examples/requirements.txt\n" + + " " * 10 + + "pip install -r ${{ github.workspace }}/examples/dev_requirements.txt" + ) + + +class CreateAoaiConnectionStep(Step): + def __init__(self) -> None: + super().__init__("Create Aoai Connection") + + def get_step(self) -> str: + return ( + Step.get_step(self) + + "\n" + + " " * 8 + + 'run: pf connection create -f ${{ github.workspace }}/examples/connections/azure_openai.yml' + + '--set api_key="${{ secrets.AOAI_API_KEY }}" api_base="${{ secrets.AOAI_API_ENDPOINT }}"' + ) + + +class RunTestStep(Step): + def __init__(self, filename: str, filepath: str) -> None: + super().__init__("Test Notebook") + self.filename = filename + self.filepath = filepath + + def get_step(self) -> str: + return ( + Step.get_step(self) + + "\n" + + " " * 8 + + f"working-directory: {self.filepath}\n" + + " " * 8 + + "run: |\n" + + " " * 10 + + f"papermill -k python {self.filename}.ipynb {self.filename}.output.ipynb" + ) + + +class UploadArtifactStep(Step): + def __init__(self, filepath: str) -> None: + super().__init__("Upload artifact") + self.filepath = filepath + + def get_step(self) -> str: + return ( + Step.get_step(self) + + "\n" + + " " * 8 + + "if: ${{ always() }}\n" + + " " * 8 + + "uses: actions/upload-artifact@v3\n" + + " " * 8 + + "with:\n" + + " " * 10 + + "name: artifact\n" + + " " * 10 + + f"path: {self.filepath}" + ) diff --git a/scripts/workflow_generator.py b/scripts/workflow_generator.py new file mode 100644 index 00000000000..ceac3cd4d4e --- /dev/null +++ b/scripts/workflow_generator.py @@ -0,0 +1,112 @@ +import os +import glob +from ghactions_driver.driver import Workflow, Jobs, Job +from ghactions_driver.steps import ( + CheckoutStep, + GenerateConfigStep, + AzureLoginStep, + SetupPythonStep, + InstallDependenciesStep, + CreateAoaiConnectionStep, + RunTestStep, + UploadArtifactStep, +) +import argparse +from pathlib import Path +import ntpath +import re + + +def format_ipynb(notebooks): + # run code formatter on .ipynb files + for notebook in notebooks: + os.system(f"black-nb --clear-output {notebook}") + + +def _get_paths(paths_list): + """ + Convert the path list to unix format. + :param paths_list: The input path list. + :returns: The same list with unix-like paths. + """ + paths_list.sort() + if ntpath.sep == os.path.sep: + return [pth.replace(ntpath.sep, "/") for pth in paths_list] + return paths_list + + +def write_notebook_workflow(notebook, name): + temp_name_list = re.split(r"/|\.", notebook) + temp_name_list.remove("examples") + temp_name_list.remove("ipynb") + temp_name_list = [x.replace("-", "") for x in temp_name_list] + workflow_name = "_".join(temp_name_list) + + place_to_write = ( + Path(__file__).parent.parent / ".github" / "workflows" / f"{workflow_name}.yml" + ) + + gh_working_dir = "/".join(notebook.split("/")[:-1]) + + # To customize workflow, add new steps in steps.py + # make another function for special cases. + workflow = Workflow( + workflow_name, + Jobs().add_job( + Job( + name, + [ + CheckoutStep(), + GenerateConfigStep(), + AzureLoginStep(), + SetupPythonStep(), + InstallDependenciesStep(), + CreateAoaiConnectionStep(), + RunTestStep(name, gh_working_dir), + UploadArtifactStep(gh_working_dir), + ], + ) + ), + ) + with open(place_to_write.resolve(), "w") as f: + f.write(workflow.get_workflow()) + print(f"Write workflow: {place_to_write.resolve()}") + + +def write_workflows(notebooks): + # process notebooks + for notebook in notebooks: + # get notebook name + nb_path = Path(notebook) + name, _ = os.path.splitext(nb_path.parts[-1]) + + # write workflow file + write_notebook_workflow(notebook, name) + + +# define functions +def main(args): + # get list of workflows + + workflows = _get_paths( + [j for i in [glob.glob(p, recursive=True) for p in args.input_glob] for j in i] + ) + + # format code + format_ipynb(workflows) + + # write workflows + write_workflows(workflows) + + +# run functions +if __name__ == "__main__": + # setup argparse + parser = argparse.ArgumentParser() + parser.add_argument( + "-g", "--input-glob", nargs="+", help="Input glob example 'examples/**/*.ipynb'" + ) + args = parser.parse_args() + + # call main + main(args) diff --git a/src/promptflow-tools/README.md b/src/promptflow-tools/README.md index 4beb8b7c286..13d90d43f56 100644 --- a/src/promptflow-tools/README.md +++ b/src/promptflow-tools/README.md @@ -1,7 +1,7 @@ ### Test your tool locally * Packages to install in order to run tool tests locally: ```cmd - pip install promptflow-sdk[builtins] --extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ + pip install promptflow --extra-index-url https://azuremlsdktestpypi.azureedge.net/promptflow/ ``` * Generate connections config.