diff --git a/.devcontainer/README.md b/.devcontainer/README.md index 8ae045f27d19..c56d9d56cc00 100644 --- a/.devcontainer/README.md +++ b/.devcontainer/README.md @@ -49,7 +49,7 @@ Feel free to modify these Dockerfiles for your specific project needs. Here are - **Setting Environment Variables**: Add environment variables using the `ENV` command for any application-specific configurations. We have prestaged the line needed to inject your OpenAI_key into the docker environment as a environmental variable. Others can be staged in the same way. Just uncomment the line. `# ENV OPENAI_API_KEY="{OpenAI-API-Key}"` to `ENV OPENAI_API_KEY="{OpenAI-API-Key}"` - **Need a less "Advanced" Autogen build**: If the `./full/Dockerfile` is to much but you need more than advanced then update this line in the Dockerfile file. -`RUN pip install pyautogen[teachable,lmm,retrievechat,mathchat,blendsearch] autogenra` to install just what you need. `RUN pip install pyautogen[retrievechat,blendsearch] autogenra` +`RUN pip install autogen-agentchat[teachable,lmm,retrievechat,mathchat,blendsearch]~=0.2 autogenra` to install just what you need. `RUN pip install autogen-agentchat[retrievechat,blendsearch]~=0.2 autogenra` - **Can't Dev without your favorite CLI tool**: if you need particular OS tools to be installed in your Docker container you can add those packages here right after the sudo for the `./base/Dockerfile` and `./full/Dockerfile` files. In the example below we are installing net-tools and vim to the environment. ```code diff --git a/.devcontainer/full/Dockerfile b/.devcontainer/full/Dockerfile index 0787ad240272..525dd3978d4a 100644 --- a/.devcontainer/full/Dockerfile +++ b/.devcontainer/full/Dockerfile @@ -22,7 +22,7 @@ WORKDIR /home/autogen # Install Python packages RUN pip install --upgrade pip -RUN pip install pyautogen[teachable,lmm,retrievechat,mathchat,blendsearch] autogenra +RUN pip install autogen-agentchat[teachable,lmm,retrievechat,mathchat,blendsearch]~=0.2 autogenra RUN pip install numpy pandas matplotlib seaborn scikit-learn requests urllib3 nltk pillow pytest beautifulsoup4 # Expose port diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index b3b0d0daeed6..000000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,57 +0,0 @@ -### Description - - -### Environment -- AutoGen version: -- Python version: -- Operating System: - -### Steps to Reproduce (for bugs) - - -1. Step 1 -2. Step 2 -3. ... - -### Expected Behavior - - -### Actual Behavior - - -### Screenshots / Logs (if applicable) - - -### Additional Information - - -### Possible Solution (if you have one) - - -### Is this a Bug or Feature Request? - - -### Priority - - -### Difficulty - - -### Any related issues? - - -### Any relevant discussions? - - -### Checklist - -- [ ] I have searched for similar issues and didn't find any duplicates. -- [ ] I have provided a clear and concise description of the issue. -- [ ] I have included the necessary environment details. -- [ ] I have outlined the steps to reproduce the issue. -- [ ] I have included any relevant logs or screenshots. -- [ ] I have indicated whether this is a bug or a feature request. -- [ ] I have set the priority and difficulty levels. - -### Additional Comments - diff --git a/.github/ISSUE_TEMPLATE/bug_report.yml b/.github/ISSUE_TEMPLATE/bug_report.yml index 434226b3e884..090fa6cc5939 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.yml +++ b/.github/ISSUE_TEMPLATE/bug_report.yml @@ -1,53 +1,55 @@ name: Bug Report -description: File a bug report -title: "[Bug]: " +description: Report a bug labels: ["bug"] body: - type: textarea - id: description attributes: - label: Describe the bug - description: A clear and concise description of what the bug is. - placeholder: What went wrong? + label: What happened? + description: Please provide as much information as possible, this helps us address the issue. + validations: + required: true - type: textarea - id: reproduce attributes: - label: Steps to reproduce - description: | - Steps to reproduce the behavior: - - 1. Step 1 - 2. Step 2 - 3. ... - 4. See error - placeholder: How can we replicate the issue? + label: What did you expect to happen? + validations: + required: true - type: textarea - id: modelused attributes: - label: Model Used - description: A description of the model that was used when the error was encountered + label: How can we reproduce it (as minimally and precisely as possible)? + description: Please provide steps to reproduce. Provide code that can be run if possible. + validations: + required: true + - type: input + attributes: + label: AutoGen version + description: What version or commit of the library was used + validations: + required: true + - type: dropdown + attributes: + label: Which package was this bug in + options: + - Core + - AgentChat + - Extensions + - AutoGen Studio + - Magentic One + - AutoGen Bench + - Other + validations: + required: true + - type: input + attributes: + label: Model used + description: If a model was used, please describe it here, indicating whether it is a local model or a cloud-hosted model placeholder: gpt-4, mistral-7B etc - - type: textarea - id: expected_behavior + - type: input attributes: - label: Expected Behavior - description: A clear and concise description of what you expected to happen. - placeholder: What should have happened? - - type: textarea - id: screenshots + label: Python version + - type: input attributes: - label: Screenshots and logs - description: If applicable, add screenshots and logs to help explain your problem. - placeholder: Add screenshots here + label: Operating system - type: textarea - id: additional_information - attributes: - label: Additional Information - description: | - - AutoGen Version: - - Operating System: - - Python Version: - - Related Issues: - - Any other relevant information. - placeholder: Any additional details + attributes: + label: Any additional info you think would be helpful for fixing this bug diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml index 0086358db1eb..76afcbcc5f87 100644 --- a/.github/ISSUE_TEMPLATE/config.yml +++ b/.github/ISSUE_TEMPLATE/config.yml @@ -1 +1,5 @@ blank_issues_enabled: true +contact_links: + - name: Questions or general help 💬 + url: https://github.com/microsoft/autogen/discussions + about: Please ask and answer questions here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.yml b/.github/ISSUE_TEMPLATE/feature_request.yml index e8a63df7a6e2..57f360761a76 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.yml +++ b/.github/ISSUE_TEMPLATE/feature_request.yml @@ -1,26 +1,18 @@ name: Feature Request -description: File a feature request +description: Request a new feature or enhancement labels: ["enhancement"] -title: "[Feature Request]: " body: - type: textarea - id: problem_description attributes: - label: Is your feature request related to a problem? Please describe. - description: A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] - placeholder: What problem are you trying to solve? + label: What feature would you like to be added? + description: Please describe the desired feature. Be descriptive, provide examples and if possible, provide a proposed solution. + validations: + required: true - type: textarea - id: solution_description attributes: - label: Describe the solution you'd like - description: A clear and concise description of what you want to happen. - placeholder: How do you envision the solution? - - - type: textarea - id: additional_context - attributes: - label: Additional context - description: Add any other context or screenshots about the feature request here. - placeholder: Any additional information + label: Why is this needed? + description: Why is it important that this feature is implemented? What problem or need does it solve? + validations: + required: true diff --git a/.github/ISSUE_TEMPLATE/general_issue.yml b/.github/ISSUE_TEMPLATE/general_issue.yml deleted file mode 100644 index b585f4642f44..000000000000 --- a/.github/ISSUE_TEMPLATE/general_issue.yml +++ /dev/null @@ -1,41 +0,0 @@ -name: General Issue -description: File a general issue -title: "[Issue]: " -labels: [] - -body: - - type: textarea - id: description - attributes: - label: Describe the issue - description: A clear and concise description of what the issue is. - placeholder: What went wrong? - - type: textarea - id: reproduce - attributes: - label: Steps to reproduce - description: | - Steps to reproduce the behavior: - - 1. Step 1 - 2. Step 2 - 3. ... - 4. See error - placeholder: How can we replicate the issue? - - type: textarea - id: screenshots - attributes: - label: Screenshots and logs - description: If applicable, add screenshots and logs to help explain your problem. - placeholder: Add screenshots here - - type: textarea - id: additional_information - attributes: - label: Additional Information - description: | - - AutoGen Version: - - Operating System: - - Python Version: - - Related Issues: - - Any other relevant information. - placeholder: Any additional details diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index a92044f15b78..0107e7314fcc 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -5,15 +5,13 @@ name: Build on: push: - branches: ["main"] + branches: ["0.2"] pull_request: - branches: ["main"] - merge_group: - types: [checks_requested] + branches: ["0.2"] concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + cancel-in-progress: ${{ github.ref != 'refs/heads/0.2' }} permissions: {} jobs: paths-filter: diff --git a/.github/workflows/contrib-openai.yml b/.github/workflows/contrib-openai.yml index 7e8fb0033177..d084e44bd607 100644 --- a/.github/workflows/contrib-openai.yml +++ b/.github/workflows/contrib-openai.yml @@ -5,7 +5,7 @@ name: OpenAI4ContribTests on: pull_request: - branches: ["main"] + branches: ["0.2"] paths: - "autogen/**" - "test/agentchat/contrib/**" diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml index d58098c98e76..7d779bc5fae8 100644 --- a/.github/workflows/contrib-tests.yml +++ b/.github/workflows/contrib-tests.yml @@ -5,7 +5,7 @@ name: ContribTests on: pull_request: - branches: ["main"] + branches: ["0.2"] paths: - "autogen/**" - "test/agentchat/contrib/**" @@ -16,7 +16,7 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + cancel-in-progress: ${{ github.ref != 'refs/heads/0.2' }} permissions: {} # actions: read @@ -474,6 +474,46 @@ jobs: file: ./coverage.xml flags: unittests + CerebrasTest: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-2019] + python-version: ["3.9", "3.10", "3.11", "3.12"] + exclude: + - os: macos-latest + python-version: "3.9" + steps: + - uses: actions/checkout@v4 + with: + lfs: true + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install packages and dependencies for all tests + run: | + python -m pip install --upgrade pip wheel + pip install pytest-cov>=5 + - name: Install packages and dependencies for Cerebras + run: | + pip install -e .[cerebras_cloud_sdk,test] + - name: Set AUTOGEN_USE_DOCKER based on OS + shell: bash + run: | + if [[ ${{ matrix.os }} != ubuntu-latest ]]; then + echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV + fi + - name: Coverage + run: | + pytest test/oai/test_cerebras.py --skip-openai + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests + MistralTest: runs-on: ${{ matrix.os }} strategy: @@ -669,3 +709,35 @@ jobs: with: file: ./coverage.xml flags: unittests + + OllamaTest: + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, macos-latest, windows-2019] + python-version: ["3.9", "3.10", "3.11", "3.12"] + exclude: + - os: macos-latest + python-version: "3.9" + steps: + - uses: actions/checkout@v4 + with: + lfs: true + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install packages and dependencies for all tests + run: | + python -m pip install --upgrade pip wheel + pip install pytest-cov>=5 + - name: Install packages and dependencies for Ollama + run: | + pip install -e .[ollama,test] + pytest test/oai/test_ollama.py --skip-openai + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests diff --git a/.github/workflows/deploy-website.yml b/.github/workflows/deploy-website.yml index 2f2ba4d473f3..4770c8d0d0dc 100644 --- a/.github/workflows/deploy-website.yml +++ b/.github/workflows/deploy-website.yml @@ -2,26 +2,15 @@ name: docs on: pull_request: - branches: [main] - path: - - "autogen/*" - - "website/*" - - ".github/workflows/deploy-website.yml" + branches: ["0.2"] push: - branches: [main] - path: - - "autogen/*" - - "website/*" - - ".github/workflows/deploy-website.yml" + branches: ["0.2"] workflow_dispatch: - merge_group: - types: [checks_requested] permissions: id-token: write pages: write jobs: checks: - if: github.event_name != 'push' runs-on: ubuntu-latest defaults: run: @@ -67,57 +56,3 @@ jobs: npm i --legacy-peer-deps npm run build fi - gh-release: - if: github.event_name != 'pull_request' - runs-on: ubuntu-latest - defaults: - run: - working-directory: website - steps: - - uses: actions/checkout@v4 - with: - lfs: true - - uses: actions/setup-node@v4 - with: - node-version: 18.x - - name: setup python - uses: actions/setup-python@v5 - with: - python-version: "3.8" - - name: pydoc-markdown install - run: | - python -m pip install --upgrade pip - pip install pydoc-markdown pyyaml termcolor - # Pin databind packages as version 4.5.0 is not compatible with pydoc-markdown. - pip install databind.core==4.4.2 databind.json==4.4.2 - - name: pydoc-markdown run - run: | - pydoc-markdown - - name: quarto install - working-directory: ${{ runner.temp }} - run: | - wget -q https://github.com/quarto-dev/quarto-cli/releases/download/v1.5.23/quarto-1.5.23-linux-amd64.tar.gz - tar -xzf quarto-1.5.23-linux-amd64.tar.gz - echo "$(pwd)/quarto-1.5.23/bin/" >> $GITHUB_PATH - - name: Process notebooks - run: | - python process_notebooks.py render - - name: Build website - run: | - if [ -e yarn.lock ]; then - yarn install --frozen-lockfile --ignore-engines - yarn build - elif [ -e package-lock.json ]; then - npm ci - npm run build - else - npm i --legacy-peer-deps - npm run build - fi - - name: Upload artifact - uses: actions/upload-pages-artifact@v3 - with: - path: "website/build" - - name: Deploy to GitHub Pages - id: deployment - uses: actions/deploy-pages@v4 diff --git a/.github/workflows/dotnet-build.yml b/.github/workflows/dotnet-build.yml index 6aac54d3818c..b333d9065d64 100644 --- a/.github/workflows/dotnet-build.yml +++ b/.github/workflows/dotnet-build.yml @@ -6,15 +6,13 @@ name: dotnet-ci on: workflow_dispatch: pull_request: - branches: [ "main" ] + branches: [ "0.2" ] push: - branches: [ "main" ] - merge_group: - types: [checks_requested] + branches: [ "0.2" ] concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' || github.ref != 'refs/heads/dotnet' }} + cancel-in-progress: ${{ github.ref != 'refs/heads/0.2' || github.ref != 'refs/heads/dotnet' }} permissions: contents: read @@ -122,7 +120,7 @@ jobs: defaults: run: working-directory: dotnet - if: success() && (github.ref == 'refs/heads/main') + if: success() && (github.ref == 'refs/heads/0.2') needs: aot-test steps: - uses: actions/checkout@v4 @@ -228,4 +226,4 @@ jobs: env: MYGET_TOKEN: ${{ secrets.MYGET_TOKEN }} continue-on-error: true - + diff --git a/.github/workflows/issue-needs-triage.yml b/.github/workflows/issue-needs-triage.yml new file mode 100644 index 000000000000..59cb3479c808 --- /dev/null +++ b/.github/workflows/issue-needs-triage.yml @@ -0,0 +1,18 @@ +name: Label issues with needs-triage +on: + issues: + types: + - reopened + - opened +jobs: + label_issues: + runs-on: ubuntu-latest + permissions: + issues: write + steps: + - run: gh issue edit "$NUMBER" --add-label "$LABELS" + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_REPO: ${{ github.repository }} + NUMBER: ${{ github.event.issue.number }} + LABELS: needs-triage diff --git a/.github/workflows/lfs-check.yml b/.github/workflows/lfs-check.yml index 4baae925de3c..dc5e1678be2f 100644 --- a/.github/workflows/lfs-check.yml +++ b/.github/workflows/lfs-check.yml @@ -1,6 +1,7 @@ name: "Git LFS Check" - -on: pull_request +on: + pull_request: + branches: ["0.2"] permissions: {} jobs: lfs-check: diff --git a/.github/workflows/openai.yml b/.github/workflows/openai.yml index a9ab8e9e0c5f..e29f9d0f1ab2 100644 --- a/.github/workflows/openai.yml +++ b/.github/workflows/openai.yml @@ -5,7 +5,7 @@ name: OpenAI on: pull_request: - branches: ["main"] + branches: ["0.2"] paths: - "autogen/**" - "test/**" diff --git a/.github/workflows/pre-commit.yml b/.github/workflows/pre-commit.yml index 8404de61154d..7ec5a4eb1048 100644 --- a/.github/workflows/pre-commit.yml +++ b/.github/workflows/pre-commit.yml @@ -3,8 +3,7 @@ name: Code formatting # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows on: # Trigger the workflow on pull request or merge pull_request: - merge_group: - types: [checks_requested] + branches: ["0.2"] defaults: run: diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml index f2967c13f5f0..e34678ab0039 100644 --- a/.github/workflows/python-package.yml +++ b/.github/workflows/python-package.yml @@ -5,13 +5,10 @@ name: python-package on: - release: - types: [published] + push: + tags: + - "0.2.*" permissions: {} - # actions: read - # checks: read - # contents: read - # deployments: read jobs: deploy: strategy: @@ -19,38 +16,18 @@ jobs: os: ['ubuntu-latest'] python-version: [3.10] runs-on: ${{ matrix.os }} - environment: package + environment: + name: package + url: https://pypi.org/p/autogen-agentchat + permissions: + id-token: write steps: - name: Checkout uses: actions/checkout@v4 - # - name: Cache conda - # uses: actions/cache@v4 - # with: - # path: ~/conda_pkgs_dir - # key: conda-${{ matrix.os }}-python-${{ matrix.python-version }}-${{ hashFiles('environment.yml') }} - # - name: Setup Miniconda - # uses: conda-incubator/setup-miniconda@v2 - # with: - # auto-update-conda: true - # auto-activate-base: false - # activate-environment: hcrystalball - # python-version: ${{ matrix.python-version }} - # use-only-tar-bz2: true - - name: Install from source - # This is required for the pre-commit tests - shell: pwsh - run: pip install . - # - name: Conda list - # shell: pwsh - # run: conda list - name: Build shell: pwsh run: | pip install twine python setup.py sdist bdist_wheel - - name: Publish to PyPI - env: - TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }} - TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }} - shell: pwsh - run: twine upload dist/* + - name: Publish package to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/samples-tools-tests.yml b/.github/workflows/samples-tools-tests.yml index e774e5cb0b1f..9452f0e377e3 100644 --- a/.github/workflows/samples-tools-tests.yml +++ b/.github/workflows/samples-tools-tests.yml @@ -5,7 +5,7 @@ name: SamplesToolsTests on: pull_request: - branches: ["main"] + branches: ["0.2"] paths: - "autogen/**" - "samples/tools/**" @@ -14,7 +14,7 @@ on: concurrency: group: ${{ github.workflow }}-${{ github.ref }}-${{ github.head_ref }} - cancel-in-progress: ${{ github.ref != 'refs/heads/main' }} + cancel-in-progress: ${{ github.ref != 'refs/heads/0.2' }} permissions: {} jobs: SamplesToolsFineTuningTests: diff --git a/.github/workflows/type-check.yml b/.github/workflows/type-check.yml index c66fb6ad7b10..3afd32ad886f 100644 --- a/.github/workflows/type-check.yml +++ b/.github/workflows/type-check.yml @@ -2,8 +2,8 @@ name: Type check # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows on: # Trigger the workflow on pull request or merge pull_request: - merge_group: - types: [checks_requested] + branches: ["0.2"] + defaults: run: shell: bash diff --git a/LICENSE-CODE-KUBERNETES b/LICENSE-CODE-KUBERNETES new file mode 100644 index 000000000000..45cadb395ec4 --- /dev/null +++ b/LICENSE-CODE-KUBERNETES @@ -0,0 +1,201 @@ + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2014 The Kubernetes Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/OAI_CONFIG_LIST_sample b/OAI_CONFIG_LIST_sample index c1711acd7c65..7cb370fd5154 100644 --- a/OAI_CONFIG_LIST_sample +++ b/OAI_CONFIG_LIST_sample @@ -1,5 +1,4 @@ // Please modify the content, remove these four lines of comment and rename this file to OAI_CONFIG_LIST to run the sample code. -// If using pyautogen v0.1.x with Azure OpenAI, please replace "base_url" with "api_base" (line 14 and line 21 below). Use "pip list" to check version of pyautogen installed. // // NOTE: This configuration lists GPT-4 as the default model, as this represents our current recommendation, and is known to work well with AutoGen. If you use a model other than GPT-4, you may need to revise various system prompts (especially if using weaker models like GPT-3.5-turbo). Moreover, if you use models other than those hosted by OpenAI or Azure, you may incur additional risks related to alignment and safety. Proceed with caution if updating this default. [ diff --git a/README.md b/README.md index 8595bb60506c..86118d4c46a6 100644 --- a/README.md +++ b/README.md @@ -3,14 +3,11 @@
-AutoGen Logo +AutoGen Logo -![Python Version](https://img.shields.io/badge/3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue) [![PyPI version](https://img.shields.io/badge/PyPI-v0.2.34-blue.svg)](https://pypi.org/project/pyautogen/) +![Python Version](https://img.shields.io/badge/3.8%20%7C%203.9%20%7C%203.10%20%7C%203.11%20%7C%203.12-blue) [![PyPI - Version](https://img.shields.io/pypi/v/autogen-agentchat)](https://pypi.org/project/autogen-agentchat/) [![NuGet version](https://badge.fury.io/nu/AutoGen.Core.svg)](https://badge.fury.io/nu/AutoGen.Core) -[![Downloads](https://static.pepy.tech/badge/pyautogen/week)](https://pepy.tech/project/pyautogen) -[![Discord](https://img.shields.io/discord/1153072414184452236?logo=discord&style=flat)](https://aka.ms/autogen-dc) - [![Twitter](https://img.shields.io/twitter/url/https/twitter.com/cloudposse.svg?style=social&label=Follow%20%40pyautogen)](https://twitter.com/pyautogen)
@@ -20,12 +17,16 @@ AutoGen is an open-source programming framework for building AI agents and facilitating cooperation among multiple agents to solve tasks. AutoGen aims to streamline the development and research of agentic AI, much like PyTorch does for Deep Learning. It offers features such as agents capable of interacting with each other, facilitates the use of various large language models (LLMs) and tool use support, autonomous and human-in-the-loop workflows, and multi-agent conversation patterns. > [!IMPORTANT] -> *Note for contributors and users*: [microsoft/autogen](https://aka.ms/autogen-gh) is the official repository of AutoGen project and it is under active development and maintenance under MIT license. We welcome contributions from developers and organizations worldwide. Our goal is to foster a collaborative and inclusive community where diverse perspectives and expertise can drive innovation and enhance the project's capabilities. We acknowledge the invaluable contributions from our existing contributors, as listed in [contributors.md](./CONTRIBUTORS.md). Whether you are an individual contributor or represent an organization, we invite you to join us in shaping the future of this project. For further information please also see [Microsoft open-source contributing guidelines](https://github.com/microsoft/autogen?tab=readme-ov-file#contributing). +> In order to better align with a new multi-packaging structure we have coming very soon, AutoGen is now available on PyPi as [`autogen-agentchat`](https://pypi.org/project/autogen-agentchat/) as of version `0.2.36`. This is the official package for the AutoGen project. + + +> [!NOTE] +> *Note for contributors and users*: [microsoft/autogen](https://aka.ms/autogen-gh) is the original repository of AutoGen project and it is under active development and maintenance under MIT license. We welcome contributions from developers and organizations worldwide. Our goal is to foster a collaborative and inclusive community where diverse perspectives and expertise can drive innovation and enhance the project's capabilities. We acknowledge the invaluable contributions from our existing contributors, as listed in [contributors.md](./CONTRIBUTORS.md). Whether you are an individual contributor or represent an organization, we invite you to join us in shaping the future of this project. For further information please also see [Microsoft open-source contributing guidelines](https://github.com/microsoft/autogen?tab=readme-ov-file#contributing). > > -_Maintainers (Sept 6th, 2024)_ -![AutoGen Overview](https://github.com/microsoft/autogen/blob/main/website/static/img/autogen_agentchat.png) +![AutoGen Overview](https://github.com/microsoft/autogen/blob/0.2/website/static/img/autogen_agentchat.png) - AutoGen enables building next-gen LLM applications based on [multi-agent conversations](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat) with minimal effort. It simplifies the orchestration, automation, and optimization of a complex LLM workflow. It maximizes the performance of LLM models and overcomes their weaknesses. - It supports [diverse conversation patterns](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat#supporting-diverse-conversation-patterns) for complex workflows. With customizable and conversable agents, developers can use AutoGen to build a wide range of conversation patterns concerning conversation autonomy, @@ -135,14 +136,14 @@ Find detailed instructions for users [here](https://microsoft.github.io/autogen/ AutoGen requires **Python version >= 3.8, < 3.13**. It can be installed from pip: ```bash -pip install pyautogen +pip install autogen-agentchat~=0.2 ``` Minimal dependencies are installed without extra options. You can install extra options based on the feature you need. Find more options in [Installation](https://microsoft.github.io/autogen/docs/Installation#option-2-install-autogen-locally-using-virtual-environment). @@ -170,7 +171,7 @@ Features of this use case include: - **Customization**: AutoGen agents can be customized to meet the specific needs of an application. This includes the ability to choose the LLMs to use, the types of human input to allow, and the tools to employ. - **Human participation**: AutoGen seamlessly allows human participation. This means that humans can provide input and feedback to the agents as needed. -For [example](https://github.com/microsoft/autogen/blob/main/test/twoagent.py), +For [example](https://github.com/microsoft/autogen/blob/0.2/test/twoagent.py), ```python from autogen import AssistantAgent, UserProxyAgent, config_list_from_json @@ -193,9 +194,9 @@ python test/twoagent.py After the repo is cloned. The figure below shows an example conversation flow with AutoGen. -![Agent Chat Example](https://github.com/microsoft/autogen/blob/main/website/static/img/chat_example.png) +![Agent Chat Example](https://github.com/microsoft/autogen/blob/0.2/website/static/img/chat_example.png) -Alternatively, the [sample code](https://github.com/microsoft/autogen/blob/main/samples/simple_chat.py) here allows a user to chat with an AutoGen agent in ChatGPT style. +Alternatively, the [sample code](https://github.com/microsoft/autogen/blob/0.2/samples/simple_chat.py) here allows a user to chat with an AutoGen agent in ChatGPT style. Please find more [code examples](https://microsoft.github.io/autogen/docs/Examples#automated-multi-agent-chat) for this feature.

@@ -239,9 +240,7 @@ You can find detailed documentation about AutoGen [here](https://microsoft.githu In addition, you can find: -- [Research](https://microsoft.github.io/autogen/docs/Research), [blogposts](https://microsoft.github.io/autogen/blog) around AutoGen, and [Transparency FAQs](https://github.com/microsoft/autogen/blob/main/TRANSPARENCY_FAQS.md) - -- [Discord](https://aka.ms/autogen-dc) +- [Research](https://microsoft.github.io/autogen/docs/Research), [blogposts](https://microsoft.github.io/autogen/blog) around AutoGen, and [Transparency FAQs](https://github.com/microsoft/autogen/blob/0.2/TRANSPARENCY_FAQS.md) - [Contributing guide](https://microsoft.github.io/autogen/docs/Contribute) diff --git a/autogen/agentchat/contrib/agent_builder.py b/autogen/agentchat/contrib/agent_builder.py index 430017d13fc9..7eaec3eef747 100644 --- a/autogen/agentchat/contrib/agent_builder.py +++ b/autogen/agentchat/contrib/agent_builder.py @@ -172,6 +172,26 @@ class AgentBuilder: ``` """ + AGENT_FUNCTION_MAP_PROMPT = """Consider the following function. + Function Name: {function_name} + Function Description: {function_description} + + The agent details are given in the format: {format_agent_details} + + Which one of the following agents should be able to execute this function, preferably an agent with programming background? + {agent_details} + + Hint: + # Only respond with the name of the agent that is most suited to execute the function and nothing else. + """ + + UPDATED_AGENT_SYSTEM_MESSAGE = """ + {agent_system_message} + + You have access to execute the function: {function_name}. + With following description: {function_description} + """ + def __init__( self, config_file_or_env: Optional[str] = "OAI_CONFIG_LIST", @@ -358,6 +378,7 @@ def build( self, building_task: str, default_llm_config: Dict, + list_of_functions: Optional[List[Dict]] = None, coding: Optional[bool] = None, code_execution_config: Optional[Dict] = None, use_oai_assistant: Optional[bool] = False, @@ -373,6 +394,7 @@ def build( coding: use to identify if the user proxy (a code interpreter) should be added. code_execution_config: specific configs for user proxy (e.g., last_n_messages, work_dir, ...). default_llm_config: specific configs for LLM (e.g., config_list, seed, temperature, ...). + list_of_functions: list of functions to be associated with Agents use_oai_assistant: use OpenAI assistant api instead of self-constructed agent. user_proxy: user proxy's class that can be used to replace the default user proxy. @@ -480,8 +502,9 @@ def build( "code_execution_config": code_execution_config, } ) + _config_check(self.cached_configs) - return self._build_agents(use_oai_assistant, user_proxy=user_proxy, **kwargs) + return self._build_agents(use_oai_assistant, list_of_functions, user_proxy=user_proxy, **kwargs) def build_from_library( self, @@ -653,13 +676,18 @@ def build_from_library( return self._build_agents(use_oai_assistant, user_proxy=user_proxy, **kwargs) def _build_agents( - self, use_oai_assistant: Optional[bool] = False, user_proxy: Optional[autogen.ConversableAgent] = None, **kwargs + self, + use_oai_assistant: Optional[bool] = False, + list_of_functions: Optional[List[Dict]] = None, + user_proxy: Optional[autogen.ConversableAgent] = None, + **kwargs, ) -> Tuple[List[autogen.ConversableAgent], Dict]: """ Build agents with generated configs. Args: use_oai_assistant: use OpenAI assistant api instead of self-constructed agent. + list_of_functions: list of functions to be associated to Agents user_proxy: user proxy's class that can be used to replace the default user proxy. Returns: @@ -695,6 +723,53 @@ def _build_agents( ) agent_list = agent_list + [user_proxy] + agent_details = [] + + for agent in agent_list[:-1]: + agent_details.append({"name": agent.name, "description": agent.description}) + + if list_of_functions: + for func in list_of_functions: + resp = ( + self.builder_model.create( + messages=[ + { + "role": "user", + "content": self.AGENT_FUNCTION_MAP_PROMPT.format( + function_name=func["name"], + function_description=func["description"], + format_agent_details='[{"name": "agent_name", "description": "agent description"}, ...]', + agent_details=str(json.dumps(agent_details)), + ), + } + ] + ) + .choices[0] + .message.content + ) + + autogen.agentchat.register_function( + func["function"], + caller=self.agent_procs_assign[resp][0], + executor=agent_list[0], + name=func["name"], + description=func["description"], + ) + + agents_current_system_message = [ + agent["system_message"] for agent in agent_configs if agent["name"] == resp + ][0] + + self.agent_procs_assign[resp][0].update_system_message( + self.UPDATED_AGENT_SYSTEM_MESSAGE.format( + agent_system_message=agents_current_system_message, + function_name=func["name"], + function_description=func["description"], + ) + ) + + print(f"Function {func['name']} is registered to agent {resp}.") + return agent_list, self.cached_configs.copy() def save(self, filepath: Optional[str] = None) -> str: diff --git a/autogen/agentchat/contrib/capabilities/text_compressors.py b/autogen/agentchat/contrib/capabilities/text_compressors.py index 78554bdc9357..fd203c35fcab 100644 --- a/autogen/agentchat/contrib/capabilities/text_compressors.py +++ b/autogen/agentchat/contrib/capabilities/text_compressors.py @@ -5,7 +5,7 @@ import llmlingua except ImportError: IMPORT_ERROR = ImportError( - "LLMLingua is not installed. Please install it with `pip install pyautogen[long-context]`" + "LLMLingua is not installed. Please install it with `pip install autogen-agentchat[long-context]~=0.2`" ) PromptCompressor = object else: diff --git a/autogen/agentchat/contrib/retrieve_user_proxy_agent.py b/autogen/agentchat/contrib/retrieve_user_proxy_agent.py index b247d7a158f6..ee8f74bb9a6c 100644 --- a/autogen/agentchat/contrib/retrieve_user_proxy_agent.py +++ b/autogen/agentchat/contrib/retrieve_user_proxy_agent.py @@ -9,7 +9,9 @@ try: import chromadb except ImportError as e: - raise ImportError(f"{e}. You can try `pip install pyautogen[retrievechat]`, or install `chromadb` manually.") + raise ImportError( + f"{e}. You can try `pip install autogen-agentchat[retrievechat]~=0.2`, or install `chromadb` manually." + ) from autogen.agentchat import UserProxyAgent from autogen.agentchat.agent import Agent from autogen.agentchat.contrib.vectordb.base import Document, QueryResults, VectorDB, VectorDBFactory diff --git a/autogen/agentchat/contrib/vectordb/couchbase.py b/autogen/agentchat/contrib/vectordb/couchbase.py index 66691fa2f2b7..ae978ef126b4 100644 --- a/autogen/agentchat/contrib/vectordb/couchbase.py +++ b/autogen/agentchat/contrib/vectordb/couchbase.py @@ -56,16 +56,7 @@ def __init__( wait_until_index_ready (float | None): Blocking call to wait until the database indexes are ready. None means no wait. Default is None. wait_until_document_ready (float | None): Blocking call to wait until the database documents are ready. None means no wait. Default is None. """ - print( - "CouchbaseVectorDB", - connection_string, - username, - password, - bucket_name, - scope_name, - collection_name, - index_name, - ) + self.embedding_function = embedding_function self.index_name = index_name @@ -119,6 +110,7 @@ def create_collection( try: collection_mgr = self.bucket.collections() collection_mgr.create_collection(self.scope.name, collection_name) + self.cluster.query(f"CREATE PRIMARY INDEX ON {self.bucket.name}.{self.scope.name}.{collection_name}") except Exception: if not get_or_create: @@ -287,7 +279,12 @@ def upsert_docs( [doc["content"]] ).tolist() # Gets new embedding even in case of document update - doc_content = {TEXT_KEY: doc["content"], "metadata": doc.get("metadata", {}), EMBEDDING_KEY: embedding} + doc_content = { + TEXT_KEY: doc["content"], + "metadata": doc.get("metadata", {}), + EMBEDDING_KEY: embedding, + "id": doc_id, + } docs_to_upsert[doc_id] = doc_content collection.upsert_multi(docs_to_upsert) diff --git a/autogen/agentchat/contrib/vectordb/pgvectordb.py b/autogen/agentchat/contrib/vectordb/pgvectordb.py index 6fce4a6db803..64f4ff333a90 100644 --- a/autogen/agentchat/contrib/vectordb/pgvectordb.py +++ b/autogen/agentchat/contrib/vectordb/pgvectordb.py @@ -4,16 +4,17 @@ from typing import Callable, List, Optional, Union import numpy as np + +# try: +import pgvector +from pgvector.psycopg import register_vector from sentence_transformers import SentenceTransformer from .base import Document, ItemID, QueryResults, VectorDB from .utils import get_logger -try: - import pgvector - from pgvector.psycopg import register_vector -except ImportError: - raise ImportError("Please install pgvector: `pip install pgvector`") +# except ImportError: +# raise ImportError("Please install pgvector: `pip install pgvector`") try: import psycopg @@ -416,6 +417,7 @@ def query( results = [] for query_text in query_texts: vector = self.embedding_function(query_text) + vector_string = "[" + ",".join([f"{x:.8f}" for x in vector]) + "]" if distance_type.lower() == "cosine": index_function = "<=>" @@ -428,7 +430,7 @@ def query( query = ( f"SELECT id, documents, embedding, metadatas " f"FROM {self.name} " - f"{clause} embedding {index_function} '{str(vector)}' {distance_threshold} " + f"{clause} embedding {index_function} '{vector_string}' {distance_threshold} " f"LIMIT {n_results}" ) cursor.execute(query) diff --git a/autogen/agentchat/conversable_agent.py b/autogen/agentchat/conversable_agent.py index e19cbd56de2b..3a21c0f7d0e4 100644 --- a/autogen/agentchat/conversable_agent.py +++ b/autogen/agentchat/conversable_agent.py @@ -7,7 +7,7 @@ import re import warnings from collections import defaultdict -from typing import Any, Callable, Dict, List, Literal, Optional, Tuple, Type, TypeVar, Union +from typing import Any, Callable, Coroutine, Dict, List, Literal, Optional, Tuple, Type, TypeVar, Union from openai import BadRequestError @@ -247,10 +247,13 @@ def __init__( # Registered hooks are kept in lists, indexed by hookable method, to be called in their order of registration. # New hookable methods should be added to this list as required to support new agent capabilities. - self.hook_lists: Dict[str, List[Callable]] = { + self.hook_lists: Dict[str, List[Union[Callable, Callable[..., Coroutine]]]] = { "process_last_received_message": [], + "a_process_last_received_message": [], "process_all_messages_before_reply": [], + "a_process_all_messages_before_reply": [], "process_message_before_send": [], + "a_process_message_before_send": [], } def _validate_llm_config(self, llm_config): @@ -680,11 +683,24 @@ def _process_message_before_send( """Process the message before sending it to the recipient.""" hook_list = self.hook_lists["process_message_before_send"] for hook in hook_list: + if inspect.iscoroutinefunction(hook): + continue message = hook( sender=self, message=message, recipient=recipient, silent=ConversableAgent._is_silent(self, silent) ) return message + async def _a_process_message_before_send( + self, message: Union[Dict, str], recipient: Agent, silent: bool + ) -> Union[Dict, str]: + """(async) Process the message before sending it to the recipient.""" + hook_list = self.hook_lists["a_process_message_before_send"] + for hook in hook_list: + if not inspect.iscoroutinefunction(hook): + continue + message = await hook(sender=self, message=message, recipient=recipient, silent=silent) + return message + def send( self, message: Union[Dict, str], @@ -774,7 +790,9 @@ async def a_send( Raises: ValueError: if the message can't be converted into a valid ChatCompletion message. """ - message = self._process_message_before_send(message, recipient, ConversableAgent._is_silent(self, silent)) + message = await self._a_process_message_before_send( + message, recipient, ConversableAgent._is_silent(self, silent) + ) # When the agent composes and sends the message, the role of the message is "assistant" # unless it's "function". valid = self._append_oai_message(message, "assistant", recipient, is_sending=True) @@ -2104,11 +2122,11 @@ async def a_generate_reply( # Call the hookable method that gives registered hooks a chance to process all messages. # Message modifications do not affect the incoming messages or self._oai_messages. - messages = self.process_all_messages_before_reply(messages) + messages = await self.a_process_all_messages_before_reply(messages) # Call the hookable method that gives registered hooks a chance to process the last message. # Message modifications do not affect the incoming messages or self._oai_messages. - messages = self.process_last_received_message(messages) + messages = await self.a_process_last_received_message(messages) for reply_func_tuple in self._reply_func_list: reply_func = reply_func_tuple["reply_func"] @@ -2786,6 +2804,19 @@ def register_hook(self, hookable_method: str, hook: Callable): assert hookable_method in self.hook_lists, f"{hookable_method} is not a hookable method." hook_list = self.hook_lists[hookable_method] assert hook not in hook_list, f"{hook} is already registered as a hook." + + # async hookable checks + expected_async = hookable_method.startswith("a_") + hook_is_async = inspect.iscoroutinefunction(hook) + if expected_async != hook_is_async: + context_type = "asynchronous" if expected_async else "synchronous" + warnings.warn( + f"Hook '{hook.__name__}' is {'asynchronous' if hook_is_async else 'synchronous'}, " + f"but it's being registered in a {context_type} context ('{hookable_method}'). " + "Ensure the hook matches the expected execution context.", + UserWarning, + ) + hook_list.append(hook) def process_all_messages_before_reply(self, messages: List[Dict]) -> List[Dict]: @@ -2800,9 +2831,28 @@ def process_all_messages_before_reply(self, messages: List[Dict]) -> List[Dict]: # Call each hook (in order of registration) to process the messages. processed_messages = messages for hook in hook_list: + if inspect.iscoroutinefunction(hook): + continue processed_messages = hook(processed_messages) return processed_messages + async def a_process_all_messages_before_reply(self, messages: List[Dict]) -> List[Dict]: + """ + Calls any registered capability hooks to process all messages, potentially modifying the messages. + """ + hook_list = self.hook_lists["a_process_all_messages_before_reply"] + # If no hooks are registered, or if there are no messages to process, return the original message list. + if len(hook_list) == 0 or messages is None: + return messages + + # Call each hook (in order of registration) to process the messages. + processed_messages = messages + for hook in hook_list: + if not inspect.iscoroutinefunction(hook): + continue + processed_messages = await hook(processed_messages) + return processed_messages + def process_last_received_message(self, messages: List[Dict]) -> List[Dict]: """ Calls any registered capability hooks to use and potentially modify the text of the last message, @@ -2836,6 +2886,8 @@ def process_last_received_message(self, messages: List[Dict]) -> List[Dict]: # Call each hook (in order of registration) to process the user's message. processed_user_content = user_content for hook in hook_list: + if inspect.iscoroutinefunction(hook): + continue processed_user_content = hook(processed_user_content) if processed_user_content == user_content: @@ -2846,6 +2898,51 @@ def process_last_received_message(self, messages: List[Dict]) -> List[Dict]: messages[-1]["content"] = processed_user_content return messages + async def a_process_last_received_message(self, messages: List[Dict]) -> List[Dict]: + """ + Calls any registered capability hooks to use and potentially modify the text of the last message, + as long as the last message is not a function call or exit command. + """ + + # If any required condition is not met, return the original message list. + hook_list = self.hook_lists["a_process_last_received_message"] + if len(hook_list) == 0: + return messages # No hooks registered. + if messages is None: + return None # No message to process. + if len(messages) == 0: + return messages # No message to process. + last_message = messages[-1] + if "function_call" in last_message: + return messages # Last message is a function call. + if "context" in last_message: + return messages # Last message contains a context key. + if "content" not in last_message: + return messages # Last message has no content. + + user_content = last_message["content"] + if not isinstance(user_content, str) and not isinstance(user_content, list): + # if the user_content is a string, it is for regular LLM + # if the user_content is a list, it should follow the multimodal LMM format. + return messages + if user_content == "exit": + return messages # Last message is an exit command. + + # Call each hook (in order of registration) to process the user's message. + processed_user_content = user_content + for hook in hook_list: + if not inspect.iscoroutinefunction(hook): + continue + processed_user_content = await hook(processed_user_content) + + if processed_user_content == user_content: + return messages # No hooks actually modified the user's message. + + # Replace the last user message with the expanded one. + messages = messages.copy() + messages[-1]["content"] = processed_user_content + return messages + def print_usage_summary(self, mode: Union[str, List[str]] = ["actual", "total"]) -> None: """Print the usage summary.""" iostream = IOStream.get_default() diff --git a/autogen/agentchat/groupchat.py b/autogen/agentchat/groupchat.py index c6355a13b94d..b0f0b9520aaf 100644 --- a/autogen/agentchat/groupchat.py +++ b/autogen/agentchat/groupchat.py @@ -12,9 +12,9 @@ from ..formatting_utils import colored from ..graph_utils import check_graph_validity, invert_disallowed_to_allowed from ..io.base import IOStream +from ..oai.client import ModelClient from ..runtime_logging import log_new_agent, logging_enabled from .agent import Agent -from .chat import ChatResult from .conversable_agent import ConversableAgent try: @@ -105,6 +105,8 @@ def custom_speaker_selection_func( "clear history" phrase in user prompt. This is experimental feature. See description of GroupChatManager.clear_agents_history function for more info. - send_introductions: send a round of introductions at the start of the group chat, so agents know who they can speak to (default: False) + - select_speaker_auto_model_client_cls: Custom model client class for the internal speaker select agent used during 'auto' speaker selection (optional) + - select_speaker_auto_llm_config: LLM config for the internal speaker select agent used during 'auto' speaker selection (optional) - role_for_select_speaker_messages: sets the role name for speaker selection when in 'auto' mode, typically 'user' or 'system'. (default: 'system') """ @@ -142,6 +144,8 @@ def custom_speaker_selection_func( Respond with ONLY the name of the speaker and DO NOT provide a reason.""" select_speaker_transform_messages: Optional[Any] = None select_speaker_auto_verbose: Optional[bool] = False + select_speaker_auto_model_client_cls: Optional[Union[ModelClient, List[ModelClient]]] = None + select_speaker_auto_llm_config: Optional[Union[Dict, Literal[False]]] = None role_for_select_speaker_messages: Optional[str] = "system" _VALID_SPEAKER_SELECTION_METHODS = ["auto", "manual", "random", "round_robin"] @@ -591,6 +595,79 @@ def _finalize_speaker(self, last_speaker: Agent, final: bool, name: str, agents: agent = self.agent_by_name(name) return agent if agent else self.next_agent(last_speaker, agents) + def _register_client_from_config(self, agent: Agent, config: Dict): + model_client_cls_to_match = config.get("model_client_cls") + if model_client_cls_to_match: + if not self.select_speaker_auto_model_client_cls: + raise ValueError( + "A custom model was detected in the config but no 'model_client_cls' " + "was supplied for registration in GroupChat." + ) + + if isinstance(self.select_speaker_auto_model_client_cls, list): + # Register the first custom model client class matching the name specified in the config + matching_model_cls = [ + client_cls + for client_cls in self.select_speaker_auto_model_client_cls + if client_cls.__name__ == model_client_cls_to_match + ] + if len(set(matching_model_cls)) > 1: + raise RuntimeError( + f"More than one unique 'model_client_cls' with __name__ '{model_client_cls_to_match}'." + ) + if not matching_model_cls: + raise ValueError( + "No model's __name__ matches the model client class " + f"'{model_client_cls_to_match}' specified in select_speaker_auto_llm_config." + ) + select_speaker_auto_model_client_cls = matching_model_cls[0] + else: + # Register the only custom model client + select_speaker_auto_model_client_cls = self.select_speaker_auto_model_client_cls + + agent.register_model_client(select_speaker_auto_model_client_cls) + + def _register_custom_model_clients(self, agent: ConversableAgent): + if not self.select_speaker_auto_llm_config: + return + + config_format_is_list = "config_list" in self.select_speaker_auto_llm_config.keys() + if config_format_is_list: + for config in self.select_speaker_auto_llm_config["config_list"]: + self._register_client_from_config(agent, config) + elif not config_format_is_list: + self._register_client_from_config(agent, self.select_speaker_auto_llm_config) + + def _create_internal_agents( + self, agents, max_attempts, messages, validate_speaker_name, selector: Optional[ConversableAgent] = None + ): + checking_agent = ConversableAgent("checking_agent", default_auto_reply=max_attempts) + + # Register the speaker validation function with the checking agent + checking_agent.register_reply( + [ConversableAgent, None], + reply_func=validate_speaker_name, # Validate each response + remove_other_reply_funcs=True, + ) + + # Override the selector's config if one was passed as a parameter to this class + speaker_selection_llm_config = self.select_speaker_auto_llm_config or selector.llm_config + + # Agent for selecting a single agent name from the response + speaker_selection_agent = ConversableAgent( + "speaker_selection_agent", + system_message=self.select_speaker_msg(agents), + chat_messages={checking_agent: messages}, + llm_config=speaker_selection_llm_config, + human_input_mode="NEVER", + # Suppresses some extra terminal outputs, outputs will be handled by select_speaker_auto_verbose + ) + + # Register any custom model passed in select_speaker_auto_llm_config with the speaker_selection_agent + self._register_custom_model_clients(speaker_selection_agent) + + return checking_agent, speaker_selection_agent + def _auto_select_speaker( self, last_speaker: Agent, @@ -644,28 +721,8 @@ def validate_speaker_name(recipient, messages, sender, config) -> Tuple[bool, Un # Two-agent chat for speaker selection # Agent for checking the response from the speaker_select_agent - checking_agent = ConversableAgent("checking_agent", default_auto_reply=max_attempts) - - # Register the speaker validation function with the checking agent - checking_agent.register_reply( - [ConversableAgent, None], - reply_func=validate_speaker_name, # Validate each response - remove_other_reply_funcs=True, - ) - - # NOTE: Do we have a speaker prompt (select_speaker_prompt_template is not None)? If we don't, we need to feed in the last message to start the nested chat - - # Agent for selecting a single agent name from the response - speaker_selection_agent = ConversableAgent( - "speaker_selection_agent", - system_message=self.select_speaker_msg(agents), - chat_messages=( - {checking_agent: messages} - if self.select_speaker_prompt_template is not None - else {checking_agent: messages[:-1]} - ), - llm_config=selector.llm_config, - human_input_mode="NEVER", # Suppresses some extra terminal outputs, outputs will be handled by select_speaker_auto_verbose + checking_agent, speaker_selection_agent = self._create_internal_agents( + agents, max_attempts, messages, validate_speaker_name, selector ) # Create the starting message @@ -747,24 +804,8 @@ def validate_speaker_name(recipient, messages, sender, config) -> Tuple[bool, Un # Two-agent chat for speaker selection # Agent for checking the response from the speaker_select_agent - checking_agent = ConversableAgent("checking_agent", default_auto_reply=max_attempts) - - # Register the speaker validation function with the checking agent - checking_agent.register_reply( - [ConversableAgent, None], - reply_func=validate_speaker_name, # Validate each response - remove_other_reply_funcs=True, - ) - - # NOTE: Do we have a speaker prompt (select_speaker_prompt_template is not None)? If we don't, we need to feed in the last message to start the nested chat - - # Agent for selecting a single agent name from the response - speaker_selection_agent = ConversableAgent( - "speaker_selection_agent", - system_message=self.select_speaker_msg(agents), - chat_messages={checking_agent: messages}, - llm_config=selector.llm_config, - human_input_mode="NEVER", # Suppresses some extra terminal outputs, outputs will be handled by select_speaker_auto_verbose + checking_agent, speaker_selection_agent = self._create_internal_agents( + agents, max_attempts, messages, validate_speaker_name, selector ) # Create the starting message diff --git a/autogen/coding/kubernetes/__init__.py b/autogen/coding/kubernetes/__init__.py new file mode 100644 index 000000000000..3129ec86bf3e --- /dev/null +++ b/autogen/coding/kubernetes/__init__.py @@ -0,0 +1,5 @@ +from .pod_commandline_code_executor import PodCommandLineCodeExecutor + +__all__ = [ + "PodCommandLineCodeExecutor", +] diff --git a/autogen/coding/kubernetes/pod_commandline_code_executor.py b/autogen/coding/kubernetes/pod_commandline_code_executor.py new file mode 100644 index 000000000000..a232a3922717 --- /dev/null +++ b/autogen/coding/kubernetes/pod_commandline_code_executor.py @@ -0,0 +1,323 @@ +from __future__ import annotations + +import atexit +import importlib +import sys +import textwrap +import uuid +from hashlib import md5 +from pathlib import Path +from time import sleep +from types import TracebackType +from typing import Any, ClassVar, Dict, List, Optional, Type, Union + +client = importlib.import_module("kubernetes.client") +config = importlib.import_module("kubernetes.config") +ApiException = importlib.import_module("kubernetes.client.rest").ApiException +stream = importlib.import_module("kubernetes.stream").stream + +from ...code_utils import TIMEOUT_MSG, _cmd +from ..base import CodeBlock, CodeExecutor, CodeExtractor, CommandLineCodeResult +from ..markdown_code_extractor import MarkdownCodeExtractor +from ..utils import _get_file_name_from_content, silence_pip + +if sys.version_info >= (3, 11): + from typing import Self +else: + from typing_extensions import Self + + +class PodCommandLineCodeExecutor(CodeExecutor): + DEFAULT_EXECUTION_POLICY: ClassVar[Dict[str, bool]] = { + "bash": True, + "shell": True, + "sh": True, + "pwsh": False, + "powershell": False, + "ps1": False, + "python": True, + "javascript": False, + "html": False, + "css": False, + } + LANGUAGE_ALIASES: ClassVar[Dict[str, str]] = { + "py": "python", + "js": "javascript", + } + LANGUAGE_FILE_EXTENSION: ClassVar[Dict[str, str]] = { + "python": "py", + "javascript": "js", + "bash": "sh", + "shell": "sh", + "sh": "sh", + } + + def __init__( + self, + image: str = "python:3-slim", + pod_name: Optional[str] = None, + namespace: Optional[str] = None, + pod_spec: Optional[client.V1Pod] = None, # type: ignore + container_name: Optional[str] = "autogen-code-exec", + timeout: int = 60, + work_dir: Union[Path, str] = Path("/workspace"), + kube_config_file: Optional[str] = None, + stop_container: bool = True, + execution_policies: Optional[Dict[str, bool]] = None, + ): + """(Experimental) A code executor class that executes code through + a command line environment in a kubernetes pod. + + The executor first saves each code block in a file in the working + directory, and then executes the code file in the container. + The executor executes the code blocks in the order they are received. + Currently, the executor only supports Python and shell scripts. + For Python code, use the language "python" for the code block. + For shell scripts, use the language "bash", "shell", or "sh" for the code + block. + + Args: + image (_type_, optional): Docker image to use for code execution. + Defaults to "python:3-slim". + pod_name (Optional[str], optional): Name of the kubernetes pod + which is created. If None, will autogenerate a name. Defaults to None. + namespace (Optional[str], optional): Namespace of kubernetes pod + which is created. If None, will use current namespace of this instance + pod_spec (Optional[client.V1Pod], optional): Specification of kubernetes pod. + custom pod spec can be provided with this param. + if pod_spec is provided, params above(image, pod_name, namespace) are neglected. + container_name (Optional[str], optional): Name of the container where code block will be + executed. if pod_spec param is provided, container_name must be provided also. + timeout (int, optional): The timeout for code execution. Defaults to 60. + work_dir (Union[Path, str], optional): The working directory for the code + execution. Defaults to Path("/workspace"). + kube_config_file (Optional[str], optional): kubernetes configuration file path. + If None, will use KUBECONFIG environment variables or service account token(incluster config) + stop_container (bool, optional): If true, will automatically stop the + container when stop is called, when the context manager exits or when + the Python process exits with atext. Defaults to True. + execution_policies (dict[str, bool], optional): defines supported execution language + + Raises: + ValueError: On argument error, or if the container fails to start. + """ + if kube_config_file is None: + config.load_config() + else: + config.load_config(config_file=kube_config_file) + + self._api_client = client.CoreV1Api() + + if timeout < 1: + raise ValueError("Timeout must be greater than or equal to 1.") + self._timeout = timeout + + if isinstance(work_dir, str): + work_dir = Path(work_dir) + self._work_dir: Path = work_dir + + if container_name is None: + container_name = "autogen-code-exec" + self._container_name = container_name + + # Start a container from the image, read to exec commands later + if pod_spec: + pod = pod_spec + else: + if pod_name is None: + pod_name = f"autogen-code-exec-{uuid.uuid4()}" + if namespace is None: + namespace_path = "/var/run/secrets/kubernetes.io/serviceaccount/namespace" + if not Path(namespace_path).is_file(): + raise ValueError("Namespace where the pod will be launched must be provided") + with open(namespace_path, "r") as f: + namespace = f.read() + + pod = client.V1Pod( + metadata=client.V1ObjectMeta(name=pod_name, namespace=namespace), + spec=client.V1PodSpec( + restart_policy="Never", + containers=[ + client.V1Container( + args=["-c", "while true;do sleep 5; done"], + command=["/bin/sh"], + name=container_name, + image=image, + ) + ], + ), + ) + + try: + pod_name = pod.metadata.name + namespace = pod.metadata.namespace + self._pod = self._api_client.create_namespaced_pod(namespace=namespace, body=pod) + except ApiException as e: + raise ValueError(f"Creating pod failed: {e}") + + self._wait_for_ready() + + def cleanup() -> None: + try: + self._api_client.delete_namespaced_pod(pod_name, namespace) + except ApiException: + pass + atexit.unregister(cleanup) + + self._cleanup = cleanup + + if stop_container: + atexit.register(cleanup) + + self.execution_policies = self.DEFAULT_EXECUTION_POLICY.copy() + if execution_policies is not None: + self.execution_policies.update(execution_policies) + + def _wait_for_ready(self, stop_time: float = 0.1) -> None: + elapsed_time = 0.0 + name = self._pod.metadata.name + namespace = self._pod.metadata.namespace + while True: + sleep(stop_time) + elapsed_time += stop_time + if elapsed_time > self._timeout: + raise ValueError( + f"pod name {name} on namespace {namespace} is not Ready after timeout {self._timeout} seconds" + ) + try: + pod_status = self._api_client.read_namespaced_pod_status(name, namespace) + if pod_status.status.phase == "Running": + break + except ApiException as e: + raise ValueError(f"reading pod status failed: {e}") + + @property + def timeout(self) -> int: + """(Experimental) The timeout for code execution.""" + return self._timeout + + @property + def work_dir(self) -> Path: + """(Experimental) The working directory for the code execution.""" + return self._work_dir + + @property + def code_extractor(self) -> CodeExtractor: + """(Experimental) Export a code extractor that can be used by an agent.""" + return MarkdownCodeExtractor() + + def execute_code_blocks(self, code_blocks: List[CodeBlock]) -> CommandLineCodeResult: + """(Experimental) Execute the code blocks and return the result. + + Args: + code_blocks (List[CodeBlock]): The code blocks to execute. + + Returns: + CommandlineCodeResult: The result of the code execution.""" + + if len(code_blocks) == 0: + raise ValueError("No code blocks to execute.") + + outputs = [] + files = [] + last_exit_code = 0 + for code_block in code_blocks: + lang = self.LANGUAGE_ALIASES.get(code_block.language.lower(), code_block.language.lower()) + if lang not in self.DEFAULT_EXECUTION_POLICY: + outputs.append(f"Unsupported language {lang}\n") + last_exit_code = 1 + break + + execute_code = self.execution_policies.get(lang, False) + code = silence_pip(code_block.code, lang) + if lang in ["bash", "shell", "sh"]: + code = "\n".join(["#!/bin/bash", code]) + + try: + filename = _get_file_name_from_content(code, self._work_dir) + except ValueError: + outputs.append("Filename is not in the workspace") + last_exit_code = 1 + break + + if not filename: + extension = self.LANGUAGE_FILE_EXTENSION.get(lang, lang) + filename = f"tmp_code_{md5(code.encode()).hexdigest()}.{extension}" + + code_path = self._work_dir / filename + + exec_script = textwrap.dedent( + """ + if [ ! -d "{workspace}" ]; then + mkdir {workspace} + fi + cat <{code_path}\n + {code} + EOM + chmod +x {code_path}""" + ) + exec_script = exec_script.format(workspace=str(self._work_dir), code_path=code_path, code=code) + stream( + self._api_client.connect_get_namespaced_pod_exec, + self._pod.metadata.name, + self._pod.metadata.namespace, + command=["/bin/sh", "-c", exec_script], + container=self._container_name, + stderr=True, + stdin=False, + stdout=True, + tty=False, + ) + + files.append(code_path) + + if not execute_code: + outputs.append(f"Code saved to {str(code_path)}\n") + continue + + resp = stream( + self._api_client.connect_get_namespaced_pod_exec, + self._pod.metadata.name, + self._pod.metadata.namespace, + command=["timeout", str(self._timeout), _cmd(lang), str(code_path)], + container=self._container_name, + stderr=True, + stdin=False, + stdout=True, + tty=False, + _preload_content=False, + ) + + stdout_messages = [] + stderr_messages = [] + while resp.is_open(): + resp.update(timeout=1) + if resp.peek_stderr(): + stderr_messages.append(resp.read_stderr()) + if resp.peek_stdout(): + stdout_messages.append(resp.read_stdout()) + outputs.extend(stdout_messages + stderr_messages) + exit_code = resp.returncode + resp.close() + + if exit_code == 124: + outputs.append("\n" + TIMEOUT_MSG) + + last_exit_code = exit_code + if exit_code != 0: + break + + code_file = str(files[0]) if files else None + return CommandLineCodeResult(exit_code=last_exit_code, output="".join(outputs), code_file=code_file) + + def stop(self) -> None: + """(Experimental) Stop the code executor.""" + self._cleanup() + + def __enter__(self) -> Self: + return self + + def __exit__( + self, exc_type: Optional[Type[BaseException]], exc_val: Optional[BaseException], exc_tb: Optional[TracebackType] + ) -> None: + self.stop() diff --git a/autogen/logger/file_logger.py b/autogen/logger/file_logger.py index 07c9c3b76a76..b902652fd4b1 100644 --- a/autogen/logger/file_logger.py +++ b/autogen/logger/file_logger.py @@ -19,10 +19,12 @@ from autogen import Agent, ConversableAgent, OpenAIWrapper from autogen.oai.anthropic import AnthropicClient from autogen.oai.bedrock import BedrockClient + from autogen.oai.cerebras import CerebrasClient from autogen.oai.cohere import CohereClient from autogen.oai.gemini import GeminiClient from autogen.oai.groq import GroqClient from autogen.oai.mistral import MistralAIClient + from autogen.oai.ollama import OllamaClient from autogen.oai.together import TogetherClient logger = logging.getLogger(__name__) @@ -210,12 +212,14 @@ def log_new_client( client: ( AzureOpenAI | OpenAI + | CerebrasClient | GeminiClient | AnthropicClient | MistralAIClient | TogetherClient | GroqClient | CohereClient + | OllamaClient | BedrockClient ), wrapper: OpenAIWrapper, diff --git a/autogen/logger/sqlite_logger.py b/autogen/logger/sqlite_logger.py index f76d039ce9de..bdfa9928d141 100644 --- a/autogen/logger/sqlite_logger.py +++ b/autogen/logger/sqlite_logger.py @@ -20,10 +20,12 @@ from autogen import Agent, ConversableAgent, OpenAIWrapper from autogen.oai.anthropic import AnthropicClient from autogen.oai.bedrock import BedrockClient + from autogen.oai.cerebras import CerebrasClient from autogen.oai.cohere import CohereClient from autogen.oai.gemini import GeminiClient from autogen.oai.groq import GroqClient from autogen.oai.mistral import MistralAIClient + from autogen.oai.ollama import OllamaClient from autogen.oai.together import TogetherClient logger = logging.getLogger(__name__) @@ -397,12 +399,14 @@ def log_new_client( client: Union[ AzureOpenAI, OpenAI, + CerebrasClient, GeminiClient, AnthropicClient, MistralAIClient, TogetherClient, GroqClient, CohereClient, + OllamaClient, BedrockClient, ], wrapper: OpenAIWrapper, diff --git a/autogen/oai/cerebras.py b/autogen/oai/cerebras.py new file mode 100644 index 000000000000..e87b048e1366 --- /dev/null +++ b/autogen/oai/cerebras.py @@ -0,0 +1,270 @@ +"""Create an OpenAI-compatible client using Cerebras's API. + +Example: + llm_config={ + "config_list": [{ + "api_type": "cerebras", + "model": "llama3.1-8b", + "api_key": os.environ.get("CEREBRAS_API_KEY") + }] + } + + agent = autogen.AssistantAgent("my_agent", llm_config=llm_config) + +Install Cerebras's python library using: pip install --upgrade cerebras_cloud_sdk + +Resources: +- https://inference-docs.cerebras.ai/quickstart +""" + +from __future__ import annotations + +import copy +import os +import time +import warnings +from typing import Any, Dict, List + +from cerebras.cloud.sdk import Cerebras, Stream +from openai.types.chat import ChatCompletion, ChatCompletionMessageToolCall +from openai.types.chat.chat_completion import ChatCompletionMessage, Choice +from openai.types.completion_usage import CompletionUsage + +from autogen.oai.client_utils import should_hide_tools, validate_parameter + +CEREBRAS_PRICING_1K = { + # Convert pricing per million to per thousand tokens. + "llama3.1-8b": (0.10 / 1000, 0.10 / 1000), + "llama3.1-70b": (0.60 / 1000, 0.60 / 1000), +} + + +class CerebrasClient: + """Client for Cerebras's API.""" + + def __init__(self, api_key=None, **kwargs): + """Requires api_key or environment variable to be set + + Args: + api_key (str): The API key for using Cerebras (or environment variable CEREBRAS_API_KEY needs to be set) + """ + # Ensure we have the api_key upon instantiation + self.api_key = api_key + if not self.api_key: + self.api_key = os.getenv("CEREBRAS_API_KEY") + + assert ( + self.api_key + ), "Please include the api_key in your config list entry for Cerebras or set the CEREBRAS_API_KEY env variable." + + def message_retrieval(self, response: ChatCompletion) -> List: + """ + Retrieve and return a list of strings or a list of Choice.Message from the response. + + NOTE: if a list of Choice.Message is returned, it currently needs to contain the fields of OpenAI's ChatCompletion Message object, + since that is expected for function or tool calling in the rest of the codebase at the moment, unless a custom agent is being used. + """ + return [choice.message for choice in response.choices] + + def cost(self, response: ChatCompletion) -> float: + # Note: This field isn't explicitly in `ChatCompletion`, but is injected during chat creation. + return response.cost + + @staticmethod + def get_usage(response: ChatCompletion) -> Dict: + """Return usage summary of the response using RESPONSE_USAGE_KEYS.""" + # ... # pragma: no cover + return { + "prompt_tokens": response.usage.prompt_tokens, + "completion_tokens": response.usage.completion_tokens, + "total_tokens": response.usage.total_tokens, + "cost": response.cost, + "model": response.model, + } + + def parse_params(self, params: Dict[str, Any]) -> Dict[str, Any]: + """Loads the parameters for Cerebras API from the passed in parameters and returns a validated set. Checks types, ranges, and sets defaults""" + cerebras_params = {} + + # Check that we have what we need to use Cerebras's API + # We won't enforce the available models as they are likely to change + cerebras_params["model"] = params.get("model", None) + assert cerebras_params[ + "model" + ], "Please specify the 'model' in your config list entry to nominate the Cerebras model to use." + + # Validate allowed Cerebras parameters + # https://inference-docs.cerebras.ai/api-reference/chat-completions + cerebras_params["max_tokens"] = validate_parameter(params, "max_tokens", int, True, None, (0, None), None) + cerebras_params["seed"] = validate_parameter(params, "seed", int, True, None, None, None) + cerebras_params["stream"] = validate_parameter(params, "stream", bool, True, False, None, None) + cerebras_params["temperature"] = validate_parameter( + params, "temperature", (int, float), True, 1, (0, 1.5), None + ) + cerebras_params["top_p"] = validate_parameter(params, "top_p", (int, float), True, None, None, None) + + return cerebras_params + + def create(self, params: Dict) -> ChatCompletion: + + messages = params.get("messages", []) + + # Convert AutoGen messages to Cerebras messages + cerebras_messages = oai_messages_to_cerebras_messages(messages) + + # Parse parameters to the Cerebras API's parameters + cerebras_params = self.parse_params(params) + + # Add tools to the call if we have them and aren't hiding them + if "tools" in params: + hide_tools = validate_parameter( + params, "hide_tools", str, False, "never", None, ["if_all_run", "if_any_run", "never"] + ) + if not should_hide_tools(cerebras_messages, params["tools"], hide_tools): + cerebras_params["tools"] = params["tools"] + + cerebras_params["messages"] = cerebras_messages + + # We use chat model by default, and set max_retries to 5 (in line with typical retries loop) + client = Cerebras(api_key=self.api_key, max_retries=5) + + # Token counts will be returned + prompt_tokens = 0 + completion_tokens = 0 + total_tokens = 0 + + # Streaming tool call recommendations + streaming_tool_calls = [] + + ans = None + try: + response = client.chat.completions.create(**cerebras_params) + except Exception as e: + raise RuntimeError(f"Cerebras exception occurred: {e}") + else: + + if cerebras_params["stream"]: + # Read in the chunks as they stream, taking in tool_calls which may be across + # multiple chunks if more than one suggested + ans = "" + for chunk in response: + # Grab first choice, which _should_ always be generated. + ans = ans + (chunk.choices[0].delta.content or "") + + if chunk.choices[0].delta.tool_calls: + # We have a tool call recommendation + for tool_call in chunk.choices[0].delta.tool_calls: + streaming_tool_calls.append( + ChatCompletionMessageToolCall( + id=tool_call.id, + function={ + "name": tool_call.function.name, + "arguments": tool_call.function.arguments, + }, + type="function", + ) + ) + + if chunk.choices[0].finish_reason: + prompt_tokens = chunk.x_cerebras.usage.prompt_tokens + completion_tokens = chunk.x_cerebras.usage.completion_tokens + total_tokens = chunk.x_cerebras.usage.total_tokens + else: + # Non-streaming finished + ans: str = response.choices[0].message.content + + prompt_tokens = response.usage.prompt_tokens + completion_tokens = response.usage.completion_tokens + total_tokens = response.usage.total_tokens + + if response is not None: + if isinstance(response, Stream): + # Streaming response + if chunk.choices[0].finish_reason == "tool_calls": + cerebras_finish = "tool_calls" + tool_calls = streaming_tool_calls + else: + cerebras_finish = "stop" + tool_calls = None + + response_content = ans + response_id = chunk.id + else: + # Non-streaming response + # If we have tool calls as the response, populate completed tool calls for our return OAI response + if response.choices[0].finish_reason == "tool_calls": + cerebras_finish = "tool_calls" + tool_calls = [] + for tool_call in response.choices[0].message.tool_calls: + tool_calls.append( + ChatCompletionMessageToolCall( + id=tool_call.id, + function={"name": tool_call.function.name, "arguments": tool_call.function.arguments}, + type="function", + ) + ) + else: + cerebras_finish = "stop" + tool_calls = None + + response_content = response.choices[0].message.content + response_id = response.id + else: + raise RuntimeError("Failed to get response from Cerebras after retrying 5 times.") + + # 3. convert output + message = ChatCompletionMessage( + role="assistant", + content=response_content, + function_call=None, + tool_calls=tool_calls, + ) + choices = [Choice(finish_reason=cerebras_finish, index=0, message=message)] + + response_oai = ChatCompletion( + id=response_id, + model=cerebras_params["model"], + created=int(time.time()), + object="chat.completion", + choices=choices, + usage=CompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ), + # Note: This seems to be a field that isn't in the schema of `ChatCompletion`, so Pydantic + # just adds it dynamically. + cost=calculate_cerebras_cost(prompt_tokens, completion_tokens, cerebras_params["model"]), + ) + + return response_oai + + +def oai_messages_to_cerebras_messages(messages: list[Dict[str, Any]]) -> list[dict[str, Any]]: + """Convert messages from OAI format to Cerebras's format. + We correct for any specific role orders and types. + """ + + cerebras_messages = copy.deepcopy(messages) + + # Remove the name field + for message in cerebras_messages: + if "name" in message: + message.pop("name", None) + + return cerebras_messages + + +def calculate_cerebras_cost(input_tokens: int, output_tokens: int, model: str) -> float: + """Calculate the cost of the completion using the Cerebras pricing.""" + total = 0.0 + + if model in CEREBRAS_PRICING_1K: + input_cost_per_k, output_cost_per_k = CEREBRAS_PRICING_1K[model] + input_cost = (input_tokens / 1000) * input_cost_per_k + output_cost = (output_tokens / 1000) * output_cost_per_k + total = input_cost + output_cost + else: + warnings.warn(f"Cost calculation not available for model {model}", UserWarning) + + return total diff --git a/autogen/oai/client.py b/autogen/oai/client.py index 4b77815e7eb7..833cc7772196 100644 --- a/autogen/oai/client.py +++ b/autogen/oai/client.py @@ -6,7 +6,6 @@ import uuid from typing import Any, Callable, Dict, List, Optional, Protocol, Tuple, Union -from flaml.automl.logger import logger_formatter from pydantic import BaseModel from autogen.cache import Cache @@ -16,6 +15,7 @@ from autogen.runtime_logging import log_chat_completion, log_new_client, log_new_wrapper, logging_enabled from autogen.token_count_utils import count_token +from .client_utils import logger_formatter from .rate_limiters import RateLimiter, TimeRateLimiter TOOL_ENABLED = False @@ -44,6 +44,13 @@ TOOL_ENABLED = True ERROR = None +try: + from autogen.oai.cerebras import CerebrasClient + + cerebras_import_exception: Optional[ImportError] = None +except ImportError as e: + cerebras_import_exception = e + try: from autogen.oai.gemini import GeminiClient @@ -86,6 +93,13 @@ except ImportError as e: cohere_import_exception = e +try: + from autogen.oai.ollama import OllamaClient + + ollama_import_exception: Optional[ImportError] = None +except ImportError as e: + ollama_import_exception = e + try: from autogen.oai.bedrock import BedrockClient @@ -165,10 +179,6 @@ class OpenAIClient: def __init__(self, client: Union[OpenAI, AzureOpenAI]): self._oai_client = client - if not isinstance(client, openai.AzureOpenAI) and str(client.base_url).startswith(OPEN_API_BASE_URL_PREFIX): - logger.warning( - "The API key specified is not a valid OpenAI format; it won't work with the OpenAI-hosted model." - ) def message_retrieval( self, response: Union[ChatCompletion, Completion] @@ -505,6 +515,11 @@ def _register_default_client(self, config: Dict[str, Any], openai_config: Dict[s self._configure_azure_openai(config, openai_config) client = AzureOpenAI(**openai_config) self._clients.append(OpenAIClient(client)) + elif api_type is not None and api_type.startswith("cerebras"): + if cerebras_import_exception: + raise ImportError("Please install `cerebras_cloud_sdk` to use Cerebras OpenAI API.") + client = CerebrasClient(**openai_config) + self._clients.append(client) elif api_type is not None and api_type.startswith("google"): if gemini_import_exception: raise ImportError("Please install `google-generativeai` to use Google OpenAI API.") @@ -537,6 +552,11 @@ def _register_default_client(self, config: Dict[str, Any], openai_config: Dict[s raise ImportError("Please install `cohere` to use the Cohere API.") client = CohereClient(**openai_config) self._clients.append(client) + elif api_type is not None and api_type.startswith("ollama"): + if ollama_import_exception: + raise ImportError("Please install with `[ollama]` option to use the Ollama API.") + client = OllamaClient(**openai_config) + self._clients.append(client) elif api_type is not None and api_type.startswith("bedrock"): self._configure_openai_config_for_bedrock(config, openai_config) if bedrock_import_exception: diff --git a/autogen/oai/client_utils.py b/autogen/oai/client_utils.py index 55730485b40c..5da4c49c2a9b 100644 --- a/autogen/oai/client_utils.py +++ b/autogen/oai/client_utils.py @@ -1,8 +1,13 @@ """Utilities for client classes""" +import logging import warnings from typing import Any, Dict, List, Optional, Tuple +logger_formatter = logging.Formatter( + "[%(name)s: %(asctime)s] {%(lineno)d} %(levelname)s - %(message)s", "%m-%d %H:%M:%S" +) + def validate_parameter( params: Dict[str, Any], diff --git a/autogen/oai/cohere.py b/autogen/oai/cohere.py index e9a89c9cabd8..cd30c74dd872 100644 --- a/autogen/oai/cohere.py +++ b/autogen/oai/cohere.py @@ -31,12 +31,11 @@ from cohere import Client as Cohere from cohere.types import ToolParameterDefinitionsValue, ToolResult -from flaml.automl.logger import logger_formatter from openai.types.chat import ChatCompletion, ChatCompletionMessageToolCall from openai.types.chat.chat_completion import ChatCompletionMessage, Choice from openai.types.completion_usage import CompletionUsage -from autogen.oai.client_utils import validate_parameter +from .client_utils import logger_formatter, validate_parameter logger = logging.getLogger(__name__) if not logger.handlers: diff --git a/autogen/oai/completion.py b/autogen/oai/completion.py index 5a62cde33df0..d4866d345145 100644 --- a/autogen/oai/completion.py +++ b/autogen/oai/completion.py @@ -8,9 +8,9 @@ import numpy as np from flaml import BlendSearch, tune -from flaml.automl.logger import logger_formatter from flaml.tune.space import is_constant +from .client_utils import logger_formatter from .openai_utils import get_key try: diff --git a/autogen/oai/gemini.py b/autogen/oai/gemini.py index 33790c9851c6..d44e41112ac9 100644 --- a/autogen/oai/gemini.py +++ b/autogen/oai/gemini.py @@ -32,6 +32,8 @@ from __future__ import annotations import base64 +import copy +import json import logging import os import random @@ -39,24 +41,39 @@ import time import warnings from io import BytesIO -from typing import Any, Dict, List, Mapping, Union +from typing import Any, Dict, List, Union import google.generativeai as genai import requests import vertexai -from google.ai.generativelanguage import Content, Part +from google.ai.generativelanguage import Content, FunctionCall, FunctionDeclaration, FunctionResponse, Part, Tool from google.api_core.exceptions import InternalServerError from google.auth.credentials import Credentials -from openai.types.chat import ChatCompletion +from openai.types.chat import ChatCompletion, ChatCompletionMessageToolCall from openai.types.chat.chat_completion import ChatCompletionMessage, Choice +from openai.types.chat.chat_completion_message_tool_call import Function from openai.types.completion_usage import CompletionUsage from PIL import Image -from vertexai.generative_models import Content as VertexAIContent +from vertexai.generative_models import ( + Content as VertexAIContent, +) +from vertexai.generative_models import ( + FunctionDeclaration as VertexAIFunctionDeclaration, +) +from vertexai.generative_models import ( + GenerationConfig as VertexAIGenerationConfig, +) from vertexai.generative_models import GenerativeModel from vertexai.generative_models import HarmBlockThreshold as VertexAIHarmBlockThreshold from vertexai.generative_models import HarmCategory as VertexAIHarmCategory from vertexai.generative_models import Part as VertexAIPart from vertexai.generative_models import SafetySetting as VertexAISafetySetting +from vertexai.generative_models import ( + Tool as VertexAITool, +) +from vertexai.generative_models import ( + ToolConfig as VertexAIToolConfig, +) logger = logging.getLogger(__name__) @@ -107,7 +124,7 @@ def __init__(self, **kwargs): Args: api_key (str): The API key for using Gemini. - credentials (google.auth.credentials.Credentials): credentials to be used for authentication with vertexai. + credentials (google.auth.credentials.Credentials): credentials to be used for authentication with vertexai. google_application_credentials (str): Path to the JSON service account key file of the service account. Alternatively, the GOOGLE_APPLICATION_CREDENTIALS environment variable can also be set instead of using this argument. @@ -171,6 +188,8 @@ def create(self, params: Dict) -> ChatCompletion: params.get("api_type", "google") # not used messages = params.get("messages", []) + tools = params.get("tools", []) + tool_config = params.get("tool_config", {}) stream = params.get("stream", False) n_response = params.get("n", 1) system_instruction = params.get("system_instruction", None) @@ -183,6 +202,7 @@ def create(self, params: Dict) -> ChatCompletion: } if self.use_vertexai: safety_settings = GeminiClient._to_vertexai_safety_settings(params.get("safety_settings", {})) + tool_config = GeminiClient._to_vertexai_tool_config(tool_config, tools) else: safety_settings = params.get("safety_settings", {}) @@ -198,12 +218,15 @@ def create(self, params: Dict) -> ChatCompletion: if "vision" not in model_name: # A. create and call the chat model. gemini_messages = self._oai_messages_to_gemini_messages(messages) + gemini_tools = self._oai_tools_to_gemini_tools(tools) if self.use_vertexai: model = GenerativeModel( model_name, generation_config=generation_config, safety_settings=safety_settings, system_instruction=system_instruction, + tools=gemini_tools, + tool_config=tool_config, ) chat = model.start_chat(history=gemini_messages[:-1], response_validation=response_validation) else: @@ -213,12 +236,13 @@ def create(self, params: Dict) -> ChatCompletion: generation_config=generation_config, safety_settings=safety_settings, system_instruction=system_instruction, + tools=gemini_tools, ) genai.configure(api_key=self.api_key) chat = model.start_chat(history=gemini_messages[:-1]) max_retries = 5 for attempt in range(max_retries): - ans = None + ans: Union[Content, VertexAIContent] = None try: response = chat.send_message( gemini_messages[-1].parts, stream=stream, safety_settings=safety_settings @@ -234,7 +258,7 @@ def create(self, params: Dict) -> ChatCompletion: raise RuntimeError(f"Google GenAI exception occurred while calling Gemini API: {e}") else: # `ans = response.text` is unstable. Use the following code instead. - ans: str = chat.history[-1].parts[0].text + ans: Union[Content, VertexAIContent] = chat.history[-1] break if ans is None: @@ -262,7 +286,7 @@ def create(self, params: Dict) -> ChatCompletion: # Gemini's vision model does not support chat history yet # chat = model.start_chat(history=gemini_messages[:-1]) # response = chat.send_message(gemini_messages[-1].parts) - user_message = self._oai_content_to_gemini_content(messages[-1]["content"]) + user_message = self._oai_content_to_gemini_content(messages[-1]) if len(messages) > 2: warnings.warn( "Warning: Gemini's vision model does not support chat history yet.", @@ -273,16 +297,14 @@ def create(self, params: Dict) -> ChatCompletion: response = model.generate_content(user_message, stream=stream) # ans = response.text if self.use_vertexai: - ans: str = response.candidates[0].content.parts[0].text + ans: VertexAIContent = response.candidates[0].content else: - ans: str = response._result.candidates[0].content.parts[0].text + ans: Content = response._result.candidates[0].content prompt_tokens = model.count_tokens(user_message).total_tokens - completion_tokens = model.count_tokens(ans).total_tokens + completion_tokens = model.count_tokens(ans.parts[0].text).total_tokens - # 3. convert output - message = ChatCompletionMessage(role="assistant", content=ans, function_call=None, tool_calls=None) - choices = [Choice(finish_reason="stop", index=0, message=message)] + choices = self._gemini_content_to_oai_choices(ans) response_oai = ChatCompletion( id=str(random.randint(0, 1000)), @@ -295,31 +317,87 @@ def create(self, params: Dict) -> ChatCompletion: completion_tokens=completion_tokens, total_tokens=prompt_tokens + completion_tokens, ), - cost=calculate_gemini_cost(prompt_tokens, completion_tokens, model_name), + cost=self._calculate_gemini_cost(prompt_tokens, completion_tokens, model_name), ) return response_oai - def _oai_content_to_gemini_content(self, content: Union[str, List]) -> List: + # If str is not a json string return str as is + def _to_json(self, str) -> dict: + try: + return json.loads(str) + except ValueError: + return str + + def _oai_content_to_gemini_content(self, message: Dict[str, Any]) -> List: """Convert content from OAI format to Gemini format""" rst = [] - if isinstance(content, str): - if content == "": - content = "empty" # Empty content is not allowed. + if isinstance(message["content"], str): + if message["content"] == "": + message["content"] = "empty" # Empty content is not allowed. + if self.use_vertexai: + rst.append(VertexAIPart.from_text(message["content"])) + else: + rst.append(Part(text=message["content"])) + return rst + + if "tool_calls" in message: + if self.use_vertexai: + for tool_call in message["tool_calls"]: + rst.append( + VertexAIPart.from_dict( + { + "functionCall": { + "name": tool_call["function"]["name"], + "args": json.loads(tool_call["function"]["arguments"]), + } + } + ) + ) + else: + for tool_call in message["tool_calls"]: + rst.append( + Part( + function_call=FunctionCall( + name=tool_call["function"]["name"], + args=json.loads(tool_call["function"]["arguments"]), + ) + ) + ) + return rst + + if message["role"] == "tool": + if self.use_vertexai: + rst.append( + VertexAIPart.from_function_response( + name=message["name"], response={"result": self._to_json(message["content"])} + ) + ) + else: + rst.append( + Part( + function_response=FunctionResponse( + name=message["name"], response={"result": self._to_json(message["content"])} + ) + ) + ) + return rst + + if isinstance(message["content"], str): if self.use_vertexai: - rst.append(VertexAIPart.from_text(content)) + rst.append(VertexAIPart.from_text(message["content"])) else: - rst.append(Part(text=content)) + rst.append(Part(text=message["content"])) return rst - assert isinstance(content, list) + assert isinstance(message["content"], list) - for msg in content: + for msg in message["content"]: if isinstance(msg, dict): assert "type" in msg, f"Missing 'type' field in message: {msg}" if msg["type"] == "text": if self.use_vertexai: - rst.append(VertexAIPart.from_text(text=msg["text"])) + rst.append(VertexAIPart.from_text(msg["text"])) else: rst.append(Part(text=msg["text"])) elif msg["type"] == "image_url": @@ -340,34 +418,32 @@ def _oai_content_to_gemini_content(self, content: Union[str, List]) -> List: raise ValueError(f"Unsupported message type: {type(msg)}") return rst - def _concat_parts(self, parts: List[Part]) -> List: - """Concatenate parts with the same type. - If two adjacent parts both have the "text" attribute, then it will be joined into one part. - """ - if not parts: - return [] - - concatenated_parts = [] - previous_part = parts[0] - - for current_part in parts[1:]: - if previous_part.text != "": - if self.use_vertexai: - previous_part = VertexAIPart.from_text(previous_part.text + current_part.text) - else: - previous_part.text += current_part.text - else: - concatenated_parts.append(previous_part) - previous_part = current_part - - if previous_part.text == "": - if self.use_vertexai: - previous_part = VertexAIPart.from_text("empty") - else: - previous_part.text = "empty" # Empty content is not allowed. - concatenated_parts.append(previous_part) + def _calculate_gemini_cost(self, input_tokens: int, output_tokens: int, model_name: str) -> float: + if "1.5-pro" in model_name: + if (input_tokens + output_tokens) <= 128000: + # "gemini-1.5-pro" + # When total tokens is less than 128K cost is $3.5 per million input tokens and $10.5 per million output tokens + return 3.5 * input_tokens / 1e6 + 10.5 * output_tokens / 1e6 + # "gemini-1.5-pro" + # Cost is $7 per million input tokens and $21 per million output tokens + return 7.0 * input_tokens / 1e6 + 21.0 * output_tokens / 1e6 + + if "1.5-flash" in model_name: + if (input_tokens + output_tokens) <= 128000: + # "gemini-1.5-flash" + # Cost is $0.35 per million input tokens and $1.05 per million output tokens + return 0.35 * input_tokens / 1e6 + 1.05 * output_tokens / 1e6 + # "gemini-1.5-flash" + # When total tokens is less than 128K cost is $0.70 per million input tokens and $2.10 per million output tokens + return 0.70 * input_tokens / 1e6 + 2.10 * output_tokens / 1e6 + + if "gemini-pro" not in model_name and "gemini-1.0-pro" not in model_name: + warnings.warn( + f"Cost calculation is not implemented for model {model_name}. Using Gemini-1.0-Pro.", UserWarning + ) - return concatenated_parts + # Cost is $0.5 per million input tokens and $1.5 per million output tokens + return 0.5 * input_tokens / 1e6 + 1.5 * output_tokens / 1e6 def _oai_messages_to_gemini_messages(self, messages: list[Dict[str, Any]]) -> list[dict[str, Any]]: """Convert messages from OAI format to Gemini format. @@ -376,38 +452,154 @@ def _oai_messages_to_gemini_messages(self, messages: list[Dict[str, Any]]) -> li """ prev_role = None rst = [] - curr_parts = [] + + def append_parts(parts, role): + if self.use_vertexai: + rst.append(VertexAIContent(parts=parts, role=role)) + else: + rst.append(Content(parts=parts, role=role)) + + def append_text_to_last(text): + if self.use_vertexai: + rst[-1] = VertexAIContent(parts=[*rst[-1].parts, VertexAIPart.from_text(text)], role=rst[-1].role) + else: + rst[-1] = Content(parts=[*rst[-1].parts, Part(text=text)], role=rst[-1].role) + + def is_function_call(parts): + return self.use_vertexai and parts[0].function_call or not self.use_vertexai and "function_call" in parts[0] + for i, message in enumerate(messages): - parts = self._oai_content_to_gemini_content(message["content"]) + + # Since the tool call message does not have the "name" field, we need to find the corresponding tool message. + if message["role"] == "tool": + message["name"] = [ + m["tool_calls"][i]["function"]["name"] + for m in messages + if "tool_calls" in m + for i, tc in enumerate(m["tool_calls"]) + if tc["id"] == message["tool_call_id"] + ][0] + + parts = self._oai_content_to_gemini_content(message) role = "user" if message["role"] in ["user", "system"] else "model" - if (prev_role is None) or (role == prev_role): - curr_parts += parts - elif role != prev_role: - if self.use_vertexai: - rst.append(VertexAIContent(parts=curr_parts, role=prev_role)) - else: - rst.append(Content(parts=curr_parts, role=prev_role)) - curr_parts = parts - prev_role = role - # handle the last message - if self.use_vertexai: - rst.append(VertexAIContent(parts=curr_parts, role=role)) - else: - rst.append(Content(parts=curr_parts, role=role)) + # In Gemini if the current message is a function call then previous message should not be a model message. + if is_function_call(parts): + # If the previous message is a model message then add a dummy "continue" user message before the function call + if prev_role == "model": + append_parts(self._oai_content_to_gemini_content({"content": "continue"}), "user") + append_parts(parts, role) + # In Gemini if the current message is a function response then next message should be a model message. + elif role == "function": + append_parts(parts, "function") + # If the next message is not a model message then add a dummy "continue" model message after the function response + if len(messages) > (i + 1) and messages[i + 1]["role"] in ["user", "system"]: + append_parts(self._oai_content_to_gemini_content({"content": "continue"}), "model") + # If the role is the same as the previous role and both are text messages then concatenate the text + elif role == prev_role: + append_text_to_last(parts[0].text) + # If this is first message or the role is different from the previous role then append the parts + else: + # If the previous text message is empty then update the text to "empty" as Gemini does not support empty messages + if ( + (len(rst) > 0) + and hasattr(rst[-1].parts[0], "_raw_part") + and hasattr(rst[-1].parts[0]._raw_part, "text") + and (rst[-1].parts[0]._raw_part.text == "") + ): + append_text_to_last("empty") + append_parts(parts, role) + + prev_role = role # The Gemini is restrict on order of roles, such that # 1. The messages should be interleaved between user and model. # 2. The last message must be from the user role. # We add a dummy message "continue" if the last role is not the user. - if rst[-1].role != "user": + if rst[-1].role != "user" and rst[-1].role != "function": if self.use_vertexai: - rst.append(VertexAIContent(parts=self._oai_content_to_gemini_content("continue"), role="user")) + rst.append( + VertexAIContent(parts=self._oai_content_to_gemini_content({"content": "continue"}), role="user") + ) else: - rst.append(Content(parts=self._oai_content_to_gemini_content("continue"), role="user")) - + rst.append(Content(parts=self._oai_content_to_gemini_content({"content": "continue"}), role="user")) return rst + def _oai_tools_to_gemini_tools(self, tools: List[Dict[str, Any]]) -> List[Tool]: + """Convert tools from OAI format to Gemini format.""" + if len(tools) == 0: + return None + function_declarations = [] + for tool in tools: + if self.use_vertexai: + function_declaration = VertexAIFunctionDeclaration( + name=tool["function"]["name"], + description=tool["function"]["description"], + parameters=tool["function"]["parameters"], + ) + else: + function_declaration = FunctionDeclaration( + name=tool["function"]["name"], + description=tool["function"]["description"], + parameters=self._oai_function_parameters_to_gemini_function_parameters( + copy.deepcopy(tool["function"]["parameters"]) + ), + ) + function_declarations.append(function_declaration) + if self.use_vertexai: + return [VertexAITool(function_declarations=function_declarations)] + else: + return [Tool(function_declarations=function_declarations)] + + def _oai_function_parameters_to_gemini_function_parameters( + self, function_definition: dict[str, any] + ) -> dict[str, any]: + """ + Convert OpenAPI function definition parameters to Gemini function parameters definition. + The type key is renamed to type_ and the value is capitalized. + """ + assert "anyOf" not in function_definition, "Union types are not supported for function parameter in Gemini." + # Delete the default key as it is not supported in Gemini + if "default" in function_definition: + del function_definition["default"] + + function_definition["type_"] = function_definition["type"].upper() + del function_definition["type"] + if "properties" in function_definition: + for key in function_definition["properties"]: + function_definition["properties"][key] = self._oai_function_parameters_to_gemini_function_parameters( + function_definition["properties"][key] + ) + if "items" in function_definition: + function_definition["items"] = self._oai_function_parameters_to_gemini_function_parameters( + function_definition["items"] + ) + return function_definition + + def _gemini_content_to_oai_choices(self, response: Union[Content, VertexAIContent]) -> List[Choice]: + """Convert response from Gemini format to OAI format.""" + text = None + tool_calls = [] + for part in response.parts: + if part.function_call: + if self.use_vertexai: + arguments = VertexAIPart.to_dict(part)["function_call"]["args"] + else: + arguments = Part.to_dict(part)["function_call"]["args"] + tool_calls.append( + ChatCompletionMessageToolCall( + id=str(random.randint(0, 1000)), + type="function", + function=Function(name=part.function_call.name, arguments=json.dumps(arguments)), + ) + ) + elif part.text: + text = part.text + message = ChatCompletionMessage( + role="assistant", content=text, function_call=None, tool_calls=tool_calls if len(tool_calls) > 0 else None + ) + return [Choice(finish_reason="tool_calls" if tool_calls else "stop", index=0, message=message)] + @staticmethod def _to_vertexai_safety_settings(safety_settings): """Convert safety settings to VertexAI format if needed, @@ -437,6 +629,49 @@ def _to_vertexai_safety_settings(safety_settings): else: return safety_settings + @staticmethod + def _to_vertexai_tool_config(tool_config, tools): + """Convert tool config to VertexAI format, + like when specifying them in the OAI_CONFIG_LIST + """ + if ( + isinstance(tool_config, dict) + and (len(tool_config) > 0) + and all([isinstance(tool_config[tool_config_entry], dict) for tool_config_entry in tool_config]) + ): + if ( + tool_config["function_calling_config"]["mode"] + not in VertexAIToolConfig.FunctionCallingConfig.Mode.__members__ + ): + invalid_mode = tool_config["function_calling_config"] + logger.error(f"Function calling mode {invalid_mode} is invalid") + return None + else: + # Currently, there is only function calling config + func_calling_config_params = {} + func_calling_config_params["mode"] = VertexAIToolConfig.FunctionCallingConfig.Mode[ + tool_config["function_calling_config"]["mode"] + ] + if ( + (func_calling_config_params["mode"] == VertexAIToolConfig.FunctionCallingConfig.Mode.ANY) + and (len(tools) > 0) + and all(["function_name" in tool for tool in tools]) + ): + # The function names are not yet known when parsing the OAI_CONFIG_LIST + func_calling_config_params["allowed_function_names"] = [tool["function_name"] for tool in tools] + vertexai_tool_config = VertexAIToolConfig( + function_calling_config=VertexAIToolConfig.FunctionCallingConfig(**func_calling_config_params) + ) + return vertexai_tool_config + elif isinstance(tool_config, VertexAIToolConfig): + return tool_config + elif len(tool_config) == 0 and len(tools) == 0: + logger.debug("VertexAI tool config is empty!") + return None + else: + logger.error("Invalid VertexAI tool config!") + return None + def _to_pil(data: str) -> Image.Image: """ @@ -470,16 +705,3 @@ def get_image_data(image_file: str, use_b64=True) -> bytes: return base64.b64encode(content).decode("utf-8") else: return content - - -def calculate_gemini_cost(input_tokens: int, output_tokens: int, model_name: str) -> float: - if "1.5" in model_name or "gemini-experimental" in model_name: - # "gemini-1.5-pro-preview-0409" - # Cost is $7 per million input tokens and $21 per million output tokens - return 7.0 * input_tokens / 1e6 + 21.0 * output_tokens / 1e6 - - if "gemini-pro" not in model_name and "gemini-1.0-pro" not in model_name: - warnings.warn(f"Cost calculation is not implemented for model {model_name}. Using Gemini-1.0-Pro.", UserWarning) - - # Cost is $0.5 per million input tokens and $1.5 per million output tokens - return 0.5 * input_tokens / 1e6 + 1.5 * output_tokens / 1e6 diff --git a/autogen/oai/ollama.py b/autogen/oai/ollama.py new file mode 100644 index 000000000000..636d152ae85c --- /dev/null +++ b/autogen/oai/ollama.py @@ -0,0 +1,579 @@ +"""Create an OpenAI-compatible client using Ollama's API. + +Example: + llm_config={ + "config_list": [{ + "api_type": "ollama", + "model": "mistral:7b-instruct-v0.3-q6_K" + } + ]} + + agent = autogen.AssistantAgent("my_agent", llm_config=llm_config) + +Install Ollama's python library using: pip install --upgrade ollama + +Resources: +- https://github.com/ollama/ollama-python +""" + +from __future__ import annotations + +import copy +import json +import random +import re +import time +import warnings +from typing import Any, Dict, List, Tuple + +import ollama +from fix_busted_json import repair_json +from ollama import Client +from openai.types.chat import ChatCompletion, ChatCompletionMessageToolCall +from openai.types.chat.chat_completion import ChatCompletionMessage, Choice +from openai.types.completion_usage import CompletionUsage + +from autogen.oai.client_utils import should_hide_tools, validate_parameter + + +class OllamaClient: + """Client for Ollama's API.""" + + # Defaults for manual tool calling + # Instruction is added to the first system message and provides directions to follow a two step + # process + # 1. (before tools have been called) Return JSON with the functions to call + # 2. (directly after tools have been called) Return Text describing the results of the function calls in text format + + # Override using "manual_tool_call_instruction" config parameter + TOOL_CALL_MANUAL_INSTRUCTION = ( + "You are to follow a strict two step process that will occur over " + "a number of interactions, so pay attention to what step you are in based on the full " + "conversation. We will be taking turns so only do one step at a time so don't perform step " + "2 until step 1 is complete and I've told you the result. The first step is to choose one " + "or more functions based on the request given and return only JSON with the functions and " + "arguments to use. The second step is to analyse the given output of the function and summarise " + "it returning only TEXT and not Python or JSON. " + "For argument values, be sure numbers aren't strings, they should not have double quotes around them. " + "In terms of your response format, for step 1 return only JSON and NO OTHER text, " + "for step 2 return only text and NO JSON/Python/Markdown. " + 'The format for running a function is [{"name": "function_name1", "arguments":{"argument_name": "argument_value"}},{"name": "function_name2", "arguments":{"argument_name": "argument_value"}}] ' + 'Make sure the keys "name" and "arguments" are as described. ' + "If you don't get the format correct, try again. " + "The following functions are available to you:[FUNCTIONS_LIST]" + ) + + # Appended to the last user message if no tools have been called + # Override using "manual_tool_call_step1" config parameter + TOOL_CALL_MANUAL_STEP1 = " (proceed with step 1)" + + # Appended to the user message after tools have been executed. Will create a 'user' message if one doesn't exist. + # Override using "manual_tool_call_step2" config parameter + TOOL_CALL_MANUAL_STEP2 = " (proceed with step 2)" + + def __init__(self, **kwargs): + """Note that no api_key or environment variable is required for Ollama. + + Args: + None + """ + + def message_retrieval(self, response) -> List: + """ + Retrieve and return a list of strings or a list of Choice.Message from the response. + + NOTE: if a list of Choice.Message is returned, it currently needs to contain the fields of OpenAI's ChatCompletion Message object, + since that is expected for function or tool calling in the rest of the codebase at the moment, unless a custom agent is being used. + """ + return [choice.message for choice in response.choices] + + def cost(self, response) -> float: + return response.cost + + @staticmethod + def get_usage(response) -> Dict: + """Return usage summary of the response using RESPONSE_USAGE_KEYS.""" + # ... # pragma: no cover + return { + "prompt_tokens": response.usage.prompt_tokens, + "completion_tokens": response.usage.completion_tokens, + "total_tokens": response.usage.total_tokens, + "cost": response.cost, + "model": response.model, + } + + def parse_params(self, params: Dict[str, Any]) -> Dict[str, Any]: + """Loads the parameters for Ollama API from the passed in parameters and returns a validated set. Checks types, ranges, and sets defaults""" + ollama_params = {} + + # Check that we have what we need to use Ollama's API + # https://github.com/ollama/ollama/blob/main/docs/api.md#generate-a-completion + + # The main parameters are model, prompt, stream, and options + # Options is a dictionary of parameters for the model + # There are other, advanced, parameters such as format, system (to override system message), template, raw, etc. - not used + + # We won't enforce the available models + ollama_params["model"] = params.get("model", None) + assert ollama_params[ + "model" + ], "Please specify the 'model' in your config list entry to nominate the Ollama model to use." + + ollama_params["stream"] = validate_parameter(params, "stream", bool, True, False, None, None) + + # Build up the options dictionary + # https://github.com/ollama/ollama/blob/main/docs/modelfile.md#valid-parameters-and-values + options_dict = {} + + if "num_predict" in params: + # Maximum number of tokens to predict, note: -1 is infinite, -2 is fill context, 128 is default + options_dict["num_predict"] = validate_parameter(params, "num_predict", int, False, 128, None, None) + + if "repeat_penalty" in params: + options_dict["repeat_penalty"] = validate_parameter( + params, "repeat_penalty", (int, float), False, 1.1, None, None + ) + + if "seed" in params: + options_dict["seed"] = validate_parameter(params, "seed", int, False, 42, None, None) + + if "temperature" in params: + options_dict["temperature"] = validate_parameter( + params, "temperature", (int, float), False, 0.8, None, None + ) + + if "top_k" in params: + options_dict["top_k"] = validate_parameter(params, "top_k", int, False, 40, None, None) + + if "top_p" in params: + options_dict["top_p"] = validate_parameter(params, "top_p", (int, float), False, 0.9, None, None) + + if self._native_tool_calls and self._tools_in_conversation and not self._should_hide_tools: + ollama_params["tools"] = params["tools"] + + # Ollama doesn't support streaming with tools natively + if ollama_params["stream"] and self._native_tool_calls: + warnings.warn( + "Streaming is not supported when using tools and 'Native' tool calling, streaming will be disabled.", + UserWarning, + ) + + ollama_params["stream"] = False + + if not self._native_tool_calls and self._tools_in_conversation: + # For manual tool calling we have injected the available tools into the prompt + # and we don't want to force JSON mode + ollama_params["format"] = "" # Don't force JSON for manual tool calling mode + + if len(options_dict) != 0: + ollama_params["options"] = options_dict + + return ollama_params + + def create(self, params: Dict) -> ChatCompletion: + + messages = params.get("messages", []) + + # Are tools involved in this conversation? + self._tools_in_conversation = "tools" in params + + # We provide second-level filtering out of tools to avoid LLMs re-calling tools continuously + if self._tools_in_conversation: + hide_tools = validate_parameter( + params, "hide_tools", str, False, "never", None, ["if_all_run", "if_any_run", "never"] + ) + self._should_hide_tools = should_hide_tools(messages, params["tools"], hide_tools) + else: + self._should_hide_tools = False + + # Are we using native Ollama tool calling, otherwise we're doing manual tool calling + # We allow the user to decide if they want to use Ollama's tool calling + # or for tool calling to be handled manually through text messages + # Default is True = Ollama's tool calling + self._native_tool_calls = validate_parameter(params, "native_tool_calls", bool, False, True, None, None) + + if not self._native_tool_calls: + # Load defaults + self._manual_tool_call_instruction = validate_parameter( + params, "manual_tool_call_instruction", str, False, self.TOOL_CALL_MANUAL_INSTRUCTION, None, None + ) + self._manual_tool_call_step1 = validate_parameter( + params, "manual_tool_call_step1", str, False, self.TOOL_CALL_MANUAL_STEP1, None, None + ) + self._manual_tool_call_step2 = validate_parameter( + params, "manual_tool_call_step2", str, False, self.TOOL_CALL_MANUAL_STEP2, None, None + ) + + # Convert AutoGen messages to Ollama messages + ollama_messages = self.oai_messages_to_ollama_messages( + messages, + ( + params["tools"] + if (not self._native_tool_calls and self._tools_in_conversation) and not self._should_hide_tools + else None + ), + ) + + # Parse parameters to the Ollama API's parameters + ollama_params = self.parse_params(params) + + ollama_params["messages"] = ollama_messages + + # Token counts will be returned + prompt_tokens = 0 + completion_tokens = 0 + total_tokens = 0 + + ans = None + try: + if "client_host" in params: + client = Client(host=params["client_host"]) + response = client.chat(**ollama_params) + else: + response = ollama.chat(**ollama_params) + except Exception as e: + raise RuntimeError(f"Ollama exception occurred: {e}") + else: + + if ollama_params["stream"]: + # Read in the chunks as they stream, taking in tool_calls which may be across + # multiple chunks if more than one suggested + ans = "" + for chunk in response: + ans = ans + (chunk["message"]["content"] or "") + + if "done_reason" in chunk: + prompt_tokens = chunk["prompt_eval_count"] if "prompt_eval_count" in chunk else 0 + completion_tokens = chunk["eval_count"] if "eval_count" in chunk else 0 + total_tokens = prompt_tokens + completion_tokens + else: + # Non-streaming finished + ans: str = response["message"]["content"] + + prompt_tokens = response["prompt_eval_count"] if "prompt_eval_count" in response else 0 + completion_tokens = response["eval_count"] if "eval_count" in response else 0 + total_tokens = prompt_tokens + completion_tokens + + if response is not None: + + # Defaults + ollama_finish = "stop" + tool_calls = None + + # Id and streaming text into response + if ollama_params["stream"]: + response_content = ans + response_id = chunk["created_at"] + else: + response_content = response["message"]["content"] + response_id = response["created_at"] + + # Process tools in the response + if self._tools_in_conversation: + + if self._native_tool_calls: + + if not ollama_params["stream"]: + response_content = response["message"]["content"] + + # Native tool calling + if "tool_calls" in response["message"]: + ollama_finish = "tool_calls" + tool_calls = [] + random_id = random.randint(0, 10000) + for tool_call in response["message"]["tool_calls"]: + tool_calls.append( + ChatCompletionMessageToolCall( + id="ollama_func_{}".format(random_id), + function={ + "name": tool_call["function"]["name"], + "arguments": json.dumps(tool_call["function"]["arguments"]), + }, + type="function", + ) + ) + + random_id += 1 + + elif not self._native_tool_calls: + + # Try to convert the response to a tool call object + response_toolcalls = response_to_tool_call(ans) + + # If we can, then we've got tool call(s) + if response_toolcalls is not None: + ollama_finish = "tool_calls" + tool_calls = [] + random_id = random.randint(0, 10000) + + for json_function in response_toolcalls: + tool_calls.append( + ChatCompletionMessageToolCall( + id="ollama_manual_func_{}".format(random_id), + function={ + "name": json_function["name"], + "arguments": ( + json.dumps(json_function["arguments"]) + if "arguments" in json_function + else "{}" + ), + }, + type="function", + ) + ) + + random_id += 1 + + # Blank the message content + response_content = "" + + else: + raise RuntimeError("Failed to get response from Ollama.") + + # Convert response to AutoGen response + message = ChatCompletionMessage( + role="assistant", + content=response_content, + function_call=None, + tool_calls=tool_calls, + ) + choices = [Choice(finish_reason=ollama_finish, index=0, message=message)] + + response_oai = ChatCompletion( + id=response_id, + model=ollama_params["model"], + created=int(time.time()), + object="chat.completion", + choices=choices, + usage=CompletionUsage( + prompt_tokens=prompt_tokens, + completion_tokens=completion_tokens, + total_tokens=total_tokens, + ), + cost=0, # Local models, FREE! + ) + + return response_oai + + def oai_messages_to_ollama_messages(self, messages: list[Dict[str, Any]], tools: list) -> list[dict[str, Any]]: + """Convert messages from OAI format to Ollama's format. + We correct for any specific role orders and types, and convert tools to messages (as Ollama can't use tool messages) + """ + + ollama_messages = copy.deepcopy(messages) + + # Remove the name field + for message in ollama_messages: + if "name" in message: + message.pop("name", None) + + # Having a 'system' message on the end does not work well with Ollama, so we change it to 'user' + # 'system' messages on the end are typical of the summarisation message: summary_method="reflection_with_llm" + if len(ollama_messages) > 1 and ollama_messages[-1]["role"] == "system": + ollama_messages[-1]["role"] = "user" + + # Process messages for tool calling manually + if tools is not None and not self._native_tool_calls: + # 1. We need to append instructions to the starting system message on function calling + # 2. If we have not yet called tools we append "step 1 instruction" to the latest user message + # 3. If we have already called tools we append "step 2 instruction" to the latest user message + + have_tool_calls = False + have_tool_results = False + last_tool_result_index = -1 + + for i, message in enumerate(ollama_messages): + if "tool_calls" in message: + have_tool_calls = True + if "tool_call_id" in message: + have_tool_results = True + last_tool_result_index = i + + tool_result_is_last_msg = have_tool_results and last_tool_result_index == len(ollama_messages) - 1 + + if ollama_messages[0]["role"] == "system": + manual_instruction = self._manual_tool_call_instruction + + # Build a string of the functions available + functions_string = "" + for function in tools: + functions_string += f"""\n{function}\n""" + + # Replace single quotes with double questions - Not sure why this helps the LLM perform + # better, but it seems to. Monitor and remove if not necessary. + functions_string = functions_string.replace("'", '"') + + manual_instruction = manual_instruction.replace("[FUNCTIONS_LIST]", functions_string) + + # Update the system message with the instructions and functions + ollama_messages[0]["content"] = ollama_messages[0]["content"] + manual_instruction.rstrip() + + # If we are still in the function calling or evaluating process, append the steps instruction + if not have_tool_calls or tool_result_is_last_msg: + if ollama_messages[0]["role"] == "system": + # NOTE: we require a system message to exist for the manual steps texts + # Append the manual step instructions + content_to_append = ( + self._manual_tool_call_step1 if not have_tool_results else self._manual_tool_call_step2 + ) + + if content_to_append != "": + # Append the relevant tool call instruction to the latest user message + if ollama_messages[-1]["role"] == "user": + ollama_messages[-1]["content"] = ollama_messages[-1]["content"] + content_to_append + else: + ollama_messages.append({"role": "user", "content": content_to_append}) + + # Convert tool call and tool result messages to normal text messages for Ollama + for i, message in enumerate(ollama_messages): + if "tool_calls" in message: + # Recommended tool calls + content = "Run the following function(s):" + for tool_call in message["tool_calls"]: + content = content + "\n" + str(tool_call) + ollama_messages[i] = {"role": "assistant", "content": content} + if "tool_call_id" in message: + # Executed tool results + message["result"] = message["content"] + del message["content"] + del message["role"] + content = "The following function was run: " + str(message) + ollama_messages[i] = {"role": "user", "content": content} + + # As we are changing messages, let's merge if they have two user messages on the end and the last one is tool call step instructions + if ( + len(ollama_messages) >= 2 + and not self._native_tool_calls + and ollama_messages[-2]["role"] == "user" + and ollama_messages[-1]["role"] == "user" + and ( + ollama_messages[-1]["content"] == self._manual_tool_call_step1 + or ollama_messages[-1]["content"] == self._manual_tool_call_step2 + ) + ): + ollama_messages[-2]["content"] = ollama_messages[-2]["content"] + ollama_messages[-1]["content"] + del ollama_messages[-1] + + # Ensure the last message is a user / system message, if not, add a user message + if ollama_messages[-1]["role"] != "user" and ollama_messages[-1]["role"] != "system": + ollama_messages.append({"role": "user", "content": "Please continue."}) + + return ollama_messages + + +def response_to_tool_call(response_string: str) -> Any: + """Attempts to convert the response to an object, aimed to align with function format [{},{}]""" + + # We try and detect the list[dict] format: + # Pattern 1 is [{},{}] + # Pattern 2 is {} (without the [], so could be a single function call) + patterns = [r"\[[\s\S]*?\]", r"\{[\s\S]*\}"] + + for i, pattern in enumerate(patterns): + # Search for the pattern in the input string + matches = re.findall(pattern, response_string.strip()) + + for match in matches: + + # It has matched, extract it and load it + json_str = match.strip() + data_object = None + + try: + # Attempt to convert it as is + data_object = json.loads(json_str) + except Exception: + try: + # If that fails, attempt to repair it + + if i == 0: + # Enclose to a JSON object for repairing, which is restored upon fix + fixed_json = repair_json("{'temp':" + json_str + "}") + data_object = json.loads(fixed_json) + data_object = data_object["temp"] + else: + fixed_json = repair_json(json_str) + data_object = json.loads(fixed_json) + except json.JSONDecodeError as e: + if e.msg == "Invalid \\escape": + # Handle Mistral/Mixtral trying to escape underlines with \\ + try: + json_str = json_str.replace("\\_", "_") + if i == 0: + fixed_json = repair_json("{'temp':" + json_str + "}") + data_object = json.loads(fixed_json) + data_object = data_object["temp"] + else: + fixed_json = repair_json("{'temp':" + json_str + "}") + data_object = json.loads(fixed_json) + except Exception: + pass + except Exception: + pass + + if data_object is not None: + data_object = _object_to_tool_call(data_object) + + if data_object is not None: + return data_object + + # There's no tool call in the response + return None + + +def _object_to_tool_call(data_object: Any) -> List[Dict]: + """Attempts to convert an object to a valid tool call object List[Dict] and returns it, if it can, otherwise None""" + + # If it's a dictionary and not a list then wrap in a list + if isinstance(data_object, dict): + data_object = [data_object] + + # Validate that the data is a list of dictionaries + if isinstance(data_object, list) and all(isinstance(item, dict) for item in data_object): + # Perfect format, a list of dictionaries + + # Check that each dictionary has at least 'name', optionally 'arguments' and no other keys + is_invalid = False + for item in data_object: + if not is_valid_tool_call_item(item): + is_invalid = True + break + + # All passed, name and (optionally) arguments exist for all entries. + if not is_invalid: + return data_object + elif isinstance(data_object, list): + # If it's a list but the items are not dictionaries, check if they are strings that can be converted to dictionaries + data_copy = data_object.copy() + is_invalid = False + for i, item in enumerate(data_copy): + try: + new_item = eval(item) + if isinstance(new_item, dict): + if is_valid_tool_call_item(new_item): + data_object[i] = new_item + else: + is_invalid = True + break + else: + is_invalid = True + break + except Exception: + is_invalid = True + break + + if not is_invalid: + return data_object + + return None + + +def is_valid_tool_call_item(call_item: dict) -> bool: + """Check that a dictionary item has at least 'name', optionally 'arguments' and no other keys to match a tool call JSON""" + if "name" not in call_item or not isinstance(call_item["name"], str): + return False + + if set(call_item.keys()) - {"name", "arguments"}: + return False + + return True diff --git a/autogen/oai/openai_utils.py b/autogen/oai/openai_utils.py index 3844795c24f5..ceb7ef90c933 100644 --- a/autogen/oai/openai_utils.py +++ b/autogen/oai/openai_utils.py @@ -21,6 +21,7 @@ "azure_ad_token", "azure_ad_token_provider", "credentials", + "tool_config", ] DEFAULT_AZURE_API_VERSION = "2024-02-01" OAI_PRICE1K = { diff --git a/autogen/runtime_logging.py b/autogen/runtime_logging.py index 0fd7cc2fc8b9..0a9b1687b75c 100644 --- a/autogen/runtime_logging.py +++ b/autogen/runtime_logging.py @@ -15,10 +15,12 @@ from autogen import Agent, ConversableAgent, OpenAIWrapper from autogen.oai.anthropic import AnthropicClient from autogen.oai.bedrock import BedrockClient + from autogen.oai.cerebras import CerebrasClient from autogen.oai.cohere import CohereClient from autogen.oai.gemini import GeminiClient from autogen.oai.groq import GroqClient from autogen.oai.mistral import MistralAIClient + from autogen.oai.ollama import OllamaClient from autogen.oai.together import TogetherClient logger = logging.getLogger(__name__) @@ -116,12 +118,14 @@ def log_new_client( client: Union[ AzureOpenAI, OpenAI, + CerebrasClient, GeminiClient, AnthropicClient, MistralAIClient, TogetherClient, GroqClient, CohereClient, + OllamaClient, BedrockClient, ], wrapper: OpenAIWrapper, diff --git a/autogen/version.py b/autogen/version.py index 9b1b78b4b3a0..c971add65283 100644 --- a/autogen/version.py +++ b/autogen/version.py @@ -1 +1 @@ -__version__ = "0.2.35" +__version__ = "0.2.36" diff --git a/dotnet/nuget/NUGET.md b/dotnet/nuget/NUGET.md index 34fdbca33ca7..cfa7c9801888 100644 --- a/dotnet/nuget/NUGET.md +++ b/dotnet/nuget/NUGET.md @@ -2,7 +2,6 @@ `AutoGen for .NET` is the official .NET SDK for [AutoGen](https://github.com/microsoft/autogen). It enables you to create LLM agents and construct multi-agent workflows with ease. It also provides integration with popular platforms like OpenAI, Semantic Kernel, and LM Studio. ### Gettings started -- Find documents and examples on our [document site](https://microsoft.github.io/autogen-for-net/) -- Join our [Discord channel](https://discord.gg/pAbnFJrkgZ) to get help and discuss with the community +- Find documents and examples on our [document site](https://microsoft.github.io/autogen-for-net/) - Report a bug or request a feature by creating a new issue in our [github repo](https://github.com/microsoft/autogen) - Consume the nightly build package from one of the [nightly build feeds](https://microsoft.github.io/autogen-for-net/articles/Installation.html#nighly-build) \ No newline at end of file diff --git a/notebook/Async_human_input.ipynb b/notebook/Async_human_input.ipynb index 07459b4a86ba..5d4926bf13ca 100644 --- a/notebook/Async_human_input.ipynb +++ b/notebook/Async_human_input.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -10,179 +10,9 @@ "id": "tLIs1YRdr8jM", "outputId": "909c1c70-1a22-4e9d-b7f4-a40e2d737fb0" }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Defaulting to user installation because normal site-packages is not writeable\n", - "Requirement already satisfied: pyautogen>=0.2.3 in /home/vscode/.local/lib/python3.10/site-packages (0.2.3)\n", - "Requirement already satisfied: openai>=1.3 in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (1.6.1)\n", - "Requirement already satisfied: diskcache in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (5.6.3)\n", - "Requirement already satisfied: termcolor in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (2.4.0)\n", - "Requirement already satisfied: flaml in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (2.1.1)\n", - "Requirement already satisfied: python-dotenv in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (1.0.0)\n", - "Requirement already satisfied: tiktoken in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (0.5.2)\n", - "Requirement already satisfied: pydantic<3,>=1.10 in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (1.10.9)\n", - "Requirement already satisfied: anyio<5,>=3.5.0 in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (4.2.0)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (1.9.0)\n", - "Requirement already satisfied: httpx<1,>=0.23.0 in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (0.26.0)\n", - "Requirement already satisfied: sniffio in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (1.3.0)\n", - "Requirement already satisfied: tqdm>4 in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (4.66.1)\n", - "Requirement already satisfied: typing-extensions<5,>=4.7 in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (4.9.0)\n", - "Requirement already satisfied: NumPy>=1.17.0rc1 in /home/vscode/.local/lib/python3.10/site-packages (from flaml->pyautogen>=0.2.3) (1.26.3)\n", - "Requirement already satisfied: regex>=2022.1.18 in /home/vscode/.local/lib/python3.10/site-packages (from tiktoken->pyautogen>=0.2.3) (2023.12.25)\n", - "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/site-packages (from tiktoken->pyautogen>=0.2.3) (2.31.0)\n", - "Requirement already satisfied: idna>=2.8 in /usr/local/lib/python3.10/site-packages (from anyio<5,>=3.5.0->openai>=1.3->pyautogen>=0.2.3) (3.6)\n", - "Requirement already satisfied: exceptiongroup>=1.0.2 in /home/vscode/.local/lib/python3.10/site-packages (from anyio<5,>=3.5.0->openai>=1.3->pyautogen>=0.2.3) (1.2.0)\n", - "Requirement already satisfied: certifi in /usr/local/lib/python3.10/site-packages (from httpx<1,>=0.23.0->openai>=1.3->pyautogen>=0.2.3) (2023.11.17)\n", - "Requirement already satisfied: httpcore==1.* in /home/vscode/.local/lib/python3.10/site-packages (from httpx<1,>=0.23.0->openai>=1.3->pyautogen>=0.2.3) (1.0.2)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in /home/vscode/.local/lib/python3.10/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai>=1.3->pyautogen>=0.2.3) (0.14.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/site-packages (from requests>=2.26.0->tiktoken->pyautogen>=0.2.3) (3.3.2)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/vscode/.local/lib/python3.10/site-packages (from requests>=2.26.0->tiktoken->pyautogen>=0.2.3) (1.26.18)\n", - "Defaulting to user installation because normal site-packages is not writeable\n", - "Requirement already satisfied: chromadb in /home/vscode/.local/lib/python3.10/site-packages (0.4.22)\n", - "Requirement already satisfied: build>=1.0.3 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (1.0.3)\n", - "Requirement already satisfied: requests>=2.28 in /usr/local/lib/python3.10/site-packages (from chromadb) (2.31.0)\n", - "Requirement already satisfied: pydantic>=1.9 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (1.10.9)\n", - "Requirement already satisfied: chroma-hnswlib==0.7.3 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (0.7.3)\n", - "Requirement already satisfied: fastapi>=0.95.2 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (0.108.0)\n", - "Requirement already satisfied: uvicorn>=0.18.3 in /home/vscode/.local/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.25.0)\n", - "Requirement already satisfied: numpy>=1.22.5 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (1.26.3)\n", - "Requirement already satisfied: posthog>=2.4.0 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (3.1.0)\n", - "Requirement already satisfied: typing-extensions>=4.5.0 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (4.9.0)\n", - "Requirement already satisfied: pulsar-client>=3.1.0 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (3.4.0)\n", - "Requirement already satisfied: onnxruntime>=1.14.1 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (1.16.3)\n", - "Requirement already satisfied: opentelemetry-api>=1.2.0 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (1.22.0)\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-grpc>=1.2.0 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (1.22.0)\n", - "Requirement already satisfied: opentelemetry-instrumentation-fastapi>=0.41b0 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (0.43b0)\n", - "Requirement already satisfied: opentelemetry-sdk>=1.2.0 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (1.22.0)\n", - "Requirement already satisfied: tokenizers>=0.13.2 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (0.15.0)\n", - "Requirement already satisfied: pypika>=0.48.9 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (0.48.9)\n", - "Requirement already satisfied: tqdm>=4.65.0 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (4.66.1)\n", - "Requirement already satisfied: overrides>=7.3.1 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (7.4.0)\n", - "Requirement already satisfied: importlib-resources in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (6.1.1)\n", - "Requirement already satisfied: grpcio>=1.58.0 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (1.60.0)\n", - "Requirement already satisfied: bcrypt>=4.0.1 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (4.1.2)\n", - "Requirement already satisfied: typer>=0.9.0 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (0.9.0)\n", - "Requirement already satisfied: kubernetes>=28.1.0 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (28.1.0)\n", - "Requirement already satisfied: tenacity>=8.2.3 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (8.2.3)\n", - "Requirement already satisfied: PyYAML>=6.0.0 in /usr/local/lib/python3.10/site-packages (from chromadb) (6.0.1)\n", - "Requirement already satisfied: mmh3>=4.0.1 in /home/vscode/.local/lib/python3.10/site-packages (from chromadb) (4.0.1)\n", - "Requirement already satisfied: packaging>=19.0 in /usr/local/lib/python3.10/site-packages (from build>=1.0.3->chromadb) (23.2)\n", - "Requirement already satisfied: pyproject_hooks in /home/vscode/.local/lib/python3.10/site-packages (from build>=1.0.3->chromadb) (1.0.0)\n", - "Requirement already satisfied: tomli>=1.1.0 in /usr/local/lib/python3.10/site-packages (from build>=1.0.3->chromadb) (2.0.1)\n", - "Requirement already satisfied: starlette<0.33.0,>=0.29.0 in /home/vscode/.local/lib/python3.10/site-packages (from fastapi>=0.95.2->chromadb) (0.32.0.post1)\n", - "Requirement already satisfied: certifi>=14.05.14 in /usr/local/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (2023.11.17)\n", - "Requirement already satisfied: six>=1.9.0 in /home/vscode/.local/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (1.16.0)\n", - "Requirement already satisfied: python-dateutil>=2.5.3 in /home/vscode/.local/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (2.8.2)\n", - "Requirement already satisfied: google-auth>=1.0.1 in /home/vscode/.local/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (2.26.1)\n", - "Requirement already satisfied: websocket-client!=0.40.0,!=0.41.*,!=0.42.*,>=0.32.0 in /home/vscode/.local/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (1.7.0)\n", - "Requirement already satisfied: requests-oauthlib in /home/vscode/.local/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (1.3.1)\n", - "Requirement already satisfied: oauthlib>=3.2.2 in /home/vscode/.local/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (3.2.2)\n", - "Requirement already satisfied: urllib3<2.0,>=1.24.2 in /home/vscode/.local/lib/python3.10/site-packages (from kubernetes>=28.1.0->chromadb) (1.26.18)\n", - "Requirement already satisfied: coloredlogs in /home/vscode/.local/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb) (15.0.1)\n", - "Requirement already satisfied: flatbuffers in /home/vscode/.local/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb) (23.5.26)\n", - "Requirement already satisfied: protobuf in /home/vscode/.local/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb) (4.25.1)\n", - "Requirement already satisfied: sympy in /home/vscode/.local/lib/python3.10/site-packages (from onnxruntime>=1.14.1->chromadb) (1.12)\n", - "Requirement already satisfied: deprecated>=1.2.6 in /usr/local/lib/python3.10/site-packages (from opentelemetry-api>=1.2.0->chromadb) (1.2.14)\n", - "Requirement already satisfied: importlib-metadata<7.0,>=6.0 in /home/vscode/.local/lib/python3.10/site-packages (from opentelemetry-api>=1.2.0->chromadb) (6.11.0)\n", - "Requirement already satisfied: backoff<3.0.0,>=1.10.0 in /home/vscode/.local/lib/python3.10/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (2.2.1)\n", - "Requirement already satisfied: googleapis-common-protos~=1.52 in /home/vscode/.local/lib/python3.10/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.62.0)\n", - "Requirement already satisfied: opentelemetry-exporter-otlp-proto-common==1.22.0 in /home/vscode/.local/lib/python3.10/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.22.0)\n", - "Requirement already satisfied: opentelemetry-proto==1.22.0 in /home/vscode/.local/lib/python3.10/site-packages (from opentelemetry-exporter-otlp-proto-grpc>=1.2.0->chromadb) (1.22.0)\n", - "Requirement already satisfied: opentelemetry-instrumentation-asgi==0.43b0 in /home/vscode/.local/lib/python3.10/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.43b0)\n", - "Requirement already satisfied: opentelemetry-instrumentation==0.43b0 in /home/vscode/.local/lib/python3.10/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.43b0)\n", - "Requirement already satisfied: opentelemetry-semantic-conventions==0.43b0 in /home/vscode/.local/lib/python3.10/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.43b0)\n", - "Requirement already satisfied: opentelemetry-util-http==0.43b0 in /home/vscode/.local/lib/python3.10/site-packages (from opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (0.43b0)\n", - "Requirement already satisfied: setuptools>=16.0 in /usr/local/lib/python3.10/site-packages (from opentelemetry-instrumentation==0.43b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (69.0.2)\n", - "Requirement already satisfied: wrapt<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/site-packages (from opentelemetry-instrumentation==0.43b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (1.16.0)\n", - "Requirement already satisfied: asgiref~=3.0 in /home/vscode/.local/lib/python3.10/site-packages (from opentelemetry-instrumentation-asgi==0.43b0->opentelemetry-instrumentation-fastapi>=0.41b0->chromadb) (3.7.2)\n", - "Requirement already satisfied: monotonic>=1.5 in /home/vscode/.local/lib/python3.10/site-packages (from posthog>=2.4.0->chromadb) (1.6)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/site-packages (from requests>=2.28->chromadb) (3.3.2)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/site-packages (from requests>=2.28->chromadb) (3.6)\n", - "Requirement already satisfied: huggingface_hub<1.0,>=0.16.4 in /home/vscode/.local/lib/python3.10/site-packages (from tokenizers>=0.13.2->chromadb) (0.20.2)\n", - "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/site-packages (from typer>=0.9.0->chromadb) (8.1.7)\n", - "Requirement already satisfied: h11>=0.8 in /home/vscode/.local/lib/python3.10/site-packages (from uvicorn>=0.18.3->uvicorn[standard]>=0.18.3->chromadb) (0.14.0)\n", - "Requirement already satisfied: httptools>=0.5.0 in /home/vscode/.local/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.6.1)\n", - "Requirement already satisfied: python-dotenv>=0.13 in /home/vscode/.local/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (1.0.0)\n", - "Requirement already satisfied: uvloop!=0.15.0,!=0.15.1,>=0.14.0 in /home/vscode/.local/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.19.0)\n", - "Requirement already satisfied: watchfiles>=0.13 in /home/vscode/.local/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (0.21.0)\n", - "Requirement already satisfied: websockets>=10.4 in /home/vscode/.local/lib/python3.10/site-packages (from uvicorn[standard]>=0.18.3->chromadb) (12.0)\n", - "Requirement already satisfied: cachetools<6.0,>=2.0.0 in /home/vscode/.local/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (5.3.2)\n", - "Requirement already satisfied: pyasn1-modules>=0.2.1 in /home/vscode/.local/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.3.0)\n", - "Requirement already satisfied: rsa<5,>=3.1.4 in /home/vscode/.local/lib/python3.10/site-packages (from google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (4.9)\n", - "Requirement already satisfied: filelock in /home/vscode/.local/lib/python3.10/site-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers>=0.13.2->chromadb) (3.13.1)\n", - "Requirement already satisfied: fsspec>=2023.5.0 in /home/vscode/.local/lib/python3.10/site-packages (from huggingface_hub<1.0,>=0.16.4->tokenizers>=0.13.2->chromadb) (2023.12.2)\n", - "Requirement already satisfied: zipp>=0.5 in /usr/local/lib/python3.10/site-packages (from importlib-metadata<7.0,>=6.0->opentelemetry-api>=1.2.0->chromadb) (3.17.0)\n", - "Requirement already satisfied: anyio<5,>=3.4.0 in /home/vscode/.local/lib/python3.10/site-packages (from starlette<0.33.0,>=0.29.0->fastapi>=0.95.2->chromadb) (4.2.0)\n", - "Requirement already satisfied: humanfriendly>=9.1 in /home/vscode/.local/lib/python3.10/site-packages (from coloredlogs->onnxruntime>=1.14.1->chromadb) (10.0)\n", - "Requirement already satisfied: mpmath>=0.19 in /home/vscode/.local/lib/python3.10/site-packages (from sympy->onnxruntime>=1.14.1->chromadb) (1.3.0)\n", - "Requirement already satisfied: sniffio>=1.1 in /home/vscode/.local/lib/python3.10/site-packages (from anyio<5,>=3.4.0->starlette<0.33.0,>=0.29.0->fastapi>=0.95.2->chromadb) (1.3.0)\n", - "Requirement already satisfied: exceptiongroup>=1.0.2 in /home/vscode/.local/lib/python3.10/site-packages (from anyio<5,>=3.4.0->starlette<0.33.0,>=0.29.0->fastapi>=0.95.2->chromadb) (1.2.0)\n", - "Requirement already satisfied: pyasn1<0.6.0,>=0.4.6 in /home/vscode/.local/lib/python3.10/site-packages (from pyasn1-modules>=0.2.1->google-auth>=1.0.1->kubernetes>=28.1.0->chromadb) (0.5.1)\n", - "Defaulting to user installation because normal site-packages is not writeable\n", - "Requirement already satisfied: sentence_transformers in /home/vscode/.local/lib/python3.10/site-packages (2.2.2)\n", - "Requirement already satisfied: transformers<5.0.0,>=4.6.0 in /home/vscode/.local/lib/python3.10/site-packages (from sentence_transformers) (4.36.2)\n", - "Requirement already satisfied: tqdm in /home/vscode/.local/lib/python3.10/site-packages (from sentence_transformers) (4.66.1)\n", - "Requirement already satisfied: torch>=1.6.0 in /home/vscode/.local/lib/python3.10/site-packages (from sentence_transformers) (2.1.2)\n", - "Requirement already satisfied: torchvision in /home/vscode/.local/lib/python3.10/site-packages (from sentence_transformers) (0.16.2)\n", - "Requirement already satisfied: numpy in /home/vscode/.local/lib/python3.10/site-packages (from sentence_transformers) (1.26.3)\n", - "Requirement already satisfied: scikit-learn in /home/vscode/.local/lib/python3.10/site-packages (from sentence_transformers) (1.3.2)\n", - "Requirement already satisfied: scipy in /home/vscode/.local/lib/python3.10/site-packages (from sentence_transformers) (1.11.4)\n", - "Requirement already satisfied: nltk in /home/vscode/.local/lib/python3.10/site-packages (from sentence_transformers) (3.8.1)\n", - "Requirement already satisfied: sentencepiece in /home/vscode/.local/lib/python3.10/site-packages (from sentence_transformers) (0.1.99)\n", - "Requirement already satisfied: huggingface-hub>=0.4.0 in /home/vscode/.local/lib/python3.10/site-packages (from sentence_transformers) (0.20.2)\n", - "Requirement already satisfied: filelock in /home/vscode/.local/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence_transformers) (3.13.1)\n", - "Requirement already satisfied: fsspec>=2023.5.0 in /home/vscode/.local/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence_transformers) (2023.12.2)\n", - "Requirement already satisfied: requests in /usr/local/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence_transformers) (2.31.0)\n", - "Requirement already satisfied: pyyaml>=5.1 in /usr/local/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence_transformers) (6.0.1)\n", - "Requirement already satisfied: typing-extensions>=3.7.4.3 in /home/vscode/.local/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence_transformers) (4.9.0)\n", - "Requirement already satisfied: packaging>=20.9 in /usr/local/lib/python3.10/site-packages (from huggingface-hub>=0.4.0->sentence_transformers) (23.2)\n", - "Requirement already satisfied: sympy in /home/vscode/.local/lib/python3.10/site-packages (from torch>=1.6.0->sentence_transformers) (1.12)\n", - "Requirement already satisfied: networkx in /home/vscode/.local/lib/python3.10/site-packages (from torch>=1.6.0->sentence_transformers) (3.2.1)\n", - "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/site-packages (from torch>=1.6.0->sentence_transformers) (3.1.2)\n", - "Requirement already satisfied: nvidia-cuda-nvrtc-cu12==12.1.105 in /home/vscode/.local/lib/python3.10/site-packages (from torch>=1.6.0->sentence_transformers) (12.1.105)\n", - "Requirement already satisfied: nvidia-cuda-runtime-cu12==12.1.105 in /home/vscode/.local/lib/python3.10/site-packages (from torch>=1.6.0->sentence_transformers) (12.1.105)\n", - "Requirement already satisfied: nvidia-cuda-cupti-cu12==12.1.105 in /home/vscode/.local/lib/python3.10/site-packages (from torch>=1.6.0->sentence_transformers) (12.1.105)\n", - "Requirement already satisfied: nvidia-cudnn-cu12==8.9.2.26 in /home/vscode/.local/lib/python3.10/site-packages (from torch>=1.6.0->sentence_transformers) (8.9.2.26)\n", - "Requirement already satisfied: nvidia-cublas-cu12==12.1.3.1 in /home/vscode/.local/lib/python3.10/site-packages (from torch>=1.6.0->sentence_transformers) (12.1.3.1)\n", - "Requirement already satisfied: nvidia-cufft-cu12==11.0.2.54 in /home/vscode/.local/lib/python3.10/site-packages (from torch>=1.6.0->sentence_transformers) (11.0.2.54)\n", - "Requirement already satisfied: nvidia-curand-cu12==10.3.2.106 in /home/vscode/.local/lib/python3.10/site-packages (from torch>=1.6.0->sentence_transformers) (10.3.2.106)\n", - "Requirement already satisfied: nvidia-cusolver-cu12==11.4.5.107 in /home/vscode/.local/lib/python3.10/site-packages (from torch>=1.6.0->sentence_transformers) (11.4.5.107)\n", - "Requirement already satisfied: nvidia-cusparse-cu12==12.1.0.106 in /home/vscode/.local/lib/python3.10/site-packages (from torch>=1.6.0->sentence_transformers) (12.1.0.106)\n", - "Requirement already satisfied: nvidia-nccl-cu12==2.18.1 in /home/vscode/.local/lib/python3.10/site-packages (from torch>=1.6.0->sentence_transformers) (2.18.1)\n", - "Requirement already satisfied: nvidia-nvtx-cu12==12.1.105 in /home/vscode/.local/lib/python3.10/site-packages (from torch>=1.6.0->sentence_transformers) (12.1.105)\n", - "Requirement already satisfied: triton==2.1.0 in /home/vscode/.local/lib/python3.10/site-packages (from torch>=1.6.0->sentence_transformers) (2.1.0)\n", - "Requirement already satisfied: nvidia-nvjitlink-cu12 in /home/vscode/.local/lib/python3.10/site-packages (from nvidia-cusolver-cu12==11.4.5.107->torch>=1.6.0->sentence_transformers) (12.3.101)\n", - "Requirement already satisfied: regex!=2019.12.17 in /home/vscode/.local/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (2023.12.25)\n", - "Requirement already satisfied: tokenizers<0.19,>=0.14 in /home/vscode/.local/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (0.15.0)\n", - "Requirement already satisfied: safetensors>=0.3.1 in /home/vscode/.local/lib/python3.10/site-packages (from transformers<5.0.0,>=4.6.0->sentence_transformers) (0.4.1)\n", - "Requirement already satisfied: click in /usr/local/lib/python3.10/site-packages (from nltk->sentence_transformers) (8.1.7)\n", - "Requirement already satisfied: joblib in /home/vscode/.local/lib/python3.10/site-packages (from nltk->sentence_transformers) (1.3.2)\n", - "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/vscode/.local/lib/python3.10/site-packages (from scikit-learn->sentence_transformers) (3.2.0)\n", - "Requirement already satisfied: pillow!=8.3.*,>=5.3.0 in /home/vscode/.local/lib/python3.10/site-packages (from torchvision->sentence_transformers) (10.2.0)\n", - "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/site-packages (from jinja2->torch>=1.6.0->sentence_transformers) (2.1.3)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/site-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (3.3.2)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/site-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (3.6)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/vscode/.local/lib/python3.10/site-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (1.26.18)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/site-packages (from requests->huggingface-hub>=0.4.0->sentence_transformers) (2023.11.17)\n", - "Requirement already satisfied: mpmath>=0.19 in /home/vscode/.local/lib/python3.10/site-packages (from sympy->torch>=1.6.0->sentence_transformers) (1.3.0)\n", - "Defaulting to user installation because normal site-packages is not writeable\n", - "Requirement already satisfied: tiktoken in /home/vscode/.local/lib/python3.10/site-packages (0.5.2)\n", - "Requirement already satisfied: regex>=2022.1.18 in /home/vscode/.local/lib/python3.10/site-packages (from tiktoken) (2023.12.25)\n", - "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/site-packages (from tiktoken) (2.31.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/site-packages (from requests>=2.26.0->tiktoken) (3.3.2)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/site-packages (from requests>=2.26.0->tiktoken) (3.6)\n", - "Requirement already satisfied: urllib3<3,>=1.21.1 in /home/vscode/.local/lib/python3.10/site-packages (from requests>=2.26.0->tiktoken) (1.26.18)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/site-packages (from requests>=2.26.0->tiktoken) (2023.11.17)\n", - "Defaulting to user installation because normal site-packages is not writeable\n", - "Requirement already satisfied: pypdf in /home/vscode/.local/lib/python3.10/site-packages (3.17.4)\n" - ] - } - ], + "outputs": [], "source": [ - "!pip install \"pyautogen>=0.2.3\"\n", + "!pip install \"autogen-agentchat~=0.2\"\n", "!pip install chromadb\n", "!pip install sentence_transformers\n", "!pip install tiktoken\n", diff --git a/notebook/JSON_mode_example.ipynb b/notebook/JSON_mode_example.ipynb index c4b65c4d9f4d..eb09f51983ec 100644 --- a/notebook/JSON_mode_example.ipynb +++ b/notebook/JSON_mode_example.ipynb @@ -29,7 +29,7 @@ "JSON mode is a feature of OpenAI API, however strong models (such as Claude 3 Opus), can generate appropriate json as well.\n", "AutoGen requires `Python>=3.8`. To run this notebook example, please install:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```" ] }, @@ -40,7 +40,7 @@ "outputs": [], "source": [ "%%capture --no-stderr\n", - "# %pip install \"pyautogen>=0.2.3\"\n", + "# %pip install \"autogen-agentchat~=0.2.3\"\n", "\n", "# In Your OAI_CONFIG_LIST file, you must have two configs,\n", "# one with: \"response_format\": { \"type\": \"text\" }\n", diff --git a/notebook/agent_memory_using_zep.ipynb b/notebook/agent_memory_using_zep.ipynb new file mode 100644 index 000000000000..3212f63414f8 --- /dev/null +++ b/notebook/agent_memory_using_zep.ipynb @@ -0,0 +1,532 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Building an Agent with Long-term Memory using Autogen and Zep\n", + "\n", + "This notebook walks through how to build an Autogen Agent with long-term memory. Zep builds a knowledge graph from user interactions with the agent, enabling the agent to recall relevant facts from previous conversations or user interactions.\n", + "\n", + "In this notebook we will:\n", + "- Create an Autogen Agent class that extends `ConversableAgent` by adding long-term memory\n", + "- Create a Mental Health Assistant Agent, CareBot, that acts as a counselor and coach.\n", + "- Create a user Agent, Cathy, who stands in for our expected user.\n", + "- Demonstrate preloading chat history into Zep.\n", + "- Demonstrate the agents in conversation, with CareBot recalling facts from previous conversations with Cathy.\n", + "- Inspect Facts within Zep, and demonstrate how to use Zep's Fact Ratings to improve the quality of returned facts.\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Requirements\n", + "\n", + "````{=mdx}\n", + ":::info Requirements\n", + "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", + "\n", + "```bash\n", + "pip install autogen~=0.3 zep-cloud python-dotenv\n", + "```\n", + "\n", + "For more information, please refer to the [installation guide](/docs/installation/).\n", + ":::\n", + "````" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "flaml.automl is not available. Please install flaml[automl] to enable AutoML functionalities.\n" + ] + } + ], + "source": [ + "import os\n", + "import uuid\n", + "from typing import Dict, Union\n", + "\n", + "from dotenv import load_dotenv\n", + "\n", + "from autogen import Agent, ConversableAgent\n", + "\n", + "load_dotenv()\n", + "\n", + "config_list = [\n", + " {\n", + " \"model\": \"gpt-4o-mini\",\n", + " \"api_key\": os.environ.get(\"OPENAI_API_KEY\"),\n", + " \"max_tokens\": 1024,\n", + " }\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## initiualize the Zep Client\n", + "\n", + "You can sign up for a Zep account here: https://www.getzep.com/" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from zep_cloud import FactRatingExamples, FactRatingInstruction, Message\n", + "from zep_cloud.client import AsyncZep\n", + "\n", + "MIN_FACT_RATING = 0.3\n", + "\n", + "# Configure Zep\n", + "zep = AsyncZep(api_key=os.environ.get(\"ZEP_API_KEY\"))" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "def convert_to_zep_messages(chat_history: list[dict[str, str | None]]) -> list[Message]:\n", + " \"\"\"\n", + " Convert chat history to Zep messages.\n", + "\n", + " Args:\n", + " chat_history (list): List of dictionaries containing chat messages.\n", + "\n", + " Returns:\n", + " list: List of Zep Message objects.\n", + " \"\"\"\n", + " return [\n", + " Message(\n", + " role_type=msg[\"role\"],\n", + " role=msg.get(\"name\", None),\n", + " content=msg[\"content\"],\n", + " )\n", + " for msg in chat_history\n", + " ]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ZepConversableAgent\n", + "\n", + "The `ZepConversableAgent` is a custom implementation of the `ConversableAgent` that integrates with Zep for long-term memory management. This class extends the functionality of the base `ConversableAgent` by adding Zep-specific features for persisting and retrieving facts from long-term memory." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "class ZepConversableAgent(ConversableAgent):\n", + " \"\"\"\n", + " A custom ConversableAgent that integrates with Zep for long-term memory.\n", + " \"\"\"\n", + "\n", + " def __init__(\n", + " self,\n", + " name: str,\n", + " system_message: str,\n", + " llm_config: dict,\n", + " function_map: dict,\n", + " human_input_mode: str,\n", + " zep_session_id: str,\n", + " ):\n", + " super().__init__(\n", + " name=name,\n", + " system_message=system_message,\n", + " llm_config=llm_config,\n", + " function_map=function_map,\n", + " human_input_mode=human_input_mode,\n", + " )\n", + " self.zep_session_id = zep_session_id\n", + " # store the original system message as we will update it with relevant facts from Zep\n", + " self.original_system_message = system_message\n", + " self.register_hook(\"a_process_last_received_message\", self.persist_user_messages)\n", + " self.register_hook(\"a_process_message_before_send\", self.persist_assistant_messages)\n", + "\n", + " async def persist_assistant_messages(\n", + " self, sender: Agent, message: Union[Dict, str], recipient: Agent, silent: bool\n", + " ):\n", + " \"\"\"Agent sends a message to the user. Add the message to Zep.\"\"\"\n", + "\n", + " # Assume message is a string\n", + " zep_messages = convert_to_zep_messages([{\"role\": \"assistant\", \"name\": self.name, \"content\": message}])\n", + " await zep.memory.add(session_id=self.zep_session_id, messages=zep_messages)\n", + "\n", + " return message\n", + "\n", + " async def persist_user_messages(self, messages: list[dict[str, str]] | str):\n", + " \"\"\"\n", + " User sends a message to the agent. Add the message to Zep and\n", + " update the system message with relevant facts from Zep.\n", + " \"\"\"\n", + " # Assume messages is a string\n", + " zep_messages = convert_to_zep_messages([{\"role\": \"user\", \"content\": messages}])\n", + " await zep.memory.add(session_id=self.zep_session_id, messages=zep_messages)\n", + "\n", + " memory = await zep.memory.get(self.zep_session_id, min_rating=MIN_FACT_RATING)\n", + "\n", + " # Update the system message with the relevant facts retrieved from Zep\n", + " self.update_system_message(\n", + " self.original_system_message\n", + " + f\"\\n\\nRelevant facts about the user and their prior conversation:\\n{memory.relevant_facts}\"\n", + " )\n", + "\n", + " return messages" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Zep User and Session Management\n", + "\n", + "### Zep User\n", + "A Zep User represents an individual interacting with your application. Each User can have multiple Sessions associated with them, allowing you to track and manage interactions over time. The unique identifier for each user is their `UserID`, which can be any string value (e.g., username, email address, or UUID).\n", + "\n", + "### Zep Session\n", + "A Session represents a conversation and can be associated with Users in a one-to-many relationship. Chat messages are added to Sessions, with each session having many messages.\n", + "\n", + "### Fact Rating\n", + " \n", + "Fact Rating is a feature in Zep that allows you to rate the importance or relevance of facts extracted from conversations. This helps in prioritizing and filtering information when retrieving memory artifacts. Here, we rate facts based on poignancy. We provide a definition of poignancy and several examples of highly poignant and low-poignancy facts. When retrieving memory, you can use the `min_rating` parameter to filter facts based on their importance.\n", + " \n", + "Fact Rating helps ensure the most relevant information, especially in long or complex conversations, is used to ground the agent.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "Session(classifications=None, created_at='2024-10-07T21:12:13.952672Z', deleted_at=None, ended_at=None, fact_rating_instruction=FactRatingInstruction(examples=FactRatingExamples(high=\"The user received news of a family member's serious illness.\", low='The user bought a new brand of toothpaste.', medium='The user completed a challenging marathon.'), instruction='Rate the facts by poignancy. Highly poignant \\nfacts have a significant emotional impact or relevance to the user. \\nLow poignant facts are minimally relevant or of little emotional \\nsignificance.'), fact_version_uuid=None, facts=None, id=774, metadata=None, project_uuid='00000000-0000-0000-0000-000000000000', session_id='f3854ad0-5bd4-4814-a814-ec0880817953', updated_at='2024-10-07T21:12:13.952672Z', user_id='Cathy1023', uuid_='31ab3314-5ac8-4361-ad11-848fb7befedf')" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "bot_name = \"CareBot\"\n", + "user_name = \"Cathy\"\n", + "\n", + "user_id = user_name + str(uuid.uuid4())[:4]\n", + "session_id = str(uuid.uuid4())\n", + "\n", + "await zep.user.add(user_id=user_id)\n", + "\n", + "fact_rating_instruction = \"\"\"Rate the facts by poignancy. Highly poignant\n", + " facts have a significant emotional impact or relevance to the user.\n", + " Low poignant facts are minimally relevant or of little emotional significance.\n", + "\"\"\"\n", + "\n", + "fact_rating_examples = FactRatingExamples(\n", + " high=\"The user received news of a family member's serious illness.\",\n", + " medium=\"The user completed a challenging marathon.\",\n", + " low=\"The user bought a new brand of toothpaste.\",\n", + ")\n", + "\n", + "await zep.memory.add_session(\n", + " user_id=user_id,\n", + " session_id=session_id,\n", + " fact_rating_instruction=FactRatingInstruction(\n", + " instruction=fact_rating_instruction,\n", + " examples=fact_rating_examples,\n", + " ),\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Preload a prior conversation into Zep\n", + "\n", + "We'll load a prior conversation into long-term memory. We'll use facts derived from this conversation when Cathy restarts the conversation with CareBot, ensuring Carebot has context." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "data": { + "text/plain": [ + "SuccessResponse(message='OK')" + ] + }, + "execution_count": 6, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "chat_history = [\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"name\": \"carebot\",\n", + " \"content\": \"Hi Cathy, how are you doing today?\",\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"name\": \"Cathy\",\n", + " \"content\": \"To be honest, I've been feeling a bit down and demotivated lately. It's been tough.\",\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"name\": \"CareBot\",\n", + " \"content\": \"I'm sorry to hear that you're feeling down and demotivated, Cathy. It's understandable given the challenges you're facing. Can you tell me more about what's been going on?\",\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"name\": \"Cathy\",\n", + " \"content\": \"Well, I'm really struggling to process the passing of my mother.\",\n", + " },\n", + " {\n", + " \"role\": \"assistant\",\n", + " \"name\": \"CareBot\",\n", + " \"content\": \"I'm deeply sorry for your loss, Cathy. Losing a parent is incredibly difficult. It's normal to struggle with grief, and there's no 'right' way to process it. Would you like to talk about your mother or how you're coping?\",\n", + " },\n", + " {\n", + " \"role\": \"user\",\n", + " \"name\": \"Cathy\",\n", + " \"content\": \"Yes, I'd like to talk about my mother. She was a kind and loving person.\",\n", + " },\n", + "]\n", + "\n", + "# Convert chat history to Zep messages\n", + "zep_messages = convert_to_zep_messages(chat_history)\n", + "\n", + "await zep.memory.add(session_id=session_id, messages=zep_messages)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Review all facts in Zep\n", + "\n", + "We query all session facts for this user session. Only facts that meet the `MIN_FACT_RATING` threshold are returned." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "created_at='2024-10-07T21:12:15.96584Z' fact='Cathy describes her mother as a kind and loving person.' rating=0.5 uuid_='6a086a73-d4b8-4c1b-9b2f-08d5d326d813'\n", + "created_at='2024-10-07T21:12:15.96584Z' fact='Cathy has been feeling down and demotivated lately.' rating=0.5 uuid_='e19d959c-2a01-4cc7-9d49-108719f1a749'\n", + "created_at='2024-10-07T21:12:15.96584Z' fact='Cathy is struggling to process the passing of her mother.' rating=0.75 uuid_='d6c12a5d-d2a0-486e-b25d-3d4bdc5ff466'\n" + ] + } + ], + "source": [ + "response = await zep.memory.get_session_facts(session_id=session_id, min_rating=MIN_FACT_RATING)\n", + "\n", + "for r in response.facts:\n", + " print(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create the Autogen agent, CareBot, an instance of `ZepConversableAgent`\n", + "\n", + "We pass in the current `session_id` into the CareBot agent which allows it to retrieve relevant facts related to the conversation with Cathy." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "carebot_system_message = \"\"\"\n", + "You are a compassionate mental health bot and caregiver. Review information about the user and their prior conversation below and respond accordingly.\n", + "Keep responses empathetic and supportive. And remember, always prioritize the user's well-being and mental health. Keep your responses very concise and to the point.\n", + "\"\"\"\n", + "\n", + "agent = ZepConversableAgent(\n", + " bot_name,\n", + " system_message=carebot_system_message,\n", + " llm_config={\"config_list\": config_list},\n", + " function_map=None, # No registered functions, by default it is None.\n", + " human_input_mode=\"NEVER\", # Never ask for human input.\n", + " zep_session_id=session_id,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create the Autogen agent, Cathy\n", + "\n", + "Cathy is a stand-in for a human. When building a production application, you'd replace Cathy with a human-in-the-loop pattern.\n", + "\n", + "**Note** that we're instructing Cathy to start the conversation with CareBit by asking about her previous session. This is an opportunity for us to test whether fact retrieval from Zep's long-term memory is working. " + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [], + "source": [ + "cathy = ConversableAgent(\n", + " user_name,\n", + " system_message=\"You are returning to your conversation with CareBot, a mental health bot. Ask the bot about your previous session.\",\n", + " llm_config={\"config_list\": config_list},\n", + " human_input_mode=\"NEVER\", # Never ask for human input.\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Start the conversation\n", + "\n", + "We use Autogen's `a_initiate_chat` method to get the two agents conversing. CareBot is the primary agent.\n", + "\n", + "**NOTE** how Carebot is able to recall the past conversation about Cathy's mother in detail, having had relevant facts from Zep added to its system prompt." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "result = await agent.a_initiate_chat(\n", + " cathy,\n", + " message=\"Hi Cathy, nice to see you again. How are you doing today?\",\n", + " max_turns=3,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Review current facts in Zep\n", + "\n", + "Let's see how the facts have evolved as the conversation has progressed." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "created_at='2024-10-07T20:04:28.397184Z' fact=\"Cathy wants to reflect on a previous conversation about her mother and explore the topic of her mother's passing further.\" rating=0.75 uuid_='56488eeb-d8ac-4b2f-8acc-75f71b56ad76'\n", + "created_at='2024-10-07T20:04:28.397184Z' fact='Cathy is struggling to process the passing of her mother and has been feeling down and demotivated lately.' rating=0.75 uuid_='0fea3f05-ed1a-4e39-a092-c91f8af9e501'\n", + "created_at='2024-10-07T20:04:28.397184Z' fact='Cathy describes her mother as a kind and loving person.' rating=0.5 uuid_='131de203-2984-4cba-9aef-e500611f06d9'\n" + ] + } + ], + "source": [ + "response = await zep.memory.get_session_facts(session_id, min_rating=MIN_FACT_RATING)\n", + "\n", + "for r in response.facts:\n", + " print(r)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Search over Facts in Zep's long-term memory\n", + "\n", + "In addition to the `memory.get` method which uses the current conversation to retrieve facts, we can also search Zep with our own keywords. Here, we retrieve facts using a query. Again, we use fact ratings to limit the returned facts to only those with a high poignancy rating.\n", + "\n", + "The `memory.search_sessions` API may be used as an Agent tool, enabling an agent to search across user memory for relevant facts." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "created_at='2024-10-07T20:04:28.397184Z' fact='Cathy describes her mother as a kind and loving person.' rating=0.5 uuid_='131de203-2984-4cba-9aef-e500611f06d9'\n", + "created_at='2024-10-07T20:04:28.397184Z' fact='Cathy is struggling to process the passing of her mother and has been feeling down and demotivated lately.' rating=0.75 uuid_='0fea3f05-ed1a-4e39-a092-c91f8af9e501'\n", + "created_at='2024-10-07T20:04:28.397184Z' fact=\"Cathy wants to reflect on a previous conversation about her mother and explore the topic of her mother's passing further.\" rating=0.75 uuid_='56488eeb-d8ac-4b2f-8acc-75f71b56ad76'\n" + ] + } + ], + "source": [ + "response = await zep.memory.search_sessions(\n", + " text=\"What do you know about Cathy's family?\",\n", + " user_id=user_id,\n", + " search_scope=\"facts\",\n", + " min_fact_rating=MIN_FACT_RATING,\n", + ")\n", + "\n", + "for r in response.results:\n", + " print(r.fact)" + ] + } + ], + "metadata": { + "front_matter": { + "tags": [ + "memory" + ], + "description": "Agent Memory with Zep." + }, + "kernelspec": { + "display_name": ".venv", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebook/agentchat_MathChat.ipynb b/notebook/agentchat_MathChat.ipynb index afa00fb7562c..bb6f713eabc7 100644 --- a/notebook/agentchat_MathChat.ipynb +++ b/notebook/agentchat_MathChat.ipynb @@ -17,14 +17,14 @@ "\n", "AutoGen offers conversable agents powered by LLM, tool or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation. Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", "\n", - "MathChat is an experimental conversational framework for math problem solving. In this notebook, we demonstrate how to use MathChat to solve math problems. MathChat uses the `AssistantAgent` and `MathUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_auto_feedback_from_code_execution.ipynb)). Essentially, `MathUserProxyAgent` implements a different auto reply mechanism corresponding to the MathChat prompts. You can find more details in the paper [An Empirical Study on Challenging Math Problem Solving with GPT-4](https://arxiv.org/abs/2306.01337) or the [blogpost](https://microsoft.github.io/autogen/blog/2023/06/28/MathChat).\n", + "MathChat is an experimental conversational framework for math problem solving. In this notebook, we demonstrate how to use MathChat to solve math problems. MathChat uses the `AssistantAgent` and `MathUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_auto_feedback_from_code_execution.ipynb)). Essentially, `MathUserProxyAgent` implements a different auto reply mechanism corresponding to the MathChat prompts. You can find more details in the paper [An Empirical Study on Challenging Math Problem Solving with GPT-4](https://arxiv.org/abs/2306.01337) or the [blogpost](https://microsoft.github.io/autogen/blog/2023/06/28/MathChat).\n", "\n", "````{=mdx}\n", ":::info Requirements\n", "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", "\n", "```bash\n", - "pip install pyautogen[mathchat]\n", + "pip install autogen-agentchat[mathchat]~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", @@ -57,9 +57,7 @@ " \"OAI_CONFIG_LIST\",\n", " filter_dict={\n", " \"model\": {\n", - " \"gpt-4-1106-preview\",\n", - " \"gpt-3.5-turbo\",\n", - " \"gpt-35-turbo\",\n", + " \"gpt-4o\",\n", " }\n", " },\n", ")" diff --git a/notebook/agentchat_RetrieveChat.ipynb b/notebook/agentchat_RetrieveChat.ipynb index 0b829835a0a3..6debb2450df3 100644 --- a/notebook/agentchat_RetrieveChat.ipynb +++ b/notebook/agentchat_RetrieveChat.ipynb @@ -10,7 +10,7 @@ "AutoGen offers conversable agents powered by LLM, tool or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", "\n", - "RetrieveChat is a conversational system for retrieval-augmented code generation and question answering. In this notebook, we demonstrate how to utilize RetrieveChat to generate code and answer questions based on customized documentations that are not present in the LLM's training dataset. RetrieveChat uses the `AssistantAgent` and `RetrieveUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_auto_feedback_from_code_execution.ipynb)). Essentially, `RetrieveUserProxyAgent` implement a different auto-reply mechanism corresponding to the RetrieveChat prompts.\n", + "RetrieveChat is a conversational system for retrieval-augmented code generation and question answering. In this notebook, we demonstrate how to utilize RetrieveChat to generate code and answer questions based on customized documentations that are not present in the LLM's training dataset. RetrieveChat uses the `AssistantAgent` and `RetrieveUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_auto_feedback_from_code_execution.ipynb)). Essentially, `RetrieveUserProxyAgent` implement a different auto-reply mechanism corresponding to the RetrieveChat prompts.\n", "\n", "## Table of Contents\n", "We'll demonstrate six examples of using RetrieveChat for code generation and question answering:\n", @@ -28,7 +28,7 @@ "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", "\n", "```bash\n", - "pip install pyautogen[retrievechat] flaml[automl]\n", + "pip install autogen-agentchat[retrievechat]~=0.2 flaml[automl]\n", "```\n", "\n", "*You'll need to install `chromadb<=0.5.0` if you see issue like [#3551](https://github.com/microsoft/autogen/issues/3551).*\n", diff --git a/notebook/agentchat_RetrieveChat_couchbase.ipynb b/notebook/agentchat_RetrieveChat_couchbase.ipynb new file mode 100644 index 000000000000..17e9284a379a --- /dev/null +++ b/notebook/agentchat_RetrieveChat_couchbase.ipynb @@ -0,0 +1,579 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Using RetrieveChat Powered by Couchbase Capella for Retrieve Augmented Code Generation and Question Answering\n", + "\n", + "AutoGen offers conversable agents powered by LLM, tool or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", + "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", + "\n", + "RetrieveChat is a conversational system for retrieval-augmented code generation and question answering. In this notebook, we demonstrate how to utilize RetrieveChat to generate code and answer questions based on customized documentations that are not present in the LLM's training dataset. RetrieveChat uses the `AssistantAgent` and `RetrieveUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_auto_feedback_from_code_execution.ipynb)). Essentially, `RetrieveUserProxyAgent` implement a different auto-reply mechanism corresponding to the RetrieveChat prompts.\n", + "\n", + "## Table of Contents\n", + "We'll demonstrate six examples of using RetrieveChat for code generation and question answering:\n", + "\n", + "- [Example 1: Generate code based off docstrings w/o human feedback](#example-1)\n", + "\n", + "````{=mdx}\n", + ":::info Requirements\n", + "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", + "\n", + "```bash\n", + "pip install pyautogen[retrievechat-couchbase] flaml[automl]\n", + "```\n", + "\n", + "For more information, please refer to the [installation guide](/docs/installation/).\n", + ":::\n", + "````\n", + "\n", + "Ensure you have a Couchbase Capella cluster running. Read more on how to get started [here](https://docs.couchbase.com/cloud/get-started/intro.html)" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set your API Endpoint\n" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "models to use: ['gpt-4o-mini']\n" + ] + } + ], + "source": [ + "import os\n", + "import sys\n", + "\n", + "from autogen import AssistantAgent\n", + "\n", + "sys.path.append(os.path.abspath(\"/workspaces/autogen/autogen/agentchat/contrib\"))\n", + "\n", + "from autogen.agentchat.contrib.retrieve_user_proxy_agent import RetrieveUserProxyAgent\n", + "\n", + "# Accepted file formats for that can be stored in\n", + "# a vector database instance\n", + "from autogen.retrieve_utils import TEXT_FORMATS\n", + "\n", + "config_list = [{\"model\": \"gpt-4o-mini\", \"api_key\": os.environ[\"OPENAI_API_KEY\"], \"api_type\": \"openai\"}]\n", + "assert len(config_list) > 0\n", + "print(\"models to use: \", [config_list[i][\"model\"] for i in range(len(config_list))])" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "````{=mdx}\n", + ":::tip\n", + "Learn more about configuring LLMs for agents [here](/docs/topics/llm_configuration).\n", + ":::\n", + "````\n", + "\n", + "## Construct agents for RetrieveChat\n", + "\n", + "We start by initializing the `AssistantAgent` and `RetrieveUserProxyAgent`. The system message needs to be set to \"You are a helpful assistant.\" for AssistantAgent. The detailed instructions are given in the user message. Later we will use the `RetrieveUserProxyAgent.message_generator` to combine the instructions and a retrieval augmented generation task for an initial prompt to be sent to the LLM assistant." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Accepted file formats for `docs_path`:\n", + "['txt', 'json', 'csv', 'tsv', 'md', 'html', 'htm', 'rtf', 'rst', 'jsonl', 'log', 'xml', 'yaml', 'yml', 'pdf']\n" + ] + } + ], + "source": [ + "print(\"Accepted file formats for `docs_path`:\")\n", + "print(TEXT_FORMATS)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# 1. create an AssistantAgent instance named \"assistant\"\n", + "assistant = AssistantAgent(\n", + " name=\"assistant\",\n", + " system_message=\"You are a helpful assistant.\",\n", + " llm_config={\n", + " \"timeout\": 600,\n", + " \"cache_seed\": 42,\n", + " \"config_list\": config_list,\n", + " },\n", + ")\n", + "\n", + "# 2. create the RetrieveUserProxyAgent instance named \"ragproxyagent\"\n", + "# Refer to https://microsoft.github.io/autogen/docs/reference/agentchat/contrib/retrieve_user_proxy_agent\n", + "# and https://microsoft.github.io/autogen/docs/reference/agentchat/contrib/vectordb/couchbase\n", + "# for more information on the RetrieveUserProxyAgent and CouchbaseVectorDB\n", + "ragproxyagent = RetrieveUserProxyAgent(\n", + " name=\"ragproxyagent\",\n", + " human_input_mode=\"NEVER\",\n", + " max_consecutive_auto_reply=3,\n", + " retrieve_config={\n", + " \"task\": \"code\",\n", + " \"docs_path\": [\n", + " \"https://raw.githubusercontent.com/microsoft/FLAML/main/website/docs/Examples/Integrate%20-%20Spark.md\",\n", + " \"https://raw.githubusercontent.com/microsoft/FLAML/main/website/docs/Research.md\",\n", + " ],\n", + " \"chunk_token_size\": 2000,\n", + " \"model\": config_list[0][\"model\"],\n", + " \"vector_db\": \"couchbase\", # Couchbase Capella VectorDB\n", + " \"collection_name\": \"demo_collection\", # Couchbase Capella collection name to be utilized/created\n", + " \"db_config\": {\n", + " \"connection_string\": os.environ[\"CB_CONN_STR\"], # Couchbase Capella connection string\n", + " \"username\": os.environ[\"CB_USERNAME\"], # Couchbase Capella username\n", + " \"password\": os.environ[\"CB_PASSWORD\"], # Couchbase Capella password\n", + " \"bucket_name\": \"test_db\", # Couchbase Capella bucket name\n", + " \"scope_name\": \"test_scope\", # Couchbase Capella scope name\n", + " \"index_name\": \"vector_index\", # Couchbase Capella index name to be created\n", + " },\n", + " \"get_or_create\": True, # set to False if you don't want to reuse an existing collection\n", + " \"overwrite\": False, # set to True if you want to overwrite an existing collection, each overwrite will force a index creation and reupload of documents\n", + " },\n", + " code_execution_config=False, # set to False if you don't want to execute the code\n", + ")" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Example 1\n", + "\n", + "[Back to top](#table-of-contents)\n", + "\n", + "Use RetrieveChat to help generate sample code and automatically run the code and fix errors if there is any.\n", + "\n", + "Problem: Which API should I use if I want to use FLAML for a classification task and I want to train the model in 30 seconds. Use spark to parallel the training. Force cancel jobs if time limit is reached.\n", + "\n", + "Note: You may need to create an index on the cluster to query" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-10-16 12:08:07,062 - autogen.agentchat.contrib.retrieve_user_proxy_agent - INFO - \u001b[32mUse the existing collection `demo_collection`.\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Trying to create collection.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2024-10-16 12:08:07,953 - autogen.agentchat.contrib.retrieve_user_proxy_agent - INFO - Found 2 chunks.\u001b[0m\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "VectorDB returns doc_ids: [['bdfbc921', '7968cf3c']]\n", + "\u001b[32mAdding content of doc bdfbc921 to context.\u001b[0m\n", + "\u001b[32mAdding content of doc 7968cf3c to context.\u001b[0m\n", + "\u001b[33mragproxyagent\u001b[0m (to assistant):\n", + "\n", + "You're a retrieve augmented coding assistant. You answer user's questions based on your own knowledge and the\n", + "context provided by the user.\n", + "If you can't answer the question with or without the current context, you should reply exactly `UPDATE CONTEXT`.\n", + "For code generation, you must obey the following rules:\n", + "Rule 1. You MUST NOT install any packages because all the packages needed are already installed.\n", + "Rule 2. You must follow the formats below to write your code:\n", + "```language\n", + "# your code\n", + "```\n", + "\n", + "User's question is: How can I use FLAML to perform a classification task and use spark to do parallel training. Train 30 seconds and force cancel jobs if time limit is reached.\n", + "\n", + "Context is: # Integrate - Spark\n", + "\n", + "FLAML has integrated Spark for distributed training. There are two main aspects of integration with Spark:\n", + "\n", + "- Use Spark ML estimators for AutoML.\n", + "- Use Spark to run training in parallel spark jobs.\n", + "\n", + "## Spark ML Estimators\n", + "\n", + "FLAML integrates estimators based on Spark ML models. These models are trained in parallel using Spark, so we called them Spark estimators. To use these models, you first need to organize your data in the required format.\n", + "\n", + "### Data\n", + "\n", + "For Spark estimators, AutoML only consumes Spark data. FLAML provides a convenient function `to_pandas_on_spark` in the `flaml.automl.spark.utils` module to convert your data into a pandas-on-spark (`pyspark.pandas`) dataframe/series, which Spark estimators require.\n", + "\n", + "This utility function takes data in the form of a `pandas.Dataframe` or `pyspark.sql.Dataframe` and converts it into a pandas-on-spark dataframe. It also takes `pandas.Series` or `pyspark.sql.Dataframe` and converts it into a [pandas-on-spark](https://spark.apache.org/docs/latest/api/python/user_guide/pandas_on_spark/index.html) series. If you pass in a `pyspark.pandas.Dataframe`, it will not make any changes.\n", + "\n", + "This function also accepts optional arguments `index_col` and `default_index_type`.\n", + "\n", + "- `index_col` is the column name to use as the index, default is None.\n", + "- `default_index_type` is the default index type, default is \"distributed-sequence\". More info about default index type could be found on Spark official [documentation](https://spark.apache.org/docs/latest/api/python/user_guide/pandas_on_spark/options.html#default-index-type)\n", + "\n", + "Here is an example code snippet for Spark Data:\n", + "\n", + "```python\n", + "import pandas as pd\n", + "from flaml.automl.spark.utils import to_pandas_on_spark\n", + "\n", + "# Creating a dictionary\n", + "data = {\n", + " \"Square_Feet\": [800, 1200, 1800, 1500, 850],\n", + " \"Age_Years\": [20, 15, 10, 7, 25],\n", + " \"Price\": [100000, 200000, 300000, 240000, 120000],\n", + "}\n", + "\n", + "# Creating a pandas DataFrame\n", + "dataframe = pd.DataFrame(data)\n", + "label = \"Price\"\n", + "\n", + "# Convert to pandas-on-spark dataframe\n", + "psdf = to_pandas_on_spark(dataframe)\n", + "```\n", + "\n", + "To use Spark ML models you need to format your data appropriately. Specifically, use [`VectorAssembler`](https://spark.apache.org/docs/latest/api/python/reference/api/pyspark.ml.feature.VectorAssembler.html) to merge all feature columns into a single vector column.\n", + "\n", + "Here is an example of how to use it:\n", + "\n", + "```python\n", + "from pyspark.ml.feature import VectorAssembler\n", + "\n", + "columns = psdf.columns\n", + "feature_cols = [col for col in columns if col != label]\n", + "featurizer = VectorAssembler(inputCols=feature_cols, outputCol=\"features\")\n", + "psdf = featurizer.transform(psdf.to_spark(index_col=\"index\"))[\"index\", \"features\"]\n", + "```\n", + "\n", + "Later in conducting the experiment, use your pandas-on-spark data like non-spark data and pass them using `X_train, y_train` or `dataframe, label`.\n", + "\n", + "### Estimators\n", + "\n", + "#### Model List\n", + "\n", + "- `lgbm_spark`: The class for fine-tuning Spark version LightGBM models, using [SynapseML](https://microsoft.github.io/SynapseML/docs/features/lightgbm/about/) API.\n", + "\n", + "#### Usage\n", + "\n", + "First, prepare your data in the required format as described in the previous section.\n", + "\n", + "By including the models you intend to try in the `estimators_list` argument to `flaml.automl`, FLAML will start trying configurations for these models. If your input is Spark data, FLAML will also use estimators with the `_spark` postfix by default, even if you haven't specified them.\n", + "\n", + "Here is an example code snippet using SparkML models in AutoML:\n", + "\n", + "```python\n", + "import flaml\n", + "\n", + "# prepare your data in pandas-on-spark format as we previously mentioned\n", + "\n", + "automl = flaml.AutoML()\n", + "settings = {\n", + " \"time_budget\": 30,\n", + " \"metric\": \"r2\",\n", + " \"estimator_list\": [\"lgbm_spark\"], # this setting is optional\n", + " \"task\": \"regression\",\n", + "}\n", + "\n", + "automl.fit(\n", + " dataframe=psdf,\n", + " label=label,\n", + " **settings,\n", + ")\n", + "```\n", + "\n", + "[Link to notebook](https://github.com/microsoft/FLAML/blob/main/notebook/automl_bankrupt_synapseml.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/FLAML/blob/main/notebook/automl_bankrupt_synapseml.ipynb)\n", + "\n", + "## Parallel Spark Jobs\n", + "\n", + "You can activate Spark as the parallel backend during parallel tuning in both [AutoML](/docs/Use-Cases/Task-Oriented-AutoML#parallel-tuning) and [Hyperparameter Tuning](/docs/Use-Cases/Tune-User-Defined-Function#parallel-tuning), by setting the `use_spark` to `true`. FLAML will dispatch your job to the distributed Spark backend using [`joblib-spark`](https://github.com/joblib/joblib-spark).\n", + "\n", + "Please note that you should not set `use_spark` to `true` when applying AutoML and Tuning for Spark Data. This is because only SparkML models will be used for Spark Data in AutoML and Tuning. As SparkML models run in parallel, there is no need to distribute them with `use_spark` again.\n", + "\n", + "All the Spark-related arguments are stated below. These arguments are available in both Hyperparameter Tuning and AutoML:\n", + "\n", + "- `use_spark`: boolean, default=False | Whether to use spark to run the training in parallel spark jobs. This can be used to accelerate training on large models and large datasets, but will incur more overhead in time and thus slow down training in some cases. GPU training is not supported yet when use_spark is True. For Spark clusters, by default, we will launch one trial per executor. However, sometimes we want to launch more trials than the number of executors (e.g., local mode). In this case, we can set the environment variable `FLAML_MAX_CONCURRENT` to override the detected `num_executors`. The final number of concurrent trials will be the minimum of `n_concurrent_trials` and `num_executors`.\n", + "- `n_concurrent_trials`: int, default=1 | The number of concurrent trials. When n_concurrent_trials > 1, FLAML performes parallel tuning.\n", + "- `force_cancel`: boolean, default=False | Whether to forcely cancel Spark jobs if the search time exceeded the time budget. Spark jobs include parallel tuning jobs and Spark-based model training jobs.\n", + "\n", + "An example code snippet for using parallel Spark jobs:\n", + "\n", + "```python\n", + "import flaml\n", + "\n", + "automl_experiment = flaml.AutoML()\n", + "automl_settings = {\n", + " \"time_budget\": 30,\n", + " \"metric\": \"r2\",\n", + " \"task\": \"regression\",\n", + " \"n_concurrent_trials\": 2,\n", + " \"use_spark\": True,\n", + " \"force_cancel\": True, # Activating the force_cancel option can immediately halt Spark jobs once they exceed the allocated time_budget.\n", + "}\n", + "\n", + "automl.fit(\n", + " dataframe=dataframe,\n", + " label=label,\n", + " **automl_settings,\n", + ")\n", + "```\n", + "\n", + "[Link to notebook](https://github.com/microsoft/FLAML/blob/main/notebook/integrate_spark.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/FLAML/blob/main/notebook/integrate_spark.ipynb)\n", + "# Research\n", + "\n", + "For technical details, please check our research publications.\n", + "\n", + "- [FLAML: A Fast and Lightweight AutoML Library](https://www.microsoft.com/en-us/research/publication/flaml-a-fast-and-lightweight-automl-library/). Chi Wang, Qingyun Wu, Markus Weimer, Erkang Zhu. MLSys 2021.\n", + "\n", + "```bibtex\n", + "@inproceedings{wang2021flaml,\n", + " title={FLAML: A Fast and Lightweight AutoML Library},\n", + " author={Chi Wang and Qingyun Wu and Markus Weimer and Erkang Zhu},\n", + " year={2021},\n", + " booktitle={MLSys},\n", + "}\n", + "```\n", + "\n", + "- [Frugal Optimization for Cost-related Hyperparameters](https://arxiv.org/abs/2005.01571). Qingyun Wu, Chi Wang, Silu Huang. AAAI 2021.\n", + "\n", + "```bibtex\n", + "@inproceedings{wu2021cfo,\n", + " title={Frugal Optimization for Cost-related Hyperparameters},\n", + " author={Qingyun Wu and Chi Wang and Silu Huang},\n", + " year={2021},\n", + " booktitle={AAAI},\n", + "}\n", + "```\n", + "\n", + "- [Economical Hyperparameter Optimization With Blended Search Strategy](https://www.microsoft.com/en-us/research/publication/economical-hyperparameter-optimization-with-blended-search-strategy/). Chi Wang, Qingyun Wu, Silu Huang, Amin Saied. ICLR 2021.\n", + "\n", + "```bibtex\n", + "@inproceedings{wang2021blendsearch,\n", + " title={Economical Hyperparameter Optimization With Blended Search Strategy},\n", + " author={Chi Wang and Qingyun Wu and Silu Huang and Amin Saied},\n", + " year={2021},\n", + " booktitle={ICLR},\n", + "}\n", + "```\n", + "\n", + "- [An Empirical Study on Hyperparameter Optimization for Fine-Tuning Pre-trained Language Models](https://aclanthology.org/2021.acl-long.178.pdf). Susan Xueqing Liu, Chi Wang. ACL 2021.\n", + "\n", + "```bibtex\n", + "@inproceedings{liuwang2021hpolm,\n", + " title={An Empirical Study on Hyperparameter Optimization for Fine-Tuning Pre-trained Language Models},\n", + " author={Susan Xueqing Liu and Chi Wang},\n", + " year={2021},\n", + " booktitle={ACL},\n", + "}\n", + "```\n", + "\n", + "- [ChaCha for Online AutoML](https://www.microsoft.com/en-us/research/publication/chacha-for-online-automl/). Qingyun Wu, Chi Wang, John Langford, Paul Mineiro and Marco Rossi. ICML 2021.\n", + "\n", + "```bibtex\n", + "@inproceedings{wu2021chacha,\n", + " title={ChaCha for Online AutoML},\n", + " author={Qingyun Wu and Chi Wang and John Langford and Paul Mineiro and Marco Rossi},\n", + " year={2021},\n", + " booktitle={ICML},\n", + "}\n", + "```\n", + "\n", + "- [Fair AutoML](https://arxiv.org/abs/2111.06495). Qingyun Wu, Chi Wang. ArXiv preprint arXiv:2111.06495 (2021).\n", + "\n", + "```bibtex\n", + "@inproceedings{wuwang2021fairautoml,\n", + " title={Fair AutoML},\n", + " author={Qingyun Wu and Chi Wang},\n", + " year={2021},\n", + " booktitle={ArXiv preprint arXiv:2111.06495},\n", + "}\n", + "```\n", + "\n", + "- [Mining Robust Default Configurations for Resource-constrained AutoML](https://arxiv.org/abs/2202.09927). Moe Kayali, Chi Wang. ArXiv preprint arXiv:2202.09927 (2022).\n", + "\n", + "```bibtex\n", + "@inproceedings{kayaliwang2022default,\n", + " title={Mining Robust Default Configurations for Resource-constrained AutoML},\n", + " author={Moe Kayali and Chi Wang},\n", + " year={2022},\n", + " booktitle={ArXiv preprint arXiv:2202.09927},\n", + "}\n", + "```\n", + "\n", + "- [Targeted Hyperparameter Optimization with Lexicographic Preferences Over Multiple Objectives](https://openreview.net/forum?id=0Ij9_q567Ma). Shaokun Zhang, Feiran Jia, Chi Wang, Qingyun Wu. ICLR 2023 (notable-top-5%).\n", + "\n", + "```bibtex\n", + "@inproceedings{zhang2023targeted,\n", + " title={Targeted Hyperparameter Optimization with Lexicographic Preferences Over Multiple Objectives},\n", + " author={Shaokun Zhang and Feiran Jia and Chi Wang and Qingyun Wu},\n", + " booktitle={International Conference on Learning Representations},\n", + " year={2023},\n", + " url={https://openreview.net/forum?id=0Ij9_q567Ma},\n", + "}\n", + "```\n", + "\n", + "- [Cost-Effective Hyperparameter Optimization for Large Language Model Generation Inference](https://arxiv.org/abs/2303.04673). Chi Wang, Susan Xueqing Liu, Ahmed H. Awadallah. ArXiv preprint arXiv:2303.04673 (2023).\n", + "\n", + "```bibtex\n", + "@inproceedings{wang2023EcoOptiGen,\n", + " title={Cost-Effective Hyperparameter Optimization for Large Language Model Generation Inference},\n", + " author={Chi Wang and Susan Xueqing Liu and Ahmed H. Awadallah},\n", + " year={2023},\n", + " booktitle={ArXiv preprint arXiv:2303.04673},\n", + "}\n", + "```\n", + "\n", + "- [An Empirical Study on Challenging Math Problem Solving with GPT-4](https://arxiv.org/abs/2306.01337). Yiran Wu, Feiran Jia, Shaokun Zhang, Hangyu Li, Erkang Zhu, Yue Wang, Yin Tat Lee, Richard Peng, Qingyun Wu, Chi Wang. ArXiv preprint arXiv:2306.01337 (2023).\n", + "\n", + "```bibtex\n", + "@inproceedings{wu2023empirical,\n", + " title={An Empirical Study on Challenging Math Problem Solving with GPT-4},\n", + " author={Yiran Wu and Feiran Jia and Shaokun Zhang and Hangyu Li and Erkang Zhu and Yue Wang and Yin Tat Lee and Richard Peng and Qingyun Wu and Chi Wang},\n", + " year={2023},\n", + " booktitle={ArXiv preprint arXiv:2306.01337},\n", + "}\n", + "```\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33massistant\u001b[0m (to ragproxyagent):\n", + "\n", + "```python\n", + "import pandas as pd\n", + "from pyspark.ml.feature import VectorAssembler\n", + "import flaml\n", + "from flaml.automl.spark.utils import to_pandas_on_spark\n", + "\n", + "# Creating a dictionary for the example data\n", + "data = {\n", + " \"Square_Feet\": [800, 1200, 1800, 1500, 850],\n", + " \"Age_Years\": [20, 15, 10, 7, 25],\n", + " \"Price\": [100000, 200000, 300000, 240000, 120000],\n", + "}\n", + "\n", + "# Creating a pandas DataFrame\n", + "dataframe = pd.DataFrame(data)\n", + "label = \"Price\"\n", + "\n", + "# Convert to pandas-on-spark dataframe\n", + "psdf = to_pandas_on_spark(dataframe)\n", + "\n", + "# Prepare features using VectorAssembler\n", + "columns = psdf.columns\n", + "feature_cols = [col for col in columns if col != label]\n", + "featurizer = VectorAssembler(inputCols=feature_cols, outputCol=\"features\")\n", + "psdf = featurizer.transform(psdf.to_spark(index_col=\"index\"))[[\"index\", \"features\"]]\n", + "\n", + "# Setting up and running FLAML for AutoML with Spark\n", + "automl = flaml.AutoML()\n", + "automl_settings = {\n", + " \"time_budget\": 30, # Set the time budget to 30 seconds\n", + " \"metric\": \"r2\", # Performance metric\n", + " \"task\": \"regression\", # Problem type\n", + " \"n_concurrent_trials\": 2, # Number of concurrent trials\n", + " \"use_spark\": True, # Use Spark for parallel jobs\n", + " \"force_cancel\": True, # Force cancel jobs if time limit is reached\n", + "}\n", + "\n", + "automl.fit(\n", + " dataframe=psdf,\n", + " label=label,\n", + " **automl_settings\n", + ")\n", + "```\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mragproxyagent\u001b[0m (to assistant):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33massistant\u001b[0m (to ragproxyagent):\n", + "\n", + "UPDATE CONTEXT\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32mUpdating context and resetting conversation.\u001b[0m\n", + "VectorDB returns doc_ids: [['bdfbc921', '7968cf3c']]\n", + "\u001b[32mNo more context, will terminate.\u001b[0m\n", + "\u001b[33mragproxyagent\u001b[0m (to assistant):\n", + "\n", + "TERMINATE\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "# reset the assistant. Always reset the assistant before starting a new conversation.\n", + "assistant.reset()\n", + "\n", + "# given a problem, we use the ragproxyagent to generate a prompt to be sent to the assistant as the initial message.\n", + "# the assistant receives the message and generates a response. The response will be sent back to the ragproxyagent for processing.\n", + "# The conversation continues until the termination condition is met, in RetrieveChat, the termination condition when no human-in-loop is no code block detected.\n", + "# With human-in-loop, the conversation will continue until the user says \"exit\".\n", + "code_problem = \"How can I use FLAML to perform a classification task and use spark to do parallel training. Train 30 seconds and force cancel jobs if time limit is reached.\"\n", + "chat_result = ragproxyagent.initiate_chat(assistant, message=ragproxyagent.message_generator, problem=code_problem)" + ] + } + ], + "metadata": { + "front_matter": { + "description": "Explore the use of AutoGen's RetrieveChat for tasks like code generation from docstrings, answering complex questions with human feedback, and exploiting features like Update Context, custom prompts, and few-shot learning.", + "tags": [ + "RAG" + ] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.7" + }, + "skip_test": "Requires interactive usage" + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/notebook/agentchat_RetrieveChat_mongodb.ipynb b/notebook/agentchat_RetrieveChat_mongodb.ipynb index 09c3c44bef27..30508ad6c474 100644 --- a/notebook/agentchat_RetrieveChat_mongodb.ipynb +++ b/notebook/agentchat_RetrieveChat_mongodb.ipynb @@ -10,7 +10,7 @@ "AutoGen offers conversable agents powered by LLM, tool or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", "\n", - "RetrieveChat is a conversational system for retrieval-augmented code generation and question answering. In this notebook, we demonstrate how to utilize RetrieveChat to generate code and answer questions based on customized documentations that are not present in the LLM's training dataset. RetrieveChat uses the `AssistantAgent` and `RetrieveUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_auto_feedback_from_code_execution.ipynb)). Essentially, `RetrieveUserProxyAgent` implement a different auto-reply mechanism corresponding to the RetrieveChat prompts.\n", + "RetrieveChat is a conversational system for retrieval-augmented code generation and question answering. In this notebook, we demonstrate how to utilize RetrieveChat to generate code and answer questions based on customized documentations that are not present in the LLM's training dataset. RetrieveChat uses the `AssistantAgent` and `RetrieveUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_auto_feedback_from_code_execution.ipynb)). Essentially, `RetrieveUserProxyAgent` implement a different auto-reply mechanism corresponding to the RetrieveChat prompts.\n", "\n", "## Table of Contents\n", "We'll demonstrate six examples of using RetrieveChat for code generation and question answering:\n", @@ -22,7 +22,7 @@ "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", "\n", "```bash\n", - "pip install pyautogen[retrievechat-mongodb] flaml[automl]\n", + "pip install autogen-agentchat[retrievechat-mongodb]~=0.2 flaml[automl]\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchat_RetrieveChat_pgvector.ipynb b/notebook/agentchat_RetrieveChat_pgvector.ipynb index 4d9dd44c33d8..18f3c5199e89 100644 --- a/notebook/agentchat_RetrieveChat_pgvector.ipynb +++ b/notebook/agentchat_RetrieveChat_pgvector.ipynb @@ -10,7 +10,7 @@ "AutoGen offers conversable agents powered by LLM, tool or human, which can be used to perform tasks collectively via automated chat. This framework allows tool use and human participation through multi-agent conversation.\n", "Please find documentation about this feature [here](https://microsoft.github.io/autogen/docs/Use-Cases/agent_chat).\n", "\n", - "RetrieveChat is a conversational system for retrieval-augmented code generation and question answering. In this notebook, we demonstrate how to utilize RetrieveChat to generate code and answer questions based on customized documentations that are not present in the LLM's training dataset. RetrieveChat uses the `AssistantAgent` and `RetrieveUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_auto_feedback_from_code_execution.ipynb)). Essentially, `RetrieveUserProxyAgent` implement a different auto-reply mechanism corresponding to the RetrieveChat prompts.\n", + "RetrieveChat is a conversational system for retrieval-augmented code generation and question answering. In this notebook, we demonstrate how to utilize RetrieveChat to generate code and answer questions based on customized documentations that are not present in the LLM's training dataset. RetrieveChat uses the `AssistantAgent` and `RetrieveUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_auto_feedback_from_code_execution.ipynb)). Essentially, `RetrieveUserProxyAgent` implement a different auto-reply mechanism corresponding to the RetrieveChat prompts.\n", "\n", "## Table of Contents\n", "We'll demonstrate six examples of using RetrieveChat for code generation and question answering:\n", @@ -24,7 +24,7 @@ "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", "\n", "```bash\n", - "pip install pyautogen[retrievechat-pgvector] flaml[automl]\n", + "pip install autogen-agentchat[retrievechat-pgvector]~=0.2 flaml[automl]\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchat_RetrieveChat_qdrant.ipynb b/notebook/agentchat_RetrieveChat_qdrant.ipynb index 0035a8e30817..e56bc00e4abb 100644 --- a/notebook/agentchat_RetrieveChat_qdrant.ipynb +++ b/notebook/agentchat_RetrieveChat_qdrant.ipynb @@ -12,7 +12,7 @@ "This notebook demonstrates the usage of Qdrant for RAG, based on [agentchat_RetrieveChat.ipynb](https://colab.research.google.com/github/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat.ipynb).\n", "\n", "\n", - "RetrieveChat is a conversational system for retrieve augmented code generation and question answering. In this notebook, we demonstrate how to utilize RetrieveChat to generate code and answer questions based on customized documentations that are not present in the LLM's training dataset. RetrieveChat uses the `AssistantAgent` and `RetrieveUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_auto_feedback_from_code_execution.ipynb)).\n", + "RetrieveChat is a conversational system for retrieve augmented code generation and question answering. In this notebook, we demonstrate how to utilize RetrieveChat to generate code and answer questions based on customized documentations that are not present in the LLM's training dataset. RetrieveChat uses the `AssistantAgent` and `RetrieveUserProxyAgent`, which is similar to the usage of `AssistantAgent` and `UserProxyAgent` in other notebooks (e.g., [Automated Task Solving with Code Generation, Execution & Debugging](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_auto_feedback_from_code_execution.ipynb)).\n", "\n", "We'll demonstrate usage of RetrieveChat with Qdrant for code generation and question answering w/ human feedback.\n", "\n", @@ -21,7 +21,7 @@ "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", "\n", "```bash\n", - "pip install \"pyautogen[retrievechat-qdrant]\" \"flaml[automl]\"\n", + "pip install \"autogen-agentchat[retrievechat-qdrant]~=0.2\" \"flaml[automl]\"\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", @@ -43,7 +43,7 @@ } ], "source": [ - "%pip install \"pyautogen[retrievechat-qdrant]\" \"flaml[automl]\" -q" + "%pip install \"autogen-agentchat[retrievechat-qdrant]~=0.2\" \"flaml[automl]\" -q" ] }, { diff --git a/notebook/agentchat_agentops.ipynb b/notebook/agentchat_agentops.ipynb index 71106e45d3c4..7c5e09c9cc52 100644 --- a/notebook/agentchat_agentops.ipynb +++ b/notebook/agentchat_agentops.ipynb @@ -55,7 +55,7 @@ "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", "\n", "```bash\n", - "pip install pyautogen agentops\n", + "pip install autogen-agentchat~=0.2 agentops\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchat_agentoptimizer.ipynb b/notebook/agentchat_agentoptimizer.ipynb index 7de418b5ee70..ea3a0d88e65d 100644 --- a/notebook/agentchat_agentoptimizer.ipynb +++ b/notebook/agentchat_agentoptimizer.ipynb @@ -53,7 +53,7 @@ "source": [ "# MathUserProxy with function_call\n", "\n", - "This agent is a customized MathUserProxy inherits from its [parent class](https://github.com/microsoft/autogen/blob/main/autogen/agentchat/contrib/math_user_proxy_agent.py).\n", + "This agent is a customized MathUserProxy inherits from its [parent class](https://github.com/microsoft/autogen/blob/0.2/autogen/agentchat/contrib/math_user_proxy_agent.py).\n", "\n", "It supports using both function_call and python to solve math problems.\n" ] diff --git a/notebook/agentchat_auto_feedback_from_code_execution.ipynb b/notebook/agentchat_auto_feedback_from_code_execution.ipynb index 6ea6f662b93b..31169e1c7a0e 100644 --- a/notebook/agentchat_auto_feedback_from_code_execution.ipynb +++ b/notebook/agentchat_auto_feedback_from_code_execution.ipynb @@ -16,7 +16,7 @@ ":::info Requirements\n", "Install the following packages before running the code below:\n", "```bash\n", - "pip install pyautogen matplotlib yfinance\n", + "pip install autogen-agentchat~=0.2 matplotlib yfinance\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", @@ -37,10 +37,10 @@ "\n", "config_list = autogen.config_list_from_json(\n", " \"OAI_CONFIG_LIST\",\n", - " filter_dict={\"tags\": [\"gpt-4\"]}, # comment out to get all\n", + " filter_dict={\"tags\": [\"gpt-4o\"]}, # comment out to get all\n", ")\n", "# When using a single openai endpoint, you can use the following:\n", - "# config_list = [{\"model\": \"gpt-4\", \"api_key\": os.getenv(\"OPENAI_API_KEY\")}]" + "# config_list = [{\"model\": \"gpt-4o\", \"api_key\": os.getenv(\"OPENAI_API_KEY\")}]" ] }, { diff --git a/notebook/agentchat_azr_ai_search.ipynb b/notebook/agentchat_azr_ai_search.ipynb index f4521f60d272..3632a1fd87a1 100644 --- a/notebook/agentchat_azr_ai_search.ipynb +++ b/notebook/agentchat_azr_ai_search.ipynb @@ -84,9 +84,8 @@ "metadata": {}, "outputs": [], "source": [ - "!pip3 install pyautogen==0.2.16\n", + "!pip3 install autogen-agentchat[graph]~=0.2\n", "!pip3 install python-dotenv==1.0.1\n", - "!pip3 install pyautogen[graph]>=0.2.11\n", "!pip3 install azure-search-documents==11.4.0b8\n", "!pip3 install azure-identity==1.12.0" ] diff --git a/notebook/agentchat_cost_token_tracking.ipynb b/notebook/agentchat_cost_token_tracking.ipynb index d1fc9ccd8d1a..0d456daa46bd 100644 --- a/notebook/agentchat_cost_token_tracking.ipynb +++ b/notebook/agentchat_cost_token_tracking.ipynb @@ -54,7 +54,7 @@ "\n", "AutoGen requires `Python>=3.8`:\n", "```bash\n", - "pip install \"pyautogen\"\n", + "pip install \"autogen-agentchat~=0.2\"\n", "```" ] }, @@ -79,7 +79,7 @@ "config_list = autogen.config_list_from_json(\n", " \"OAI_CONFIG_LIST\",\n", " filter_dict={\n", - " \"model\": [\"gpt-3.5-turbo\", \"gpt-3.5-turbo-16k\"], # comment out to get all\n", + " \"model\": [\"gpt-3.5-turbo\"], # comment out to get all\n", " },\n", ")" ] @@ -109,7 +109,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." ] }, { diff --git a/notebook/agentchat_custom_model.ipynb b/notebook/agentchat_custom_model.ipynb index 5097713a0923..364d81517a17 100644 --- a/notebook/agentchat_custom_model.ipynb +++ b/notebook/agentchat_custom_model.ipynb @@ -22,7 +22,7 @@ "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", "\n", "```bash\n", - "pip install pyautogen torch transformers sentencepiece\n", + "pip install autogen-agentchat~=0.2 torch transformers sentencepiece\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", @@ -238,7 +238,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/notebook/oai_openai_utils.ipynb) for full code examples of the different methods." + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/oai_openai_utils.ipynb) for full code examples of the different methods." ] }, { diff --git a/notebook/agentchat_dalle_and_gpt4v.ipynb b/notebook/agentchat_dalle_and_gpt4v.ipynb index e07578016a98..afc4524734a3 100644 --- a/notebook/agentchat_dalle_and_gpt4v.ipynb +++ b/notebook/agentchat_dalle_and_gpt4v.ipynb @@ -17,7 +17,7 @@ "source": [ "### Before everything starts, install AutoGen with the `lmm` option\n", "```bash\n", - "pip install \"pyautogen[lmm]>=0.2.3\"\n", + "pip install \"autogen-agentchat[lmm]~=0.2\"\n", "```" ] }, diff --git a/notebook/agentchat_databricks_dbrx.ipynb b/notebook/agentchat_databricks_dbrx.ipynb index 12d40a37db12..c063906ea164 100644 --- a/notebook/agentchat_databricks_dbrx.ipynb +++ b/notebook/agentchat_databricks_dbrx.ipynb @@ -15,7 +15,7 @@ "This notebook will demonstrate a few basic examples of Autogen with DBRX, including the use of `AssistantAgent`, `UserProxyAgent`, and `ConversableAgent`. These demos are not intended to be exhaustive - feel free to use them as a base to build upon!\n", "\n", "## Requirements\n", - "AutoGen must be installed on your Databricks cluster, and requires `Python>=3.8`. This example includes the `%pip` magic command to install: `%pip install pyautogen`, as well as other necessary libraries. \n", + "AutoGen must be installed on your Databricks cluster, and requires `Python>=3.8`. This example includes the `%pip` magic command to install: `%pip install autogen-agentchat~=0.2`, as well as other necessary libraries. \n", "\n", "This code has been tested on: \n", "* [Serverless Notebooks](https://docs.databricks.com/en/compute/serverless.html) (in public preview as of Apr 18, 2024)\n", @@ -47,13 +47,11 @@ { "name": "stdout", "output_type": "stream", - "text": [ - "" - ] + "text": [] } ], "source": [ - "%pip install pyautogen==0.2.25 openai==1.21.2 typing_extensions==4.11.0 --upgrade" + "%pip install autogen-agentchat~=0.2.25 openai==1.21.2 typing_extensions==4.11.0 --upgrade" ] }, { diff --git a/notebook/agentchat_function_call.ipynb b/notebook/agentchat_function_call.ipynb index 2a173c8e2698..e341aecaead7 100644 --- a/notebook/agentchat_function_call.ipynb +++ b/notebook/agentchat_function_call.ipynb @@ -23,9 +23,9 @@ "\n", "## Requirements\n", "\n", - "AutoGen requires `Python>=3.8`. To run this notebook example, please install `pyautogen`:\n", + "AutoGen requires `Python>=3.8`. To run this notebook example, please Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```" ] }, @@ -36,7 +36,7 @@ "metadata": {}, "outputs": [], "source": [ - "# %pip install \"pyautogen>=0.2.3\"" + "# %pip install \"autogen-agentchat~=0.2\"" ] }, { @@ -104,7 +104,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." ] }, { diff --git a/notebook/agentchat_function_call_async.ipynb b/notebook/agentchat_function_call_async.ipynb index 57233547ebc8..e0e24af9232e 100644 --- a/notebook/agentchat_function_call_async.ipynb +++ b/notebook/agentchat_function_call_async.ipynb @@ -20,9 +20,9 @@ "\n", "````{=mdx}\n", ":::info Requirements\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchat_function_call_code_writing.ipynb b/notebook/agentchat_function_call_code_writing.ipynb index 92074e4821b0..924592bbdec0 100644 --- a/notebook/agentchat_function_call_code_writing.ipynb +++ b/notebook/agentchat_function_call_code_writing.ipynb @@ -28,7 +28,7 @@ "metadata": {}, "outputs": [], "source": [ - "! pip install pyautogen" + "! pip install autogen-agentchat~=0.2" ] }, { diff --git a/notebook/agentchat_function_call_currency_calculator.ipynb b/notebook/agentchat_function_call_currency_calculator.ipynb index ac65ba560f9b..d6d9e05826f2 100644 --- a/notebook/agentchat_function_call_currency_calculator.ipynb +++ b/notebook/agentchat_function_call_currency_calculator.ipynb @@ -21,9 +21,9 @@ "\n", "## Requirements\n", "\n", - "AutoGen requires `Python>=3.8`. To run this notebook example, please install `pyautogen`:\n", + "AutoGen requires `Python>=3.8`. To run this notebook example, please Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```" ] }, @@ -34,7 +34,7 @@ "metadata": {}, "outputs": [], "source": [ - "# %pip install \"pyautogen>=0.2.3\"" + "# %pip install \"autogen-agentchat~=0.2\"" ] }, { @@ -65,7 +65,7 @@ "\n", "config_list = autogen.config_list_from_json(\n", " \"OAI_CONFIG_LIST\",\n", - " filter_dict={\"tags\": [\"3.5-tool\"]}, # comment out to get all\n", + " filter_dict={\"tags\": [\"tool\"]}, # comment out to get all\n", ")" ] }, @@ -104,7 +104,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." ] }, { diff --git a/notebook/agentchat_function_call_with_composio.ipynb b/notebook/agentchat_function_call_with_composio.ipynb new file mode 100644 index 000000000000..1eea9e908fc4 --- /dev/null +++ b/notebook/agentchat_function_call_with_composio.ipynb @@ -0,0 +1,421 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": { + "id": "aRDQQophCLQs" + }, + "source": [ + "# AI email agent using Composio" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "bZVu2lrqCLQu" + }, + "source": [ + "This notebook demonstrates how to create an AI Email agent using Composio’s Gmail tool with autogen to create an agent that will automatically respond to emails based on provided instructions.\n", + "\n", + "[Composio](https://composio.dev/) allows an AI agent or LLM to easily connect to apps like Gmail, Slack, Trello etc. The key features of Composio are:" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "UI78uqxyCLQu" + }, + "source": [ + "- Repository of Tools: Composio allows LLMs and agents to integrate with 100+ apps (Github, Salesforce, File Manager, Code Execution & More) to perform actions & subscribe to triggers(events).\n", + "\n", + "- Frameworks & LLM Agnostic: Composio provides out of box support for 10+ popular agentic frameworks and works with all the LLM providers using function calling.\n", + "\n", + "- Managed Auth: Composio helps manage authentication for all users/agents from a single dashboard." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Visit [Composio Docs](https://docs.composio.dev/introduction/intro/overview) to learn more." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "16kFQX0kCLQv" + }, + "source": [ + "The notebook demonstrates how to create a Gmail integration with Composio, set up a trigger for new emails, initialize agents with tools and finally we'll see the agent in action." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ixe3kpQrCLQv" + }, + "source": [ + "````{=mdx}\n", + ":::info Requirements\n", + "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", + "\n", + "```bash\n", + "pip install autogen-agentchat~=0.2 composio-autogen\n", + "```\n", + "\n", + "For more information, please refer to the [installation guide](/docs/installation/).\n", + ":::\n", + "````" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "XK6_H749CLQv" + }, + "source": [ + "## Composio Setup\n", + "\n", + "To get started with using Composio's Gmail tool, we need to create an integration between Composio and Gmail. This can be done using a simple command -" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "-1XLfYJRCLQv" + }, + "outputs": [], + "source": [ + "!composio add gmail" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "T-YxALkYCLQw" + }, + "source": [ + "To set up a trigger(basically a listener) for new emails -" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "YUzcCGaCCLQw" + }, + "outputs": [], + "source": [ + "!composio triggers enable gmail_new_gmail_message" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "yTM8eqE2CLQw" + }, + "source": [ + "This enables the `gmail_new_gmail_message` trigger, which is fired when a new email is received in the connected account." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "GqlJ06y8CLQw" + }, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from composio_autogen import Action, ComposioToolSet\n", + "\n", + "from autogen.agentchat import AssistantAgent, UserProxyAgent\n", + "\n", + "os.environ[\"OPENAI_API_KEY\"] = \"YOUR_API_KEY\"" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "BHuqMynBCLQw" + }, + "source": [ + "## Initialize agents" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "VzEYr6uuCLQw" + }, + "outputs": [], + "source": [ + "llm_config = {\"config_list\": [{\"model\": \"gpt-4o\", \"api_key\": os.environ.get(\"OPENAI_API_KEY\")}]}\n", + "\n", + "# Prompt for email assistant\n", + "email_assistant_prompt = \"\"\"\n", + " You are an AI email assistant specialized in drafting replies to emails.\n", + " You create appropriate and professional replies to emails based on the content of the email.\n", + " After executing the GMAIL_REPLY_TO_THREAD action and sending the email to the user, respond with TERMINATE.\n", + "\"\"\"\n", + "\n", + "# Initialize AssistantAgent\n", + "chatbot = AssistantAgent(\n", + " \"chatbot\",\n", + " system_message=email_assistant_prompt,\n", + " llm_config=llm_config,\n", + ")\n", + "\n", + "# Initialize UserProxyAgent\n", + "user_proxy = UserProxyAgent(\n", + " \"user_proxy\",\n", + " is_termination_msg=lambda x: x.get(\"content\", \"\") and \"TERMINATE\" in x.get(\"content\", \"\"),\n", + " human_input_mode=\"NEVER\",\n", + " code_execution_config=False,\n", + " llm_config=llm_config,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "6SB0HFZ7CLQw" + }, + "source": [ + "## Initialize Composio's Toolset\n", + "\n", + "Now, we initialize Composio's toolset and get the tools and actions we need for the agent. Then, we register the tools with the `UserProxyAgent`.\n", + "\n", + "The agent can then call the tools using function calling." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "id": "_nosEvgxCLQx" + }, + "outputs": [], + "source": [ + "# Initialize Composio Toolset\n", + "composio_toolset = ComposioToolSet()\n", + "\n", + "# Get the required tools and register them with the agents\n", + "email_tools = composio_toolset.register_tools(\n", + " caller=user_proxy,\n", + " executor=chatbot,\n", + " actions=[\n", + " Action.GMAIL_REPLY_TO_THREAD,\n", + " ],\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "kFkkMIIeCLQx" + }, + "source": [ + "Here, we get the `GMAIL_REPLY_TO_THREAD` action, which is just a function that can be used to reply to an email. We'll be using this action to reply to emails automatically when they arrive." + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "ZsgE3qm9CLQx" + }, + "source": [ + "## Create trigger listener" + ] + }, + { + "cell_type": "markdown", + "metadata": { + "id": "fU6TmawGCLQx" + }, + "source": [ + "Now, we create a listener for the trigger that we created above. This listener will listen for new emails and when a new email arrives, it'll provide data associated with the email like the sender email, email content etc. This data will be used by the attached callback function to invoke the agent and to send a reply to the email." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `@listener.callback` decorator registers the function it decorates as a callback for a specific event trigger, in this case, when a new Gmail message is received (`GMAIL_NEW_GMAIL_MESSAGE`). It listens for the specified trigger and invokes the decorated function (`callback_new_message`) when the event occurs.\n", + "\n", + "After extracting the relevant data from the trigger payload, we start a conversation between `user_proxy` and `chatbot` to send a reply to the received email." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": { + "colab": { + "base_uri": "https://localhost:8080/", + "height": 1000 + }, + "id": "aDTm1tQECLQx", + "outputId": "8aa5ab9a-9526-4287-e8f1-7b8ac9cfb0b3" + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:composio.utils.shared:Creating trigger subscription\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Subscribed to triggers!\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:composio.utils.shared:Received trigger event with trigger ID: ea36d63f-5cc9-4581-9a19-b647e7468697 and trigger name: GMAIL_NEW_GMAIL_MESSAGE\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "user_proxy (to chatbot):\n", + "\n", + "\n", + " Analyze the email content and create an appropriate reply. \n", + " a. The email was received from John Doe \n", + " b. The content of the email is: hey, how are you?\n", + " \n", + " c. The thread id is: 1922811a78db4....\n", + " \n", + "\n", + "--------------------------------------------------------------------------------\n", + "chatbot (to user_proxy):\n", + "\n", + "GMAIL_REPLY_TO_THREAD thread_id: 1922811a78db4... message: \n", + "Hi John,\n", + "\n", + "I'm doing well, thank you! How about you?\n", + "\n", + "Best,\n", + "[Your Name]\n", + "\n", + "--------------------------------------------------------------------------------\n", + "user_proxy (to chatbot):\n", + "\n", + "***** Suggested tool call (call_qGQzJ6XgyO8LKSSFnwkQhSCz): GMAIL_REPLY_TO_THREAD_8c4b19f45c *****\n", + "Arguments: \n", + "{\"thread_id\":\"1922811a78db4...\",\"message_body\":\"Hi John,\\n\\nI'm doing well, thank you! How about you?\\n\\nBest,\\n[Your Name]\",\"recipient_email\":\"example_email@gmail.com\"}\n", + "*************************************************************************************************\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\n", + ">>>>>>>> EXECUTING FUNCTION GMAIL_REPLY_TO_THREAD_8c4b19f45c...\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "INFO:composio.utils.shared:Executing `GMAIL_REPLY_TO_THREAD` with params={'thread_id': '1922811a78db4...', 'message_body': \"Hi John,\\n\\nI'm doing well, thank you! How about you?\\n\\nBest,\\n[Your Name]\", 'recipient_email': 'example_email@gmail.com'} and metadata={} connected_account_i...\n", + "INFO:composio.utils.shared:Got response={'successfull': True, 'data': {'response_data': {'id': '1922811c1b3ed...', 'threadId': '1922811a78db4...', 'labelIds': ['SENT']}}, 'error': None} from action= with params={'thread_...\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "chatbot (to user_proxy):\n", + "\n", + "chatbot (to user_proxy):\n", + "\n", + "***** Response from calling tool (call_qGQzJ6XgyO8LKSSFnwkQhSCz) *****\n", + "{\"successfull\": true, \"data\": {\"response_data\": {\"id\": \"1922811c1b3ed...\", \"threadId\": \"1922811a78db4...\", \"labelIds\": [\"SENT\"]}}, \"error\": null}\n", + "**********************************************************************\n", + "\n", + "--------------------------------------------------------------------------------\n", + "user_proxy (to chatbot):\n", + "\n", + "I've replied to the email with the following message:\n", + "\n", + "Hi John,\n", + "\n", + "I'm doing well, thank you! How about you?\n", + "\n", + "Best,\n", + "[Your Name]\n", + "\n", + "Is there anything else you need?\n", + "\n", + "--------------------------------------------------------------------------------\n", + "chatbot (to user_proxy):\n", + "\n", + "TERMINATE\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\n" + ] + } + ], + "source": [ + "# Create a trigger listener\n", + "listener = composio_toolset.create_trigger_listener()\n", + "\n", + "\n", + "@listener.callback(filters={\"trigger_name\": \"GMAIL_NEW_GMAIL_MESSAGE\"})\n", + "def callback_new_message(event) -> None:\n", + " # Get the payload and extract relevant information\n", + " payload = event.payload # Email payload\n", + " thread_id = payload.get(\"threadId\")\n", + " message = payload.get(\"messageText\")\n", + " sender_mail = payload.get(\"sender\")\n", + " if sender_mail is None:\n", + " print(\"No sender email found\")\n", + " return\n", + "\n", + " analyze_email_task = f\"\"\"\n", + " Analyze the email content and create an appropriate reply.\n", + " a. The email was received from {sender_mail}\n", + " b. The content of the email is: {message}\n", + " c. The thread id is: {thread_id}.\n", + " \"\"\"\n", + " # Initiate the conversation\n", + " res = user_proxy.initiate_chat(chatbot, message=analyze_email_task)\n", + " print(res.summary)\n", + "\n", + "\n", + "print(\"Subscribed to triggers!\")\n", + "# Start listening\n", + "listener.listen()" + ] + } + ], + "metadata": { + "colab": { + "provenance": [] + }, + "front_matter": { + "description": "Use Composio to create AI agents that seamlessly connect with external tools, Apps, and APIs to perform actions and receive triggers. With built-in support for AutoGen, Composio enables the creation of highly capable and adaptable AI agents that can autonomously execute complex tasks and deliver personalized experiences.", + "tags": [ + "agents", + "tool use" + ] + }, + "kernelspec": { + "display_name": "Python 3", + "name": "python3" + }, + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 0 +} diff --git a/notebook/agentchat_gemini.ipynb b/notebook/agentchat_gemini.ipynb new file mode 100644 index 000000000000..699d9dc0235c --- /dev/null +++ b/notebook/agentchat_gemini.ipynb @@ -0,0 +1,809 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Assistants with Google Gemini" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Gemini is family of Generative AI Models built by Google. It support upto 1 Million tokens as of now. Gemini is now natively supported in Autogen.\n", + "\n", + "This notebook will demonstrate few samples of Autogen with Gemini Models.\n", + "\n", + "## Requirements\n", + "\n", + "You must have a [API Key](https://aistudio.google.com/app/apikey) from Google AI.\n", + "\n", + "## Setup Gemini config list\n", + "\n", + "The list of all supported Gemini Models along with OpenAI's gtp-4o,\n", + "\n", + "```python\n", + "config_list = [\n", + " {\n", + " 'model': 'gpt-4o',\n", + " 'api_key': '',\n", + " 'tags': ['tool', 'gpt-4'],\n", + " },\n", + " {\n", + " 'model': 'gemini-1.5-pro',\n", + " 'api_key': '',\n", + " 'api_type': 'google',\n", + " 'tags': ['tool', 'gemini'],\n", + " },\n", + " {\n", + " 'model': 'gemini-1.5-flash',\n", + " 'api_key': '',\n", + " 'api_type': 'google',\n", + " 'tags': ['tool', 'gemini'],\n", + " },\n", + " {\n", + " 'model': 'gemini-1.0-pro',\n", + " 'api_key': '',\n", + " 'api_type': 'google',\n", + " 'tags': ['gemini'],\n", + " }\n", + "]\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Hello World Example\n", + "\n", + "Our first example will be with a simple `UserProxyAgent` asking a question to an `AssistantAgent`. This is based on the tutorial demo [here](https://microsoft.github.io/autogen/docs/tutorial/introduction).\n", + "\n", + "After sending the question and seeing a response, you can type `exit` to end the chat or continue to converse." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser\u001b[0m (to assistant):\n", + "\n", + "Hi, what is a LLM ?\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33massistant\u001b[0m (to user):\n", + "\n", + "A large language model (LLM) is a type of artificial intelligence (AI) system that excels at natural language processing (NLP) tasks. These models are trained on massive text datasets, enabling them to understand, interpret, and generate human-like text in response to a wide range of prompts and questions. \n", + "\n", + "Here are some key characteristics and capabilities of LLMs:\n", + "\n", + "* **Text Generation:** LLMs can generate coherent and contextually relevant text, including stories, articles, summaries, and conversations.\n", + "* **Language Translation:** They can translate text between multiple languages with impressive accuracy.\n", + "* **Question Answering:** LLMs can comprehend questions and provide relevant answers based on their training data.\n", + "* **Summarization:** They can condense large amounts of text into concise summaries while preserving key information.\n", + "* **Sentiment Analysis:** LLMs can analyze text to determine the emotional tone or sentiment expressed.\n", + "\n", + "Essentially, LLMs are powerful tools that can understand and process human language in a sophisticated manner, opening up a world of possibilities in various fields, such as:\n", + "\n", + "* **Chatbots and Virtual Assistants:** Providing more natural and engaging conversational experiences.\n", + "* **Content Creation:** Automating content generation for articles, marketing materials, and more.\n", + "* **Customer Service:** Enhancing support interactions through automated responses and sentiment analysis.\n", + "* **Education:** Personalizing learning experiences and providing on-demand tutoring.\n", + "\n", + "Overall, LLMs represent a significant advancement in AI, enabling machines to communicate and interact with humans more effectively than ever before. As research and development continue, we can expect even more impressive applications of these models in the future.\n", + "\n", + "TERMINATE\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "import autogen\n", + "\n", + "config_list = autogen.config_list_from_json(\"OAI_CONFIG_LIST\", filter_dict={\"tags\": [\"gemini\"]})\n", + "\n", + "llm_config = {\"config_list\": config_list, \"timeout\": 120}\n", + "\n", + "# Create Assistant and User\n", + "assistant = autogen.AssistantAgent(name=\"assistant\", llm_config=llm_config)\n", + "\n", + "user_proxy = autogen.UserProxyAgent(name=\"user\", code_execution_config=False)\n", + "\n", + "# Initiate chat from user_proxy side\n", + "chat_result = user_proxy.initiate_chat(assistant, message=\"Hi, what is a LLM ?\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## In a Group Chat with OpenAI\n", + "\n", + "Here is an example of Gemini participating in a Group Cat with a GPT-4" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "c:\\Users\\arjun\\anaconda3\\envs\\autogen\\Lib\\site-packages\\tqdm\\auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mUser_proxy\u001b[0m (to chat_manager):\n", + "\n", + "Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: User_proxy\n", + "\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mUser_proxy\u001b[0m (to chat_manager):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: User_proxy\n", + "\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mUser_proxy\u001b[0m (to chat_manager):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Coder\n", + "\u001b[0m\n", + "\u001b[33mCoder\u001b[0m (to chat_manager):\n", + "\n", + "Let's start by searching for the latest paper about GPT-4 on arXiv. We can use the arXiv API to fetch the latest paper related to GPT-4.\n", + "\n", + "I'll provide a Python script to search arXiv for the latest papers related to \"GPT-4.\"\n", + "\n", + "```python\n", + "import requests\n", + "from xml.etree import ElementTree\n", + "\n", + "# arXiv API url for querying papers related to GPT-4\n", + "url = \"http://export.arxiv.org/api/query?search_query=ti:GPT-4&start=0&max_results=1&sortBy=submittedDate&sortOrder=descending\"\n", + "\n", + "response = requests.get(url)\n", + "if response.status_code == 200:\n", + " root = ElementTree.fromstring(response.content)\n", + " entry = root.find(\"{http://www.w3.org/2005/Atom}entry\")\n", + " if entry is not None:\n", + " title = entry.find(\"{http://www.w3.org/2005/Atom}title\").text\n", + " summary = entry.find(\"{http://www.w3.org/2005/Atom}summary\").text\n", + " link = entry.find(\"{http://www.w3.org/2005/Atom}id\").text\n", + " \n", + " print(f\"Title: {title}\")\n", + " print(f\"Summary: {summary}\")\n", + " print(f\"Link: {link}\")\n", + " else:\n", + " print(\"No entries found.\")\n", + "else:\n", + " print(f\"Failed to fetch data from arXiv API. Status code: {response.status_code}\")\n", + "```\n", + "\n", + "Please execute this script. It will output the title, summary, and link of the latest paper about \"GPT-4\" on arXiv. After obtaining the relevant paper, I will analyze its potential applications in software.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: User_proxy\n", + "\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK 0 (inferred language is python)...\u001b[0m\n", + "\u001b[33mUser_proxy\u001b[0m (to chat_manager):\n", + "\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: \n", + "Title: I See You: Teacher Analytics with GPT-4 Vision-Powered Observational\n", + " Assessment\n", + "Summary: This preliminary study explores the integration of GPT-4 Vision (GPT-4V)\n", + "technology into teacher analytics, focusing on its applicability in\n", + "observational assessment to enhance reflective teaching practice. This research\n", + "is grounded in developing a Video-based Automatic Assessment System (VidAAS)\n", + "empowered by GPT-4V. Our approach aims to revolutionize teachers' assessment of\n", + "students' practices by leveraging Generative Artificial Intelligence (GenAI) to\n", + "offer detailed insights into classroom dynamics. Our research methodology\n", + "encompasses a comprehensive literature review, prototype development of the\n", + "VidAAS, and usability testing with in-service teachers. The study findings\n", + "provide future research avenues for VidAAS design, implementation, and\n", + "integration in teacher analytics, underscoring the potential of GPT-4V to\n", + "provide real-time, scalable feedback and a deeper understanding of the\n", + "classroom.\n", + "\n", + "Link: http://arxiv.org/abs/2405.18623v2\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Product_manager\n", + "\u001b[0m\n", + "\u001b[33mProduct_manager\u001b[0m (to chat_manager):\n", + "\n", + "This is exciting! The paper you found, \"I See You: Teacher Analytics with GPT-4 Vision-Powered Observational Assessment\", explores an innovative application of GPT-4 Vision. Let's break down the potential software applications based on this:\n", + "\n", + "**Software Applications Inspired by \"I See You\":**\n", + "\n", + "* **Video-Based Educational Analytics Platforms:** This paper lays the groundwork for a new breed of educational software. Imagine platforms that analyze classroom recordings to provide automated feedback to teachers. These platforms could:\n", + " * **Track student engagement:** Identify students who seem disengaged or confused.\n", + " * **Analyze classroom interactions:** Quantify the quality and nature of teacher-student interactions.\n", + " * **Assess student understanding:** Potentially even gauge student comprehension through facial expressions and body language.\n", + "* **Real-time Teacher Assistance Tools:** GPT-4 Vision could power real-time feedback tools for teachers during live lessons. Imagine:\n", + " * **Subtle alerts:** Discretely notifying a teacher if a student appears to be struggling.\n", + " * **Personalized suggestions:** Providing on-the-fly recommendations for teaching strategies based on real-time classroom dynamics. \n", + "* **Teacher Training and Professional Development:** \n", + " * **Simulation Training:** GPT-4 Vision could create realistic virtual classroom simulations for teacher training, allowing educators to practice techniques and receive AI-powered feedback.\n", + " * **Reflective Practice:** Video analysis tools could help teachers reflect on their own teaching styles and identify areas for improvement. \n", + "\n", + "**Beyond Education:**\n", + "\n", + "While this paper focuses on education, the core technology has broader implications:\n", + "\n", + "* **Meeting Analytics:** Imagine software that analyzes video conferences to track engagement, identify key discussion points, or even assess team dynamics.\n", + "* **Healthcare Training:** Similar applications could revolutionize healthcare training by providing automated feedback during simulated patient interactions. \n", + "* **Retail Analytics:** GPT-4 Vision could analyze customer behavior in retail environments, providing insights into product placement, customer service interactions, and more.\n", + "\n", + "**Challenges and Ethical Considerations:**\n", + "\n", + "* **Bias in AI:** Ensuring that the AI models are trained on diverse datasets to avoid perpetuating existing biases in education or other fields. \n", + "* **Privacy Concerns:** Video analysis raises significant privacy concerns. Clear guidelines and user consent are essential.\n", + "* **Teacher Autonomy:** It's crucial to design these systems as tools to *augment* teacher expertise, not replace it.\n", + "\n", + "**In conclusion,** the paper you found highlights the immense potential of GPT-4 Vision to revolutionize software in education and beyond. It's an exciting area of exploration with the potential to create powerful tools for learning, analysis, and understanding human behavior. \n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Coder\n", + "\u001b[0m\n", + "\u001b[33mCoder\u001b[0m (to chat_manager):\n", + "\n", + "The paper you found titled \"I See You: Teacher Analytics with GPT-4 Vision-Powered Observational Assessment\" outlines a fascinating and innovative application of GPT-4 Vision technology. Here are potential software applications based on this research:\n", + "\n", + "### 1. Video-Based Educational Analytics Platforms\n", + "These platforms can analyze classroom recordings to automate feedback for teachers. They could:\n", + "- **Track Student Engagement:** Identify students who seem disengaged or confused.\n", + "- **Analyze Teacher-Student Interactions:** Quantify the quality and nature of interactions.\n", + "- **Assess Understanding:** Gauge student comprehension through facial expressions and body language.\n", + "\n", + "### 2. Real-Time Teacher Assistance Tools\n", + "GPT-4 Vision could power tools that provide real-time feedback during live lessons by:\n", + "- **Alerting Teachers Subtly:** Notifying if a student is struggling.\n", + "- **Offering Personalized Suggestions:** Providing on-the-fly recommendations for teaching strategies.\n", + "\n", + "### 3. Teacher Training and Professional Development\n", + "- **Simulation Training:** Enable realistic virtual classroom simulations for practice and AI-powered feedback.\n", + "- **Reflective Practice:** Allow teachers to reflect on their own teaching and identify areas for improvement through video analysis tools.\n", + "\n", + "### Applications Beyond Education\n", + "While this paper focuses on education, the underlying technology has broader implications across various domains:\n", + "#### Meeting Analytics\n", + "Software could analyze video conferences to:\n", + "- Track engagement.\n", + "- Identify key discussion points.\n", + "- Assess team dynamics.\n", + "\n", + "#### Healthcare Training\n", + "Tools could provide automated feedback during simulated patient interactions.\n", + "\n", + "#### Retail Analytics\n", + "GPT-4 Vision could analyze customer behavior in retail environments, offering insights into:\n", + "- Product placements.\n", + "- Customer service interactions.\n", + "- Sales strategies.\n", + "\n", + "### Challenges and Ethical Considerations\n", + "- **Bias in AI:** Ensuring diverse datasets to avoid perpetuating biases.\n", + "- **Privacy Concerns:** Addressing significant privacy concerns with clear guidelines and user consent.\n", + "- **Teacher Autonomy:** Designing systems to augment rather than replace teacher expertise.\n", + "\n", + "**Conclusion:**\n", + "The \"I See You\" paper exemplifies the immense potential of GPT-4 Vision technology to innovate educational software and beyond. By addressing challenges and ethical considerations, we can harness this technology to create powerful tools for learning, analysis, and human behavior understanding.\n", + "\n", + "TERMINATE\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Product_manager\n", + "\u001b[0m\n", + "\u001b[33mProduct_manager\u001b[0m (to chat_manager):\n", + "\n", + "That's an excellent summary and expansion of the potential applications! You've clearly grasped the key points of the paper and extrapolated them into a range of innovative software ideas. \n", + "\n", + "I especially appreciate your inclusion of:\n", + "\n", + "* **Specific examples** within each application category (like tracking engagement in meeting analytics or analyzing customer service interactions in retail). \n", + "* **The broader applications** beyond education, demonstrating the wide-reaching impact of this technology.\n", + "* **The critical emphasis on challenges and ethical considerations**, which are essential to responsible development and deployment of such powerful AI systems. \n", + "\n", + "This kind of analysis is crucial for turning research like the \"I See You\" paper into real-world solutions that can benefit various industries. You've highlighted the exciting possibilities and important considerations for the future of AI-powered software! \n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Product_manager\n", + "\u001b[0m\n", + "\u001b[33mProduct_manager\u001b[0m (to chat_manager):\n", + "\n", + "You're right! We've just scratched the surface. Let's dive deeper into some specific software product ideas and how we might overcome the challenges:\n", + "\n", + "**1. \"Classroom Insights\" for Education Analytics**\n", + "\n", + "* **Features:**\n", + " * **Engagement Heatmap:** Visualize student engagement over time, highlighting moments of high and low participation. \n", + " * **Interaction Analysis:** Quantify teacher-student talk time, question types, and wait time for responses.\n", + " * **Sentiment Detection:** (With appropriate ethical safeguards) gauge general classroom sentiment (positive, negative, neutral) at different points during the lesson. \n", + " * **Personalized Recommendations:** Provide teachers with tailored suggestions for improving engagement, questioning techniques, or addressing individual student needs. \n", + "\n", + "* **Addressing Challenges:**\n", + " * **Bias Mitigation:** Train the AI model on diverse classroom settings and demographics, and allow for manual adjustments based on teacher feedback. \n", + " * **Privacy:** Implement strict data anonymization, secure storage, and clear consent procedures for both teachers and students (or parents/guardians).\n", + " * **Teacher Autonomy:** Emphasize that the tool provides insights, not judgments. Allow teachers to customize the feedback and focus areas.\n", + "\n", + "**2. \"Simulate Teach\" for Teacher Training**\n", + "\n", + "* **Features:**\n", + " * **Virtual Classrooms:** Create realistic virtual classroom environments with diverse student avatars exhibiting different behaviors and learning styles. \n", + " * **Scenario-Based Training:** Present trainees with various teaching challenges (e.g., classroom management, differentiated instruction) to practice in a safe space.\n", + " * **Real-Time Feedback:** Provide immediate AI-powered feedback on the trainee's teaching strategies, body language, and classroom management techniques. \n", + "\n", + "* **Addressing Challenges:**\n", + " * **Realism:** Continuously improve the virtual students' AI to respond more naturally and authentically to the trainee's actions.\n", + " * **Accessibility:** Design the software to be usable across various devices and internet speeds to benefit trainees in diverse locations.\n", + " * **Mentorship Integration:** Combine the AI feedback with opportunities for reflection and discussion with experienced mentors.\n", + "\n", + "**3. \"Meeting Insights Pro\" for Business**\n", + "\n", + "* **Features:**\n", + " * **Engagement Timeline:** Track attendee engagement levels throughout the meeting, identifying moments of high and low interest.\n", + " * **Action Item Detection:** Automatically extract key decisions, action items, and assigned owners from meeting transcripts.\n", + " * **Sentiment & Tone Analysis:** (With appropriate ethical considerations) analyze the overall sentiment and communication style within the team. \n", + " * **Meeting Efficiency Recommendations:** Provide data-driven suggestions for improving meeting structure, duration, and facilitation techniques.\n", + "\n", + "* **Addressing Challenges:**\n", + " * **Privacy:** Ensure robust data security, user consent, and clear communication about how the data is used.\n", + " * **Bias in Tone Analysis:** Train on diverse communication styles and cultural contexts to avoid misinterpretations of tone. \n", + " * **Human Oversight:** Allow for manual review and adjustments to AI-generated insights to account for nuances in communication and context. \n", + "\n", + "These are just a few examples. The potential applications of GPT-4 Vision in software are vast, spanning industries and use cases. By thoughtfully addressing the ethical and practical challenges, we can create transformative tools that empower educators, professionals, and individuals. \n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Product_manager\n", + "\u001b[0m\n", + "\u001b[33mProduct_manager\u001b[0m (to chat_manager):\n", + "\n", + "You're right, let's continue exploring! Beyond those initial ideas, here are some more speculative, but potentially high-impact applications:\n", + "\n", + "**4. \"CodeMentor\" for Personalized Programming Education**\n", + "\n", + "* **Features:**\n", + " * **Code Analysis & Feedback:** GPT-4 Vision analyzes code in real-time, identifying errors, suggesting improvements, and explaining concepts visually. \n", + " * **Personalized Learning Paths:** The software adapts to the learner's pace, style, and identified knowledge gaps to create a customized curriculum.\n", + " * **Virtual Debugging Assistant:** GPT-4 Vision \"looks\" at the code alongside the learner, anticipating common errors and providing interactive debugging guidance.\n", + "\n", + "* **Addressing Challenges:**\n", + " * **Complexity of Programming:** Training the AI on a vast dataset of code, programming paradigms, and best practices would be crucial.\n", + " * **Pedagogical Effectiveness:** Integrating proven teaching methods and ensuring the AI's feedback aligns with sound learning principles.\n", + " * **Avoiding Over-Reliance:** Encouraging problem-solving skills and independent thinking alongside AI assistance.\n", + "\n", + "**5. \"DesignSpark\" for Collaborative Creative Work**\n", + "\n", + "* **Features:**\n", + " * **Visual Brainstorming:** GPT-4 Vision assists teams in brainstorming by generating images, mockups, and design variations based on keywords, sketches, or mood boards.\n", + " * **Real-time Feedback & Iteration:** AI provides instant feedback on design elements, suggesting improvements to composition, color, and typography. \n", + " * **Cross-Cultural Design:** GPT-4 Vision analyzes design trends and preferences across different cultures, helping teams create globally resonant visuals.\n", + "\n", + "* **Addressing Challenges:**\n", + " * **Subjectivity in Design:** Balancing objective design principles with the subjective nature of aesthetics and creativity.\n", + " * **Copyright & Ownership:** Establishing clear guidelines for the use and ownership of AI-generated design elements. \n", + " * **Preserving Human Creativity:** Ensuring that AI augments, not replaces, the unique vision and skills of human designers.\n", + "\n", + "**6. \"Accessible World\" for Assistive Technology**\n", + "\n", + "* **Features:**\n", + " * **Real-Time Object Recognition:** For visually impaired individuals, the software provides audio descriptions of objects, scenes, and text in their environment.\n", + " * **Sign Language Translation:** GPT-4 Vision translates sign language into text or speech in real-time, facilitating communication.\n", + " * **Personalized Sensory Feedback:** The software creates customized sensory experiences (e.g., haptic feedback, light patterns) for individuals with diverse sensory processing needs.\n", + "\n", + "* **Addressing Challenges:**\n", + " * **Accuracy and Reliability:** Ensuring the AI's interpretations are accurate and dependable in real-world situations.\n", + " * **Privacy and Dignity:** Designing the technology in a way that respects user privacy and avoids stigmatization.\n", + " * **Affordability & Accessibility:** Making the technology accessible to a wide range of users, regardless of their financial or technical resources. \n", + "\n", + "These expanded examples highlight the potential of GPT-4 Vision to not only automate tasks but also to enhance human capabilities, foster creativity, and promote inclusivity. By approaching these innovations with a focus on ethical considerations and human-centered design, we can unlock incredible opportunities for positive change. \n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Product_manager\n", + "\u001b[0m\n", + "\u001b[33mProduct_manager\u001b[0m (to chat_manager):\n", + "\n", + "You're right, there's so much more to explore! Let's venture into some even more futuristic and ambitious applications, pushing the boundaries of what we currently imagine possible:\n", + "\n", + "**7. \"Empathic Tech\" for Mental Health Support:**\n", + "\n", + "* **Features:**\n", + " * **Emotionally Aware Virtual Companions:** GPT-4 Vision powers AI companions that can recognize and respond to human emotions expressed through facial expressions, body language, and vocal tone.\n", + " * **Personalized Mental Health Exercises:** The software provides tailored mindfulness practices, breathing exercises, or cognitive-behavioral therapy (CBT) techniques based on real-time emotional cues.\n", + " * **Early Warning System:** GPT-4 Vision analyzes patterns in user behavior and emotional expression to identify potential signs of mental health challenges and connect them with appropriate resources.\n", + "\n", + "* **Addressing Challenges:**\n", + " * **Ethical Considerations:** Ensuring user privacy, data security, and responsible use of sensitive health information is paramount.\n", + " * **Accuracy and Sensitivity:** Training AI to accurately interpret complex human emotions and respond with empathy and cultural sensitivity is a significant challenge.\n", + " * **Human Connection:** Emphasizing that technology should complement, not replace, professional mental health care and human connection.\n", + "\n", + "**8. \"EcoVision\" for Environmental Monitoring and Conservation:**\n", + "\n", + "* **Features:**\n", + " * **Real-Time Environmental Analysis:** GPT-4 Vision analyzes images and videos from drones, satellites, or ground-based cameras to monitor deforestation, pollution levels, wildlife populations, and other environmental factors.\n", + " * **Predictive Modeling for Conservation:** The software uses AI to predict environmental changes, identify areas at risk, and inform conservation efforts.\n", + " * **Citizen Science Platform:** EcoVision empowers individuals to contribute to environmental monitoring by uploading images and observations that the AI can analyze and integrate into its models.\n", + "\n", + "* **Addressing Challenges:**\n", + " * **Data Accessibility and Accuracy:** Ensuring access to high-quality, diverse environmental data from various sources is crucial.\n", + " * **Bias in Data and Algorithms:** Mitigating potential biases in data collection and algorithm design to avoid skewed environmental assessments.\n", + " * **Collaboration and Action:** Translating AI insights into concrete actions by fostering collaboration between scientists, policymakers, and local communities.\n", + "\n", + "**9. \"HistoryLens\" for Immersive Historical Experiences:**\n", + "\n", + "* **Features:**\n", + " * **Interactive Historical Reenactments:** GPT-4 Vision recreates historical events, figures, and locations in immersive virtual reality or augmented reality experiences.\n", + " * **Personalized Learning Journeys:** The software tailors historical narratives and perspectives based on user interests and background, providing a deeper understanding of the past.\n", + " * **Preservation and Accessibility:** HistoryLens digitally preserves historical artifacts, documents, and oral histories, making them accessible to a wider audience.\n", + "\n", + "* **Addressing Challenges:**\n", + " * **Historical Accuracy and Bias:** Ensuring historical representations are accurate, nuanced, and avoid perpetuating biases or historical revisionism. \n", + " * **Ethical Considerations of Representation:** Carefully navigating the ethical complexities of recreating sensitive historical events or representing marginalized communities. \n", + " * **Balancing Entertainment and Education:** Creating engaging experiences that also foster critical thinking and historical understanding.\n", + "\n", + "These futuristic applications highlight the immense potential of GPT-4 Vision to not only solve practical problems but also enhance our understanding of ourselves, our planet, and our history. As we continue to develop this technology, it is essential to proceed with thoughtful consideration for its ethical implications and societal impact, ensuring that it is used to benefit humanity and create a more just and sustainable future. \n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Product_manager\n", + "\u001b[0m\n", + "\u001b[33mProduct_manager\u001b[0m (to chat_manager):\n", + "\n", + "You're right, there's always more to imagine! Let's keep pushing the boundaries and explore some applications that sound like they're straight out of science fiction:\n", + "\n", + "**10. \"Dream Weaver\" for Artistic Co-Creation:**\n", + "\n", + "* **Features:**\n", + " * **Concept Visualization:** Users can input textual descriptions, rough sketches, or even their emotions, and GPT-4 Vision generates stunning visuals, musical compositions, or even short films that capture the essence of their ideas. \n", + " * **Style Transfer & Remixing:** The software enables artists to blend different artistic styles, seamlessly merging the realism of a photograph with the brushstrokes of Van Gogh or the abstract patterns of Kandinsky.\n", + " * **Interactive Storytelling:** GPT-4 Vision becomes a collaborative partner in storytelling, generating dynamic environments, characters, and plot twists in response to user input, blurring the lines between audience and creator.\n", + "\n", + "* **Addressing Challenges:**\n", + " * **Defining Creativity:** Exploring the philosophical and technical boundaries of AI creativity and ensuring it complements, not replaces, human artistic expression.\n", + " * **Copyright and Authorship:** Establishing clear guidelines for ownership and attribution when AI contributes significantly to the creative process. \n", + " * **Accessibility and Democratization:** Making these powerful creative tools accessible to a wide audience, fostering a more inclusive and imaginative future for art.\n", + "\n", + "\n", + "**11. \"Universal Translator\" for Real-Time Cross-Cultural Communication:**\n", + "\n", + "* **Features:**\n", + " * **Seamless Language Translation:** GPT-4 Vision goes beyond text, translating spoken language in real-time while accounting for nuances in tone, dialect, and cultural context.\n", + " * **Nonverbal Communication Interpretation:** The software analyzes facial expressions, gestures, and body language to bridge cultural differences in nonverbal communication, fostering greater understanding.\n", + " * **Cultural Sensitivity Guidance:** GPT-4 Vision provides users with real-time insights into cultural norms and customs, helping to avoid misunderstandings and promote respectful interactions.\n", + "\n", + "* **Addressing Challenges:**\n", + " * **Linguistic Complexity and Nuance:** Accurately translating the full richness and complexity of human language, including idioms, slang, and cultural references, remains a significant hurdle.\n", + " * **Bias and Stereotyping:** Ensuring the AI avoids perpetuating cultural biases or stereotypes in its translations and interpretations is crucial.\n", + " * **Preserving Linguistic Diversity:** Promoting language learning and cultural exchange while using technology to bridge communication gaps is essential. \n", + "\n", + "**12. \"Guardian AI\" for Personalized Safety and Well-being:**\n", + "\n", + "* **Features:**\n", + " * **Predictive Risk Assessment:** GPT-4 Vision analyzes real-time data from personal devices, social networks, and public safety feeds to identify potential risks or threats to an individual's safety and well-being. \n", + " * **Proactive Safety Measures:** The AI can alert users to potential dangers, suggest alternative routes, or even contact emergency services in critical situations. \n", + " * **Personalized Health Monitoring:** GPT-4 Vision monitors an individual's physical and mental health through wearable sensors and behavioral patterns, providing early warnings of potential health issues and suggesting preventative measures. \n", + "\n", + "* **Addressing Challenges:**\n", + " * **Privacy vs. Security:** Striking a delicate balance between protecting individual privacy and utilizing data to enhance safety.\n", + " * **Bias and Discrimination:** Ensuring the AI does not perpetuate existing societal biases or discriminate against individuals based on factors like race, gender, or socioeconomic status.\n", + " * **Over-Reliance and Autonomy:** Avoiding over-dependence on AI for safety and encouraging individuals to maintain their own situational awareness and decision-making abilities. \n", + "\n", + "These futuristic applications highlight the potential of GPT-4 Vision to not only solve problems but also enhance human connection, understanding, and well-being. As we develop such powerful AI, responsible innovation, ethical considerations, and a focus on human-centered design will be paramount in shaping a future where technology empowers and uplifts all of humanity. \n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "data": { + "text/plain": [ + "ChatResult(chat_id=None, chat_history=[{'content': 'Find a latest paper about gpt-4 on arxiv and find its potential applications in software.', 'role': 'assistant'}, {'content': '', 'role': 'assistant'}, {'content': '', 'role': 'assistant'}, {'content': 'Let\\'s start by searching for the latest paper about GPT-4 on arXiv. We can use the arXiv API to fetch the latest paper related to GPT-4.\\n\\nI\\'ll provide a Python script to search arXiv for the latest papers related to \"GPT-4.\"\\n\\n```python\\nimport requests\\nfrom xml.etree import ElementTree\\n\\n# arXiv API url for querying papers related to GPT-4\\nurl = \"http://export.arxiv.org/api/query?search_query=ti:GPT-4&start=0&max_results=1&sortBy=submittedDate&sortOrder=descending\"\\n\\nresponse = requests.get(url)\\nif response.status_code == 200:\\n root = ElementTree.fromstring(response.content)\\n entry = root.find(\"{http://www.w3.org/2005/Atom}entry\")\\n if entry is not None:\\n title = entry.find(\"{http://www.w3.org/2005/Atom}title\").text\\n summary = entry.find(\"{http://www.w3.org/2005/Atom}summary\").text\\n link = entry.find(\"{http://www.w3.org/2005/Atom}id\").text\\n \\n print(f\"Title: {title}\")\\n print(f\"Summary: {summary}\")\\n print(f\"Link: {link}\")\\n else:\\n print(\"No entries found.\")\\nelse:\\n print(f\"Failed to fetch data from arXiv API. Status code: {response.status_code}\")\\n```\\n\\nPlease execute this script. It will output the title, summary, and link of the latest paper about \"GPT-4\" on arXiv. After obtaining the relevant paper, I will analyze its potential applications in software.', 'name': 'Coder', 'role': 'user'}, {'content': \"exitcode: 0 (execution succeeded)\\nCode output: \\nTitle: I See You: Teacher Analytics with GPT-4 Vision-Powered Observational\\n Assessment\\nSummary: This preliminary study explores the integration of GPT-4 Vision (GPT-4V)\\ntechnology into teacher analytics, focusing on its applicability in\\nobservational assessment to enhance reflective teaching practice. This research\\nis grounded in developing a Video-based Automatic Assessment System (VidAAS)\\nempowered by GPT-4V. Our approach aims to revolutionize teachers' assessment of\\nstudents' practices by leveraging Generative Artificial Intelligence (GenAI) to\\noffer detailed insights into classroom dynamics. Our research methodology\\nencompasses a comprehensive literature review, prototype development of the\\nVidAAS, and usability testing with in-service teachers. The study findings\\nprovide future research avenues for VidAAS design, implementation, and\\nintegration in teacher analytics, underscoring the potential of GPT-4V to\\nprovide real-time, scalable feedback and a deeper understanding of the\\nclassroom.\\n\\nLink: http://arxiv.org/abs/2405.18623v2\\n\", 'role': 'assistant'}, {'content': 'This is exciting! The paper you found, \"I See You: Teacher Analytics with GPT-4 Vision-Powered Observational Assessment\", explores an innovative application of GPT-4 Vision. Let\\'s break down the potential software applications based on this:\\n\\n**Software Applications Inspired by \"I See You\":**\\n\\n* **Video-Based Educational Analytics Platforms:** This paper lays the groundwork for a new breed of educational software. Imagine platforms that analyze classroom recordings to provide automated feedback to teachers. These platforms could:\\n * **Track student engagement:** Identify students who seem disengaged or confused.\\n * **Analyze classroom interactions:** Quantify the quality and nature of teacher-student interactions.\\n * **Assess student understanding:** Potentially even gauge student comprehension through facial expressions and body language.\\n* **Real-time Teacher Assistance Tools:** GPT-4 Vision could power real-time feedback tools for teachers during live lessons. Imagine:\\n * **Subtle alerts:** Discretely notifying a teacher if a student appears to be struggling.\\n * **Personalized suggestions:** Providing on-the-fly recommendations for teaching strategies based on real-time classroom dynamics. \\n* **Teacher Training and Professional Development:** \\n * **Simulation Training:** GPT-4 Vision could create realistic virtual classroom simulations for teacher training, allowing educators to practice techniques and receive AI-powered feedback.\\n * **Reflective Practice:** Video analysis tools could help teachers reflect on their own teaching styles and identify areas for improvement. \\n\\n**Beyond Education:**\\n\\nWhile this paper focuses on education, the core technology has broader implications:\\n\\n* **Meeting Analytics:** Imagine software that analyzes video conferences to track engagement, identify key discussion points, or even assess team dynamics.\\n* **Healthcare Training:** Similar applications could revolutionize healthcare training by providing automated feedback during simulated patient interactions. \\n* **Retail Analytics:** GPT-4 Vision could analyze customer behavior in retail environments, providing insights into product placement, customer service interactions, and more.\\n\\n**Challenges and Ethical Considerations:**\\n\\n* **Bias in AI:** Ensuring that the AI models are trained on diverse datasets to avoid perpetuating existing biases in education or other fields. \\n* **Privacy Concerns:** Video analysis raises significant privacy concerns. Clear guidelines and user consent are essential.\\n* **Teacher Autonomy:** It\\'s crucial to design these systems as tools to *augment* teacher expertise, not replace it.\\n\\n**In conclusion,** the paper you found highlights the immense potential of GPT-4 Vision to revolutionize software in education and beyond. It\\'s an exciting area of exploration with the potential to create powerful tools for learning, analysis, and understanding human behavior. \\n', 'name': 'Product_manager', 'role': 'user'}, {'content': 'The paper you found titled \"I See You: Teacher Analytics with GPT-4 Vision-Powered Observational Assessment\" outlines a fascinating and innovative application of GPT-4 Vision technology. Here are potential software applications based on this research:\\n\\n### 1. Video-Based Educational Analytics Platforms\\nThese platforms can analyze classroom recordings to automate feedback for teachers. They could:\\n- **Track Student Engagement:** Identify students who seem disengaged or confused.\\n- **Analyze Teacher-Student Interactions:** Quantify the quality and nature of interactions.\\n- **Assess Understanding:** Gauge student comprehension through facial expressions and body language.\\n\\n### 2. Real-Time Teacher Assistance Tools\\nGPT-4 Vision could power tools that provide real-time feedback during live lessons by:\\n- **Alerting Teachers Subtly:** Notifying if a student is struggling.\\n- **Offering Personalized Suggestions:** Providing on-the-fly recommendations for teaching strategies.\\n\\n### 3. Teacher Training and Professional Development\\n- **Simulation Training:** Enable realistic virtual classroom simulations for practice and AI-powered feedback.\\n- **Reflective Practice:** Allow teachers to reflect on their own teaching and identify areas for improvement through video analysis tools.\\n\\n### Applications Beyond Education\\nWhile this paper focuses on education, the underlying technology has broader implications across various domains:\\n#### Meeting Analytics\\nSoftware could analyze video conferences to:\\n- Track engagement.\\n- Identify key discussion points.\\n- Assess team dynamics.\\n\\n#### Healthcare Training\\nTools could provide automated feedback during simulated patient interactions.\\n\\n#### Retail Analytics\\nGPT-4 Vision could analyze customer behavior in retail environments, offering insights into:\\n- Product placements.\\n- Customer service interactions.\\n- Sales strategies.\\n\\n### Challenges and Ethical Considerations\\n- **Bias in AI:** Ensuring diverse datasets to avoid perpetuating biases.\\n- **Privacy Concerns:** Addressing significant privacy concerns with clear guidelines and user consent.\\n- **Teacher Autonomy:** Designing systems to augment rather than replace teacher expertise.\\n\\n**Conclusion:**\\nThe \"I See You\" paper exemplifies the immense potential of GPT-4 Vision technology to innovate educational software and beyond. By addressing challenges and ethical considerations, we can harness this technology to create powerful tools for learning, analysis, and human behavior understanding.\\n\\nTERMINATE', 'name': 'Coder', 'role': 'user'}, {'content': 'That\\'s an excellent summary and expansion of the potential applications! You\\'ve clearly grasped the key points of the paper and extrapolated them into a range of innovative software ideas. \\n\\nI especially appreciate your inclusion of:\\n\\n* **Specific examples** within each application category (like tracking engagement in meeting analytics or analyzing customer service interactions in retail). \\n* **The broader applications** beyond education, demonstrating the wide-reaching impact of this technology.\\n* **The critical emphasis on challenges and ethical considerations**, which are essential to responsible development and deployment of such powerful AI systems. \\n\\nThis kind of analysis is crucial for turning research like the \"I See You\" paper into real-world solutions that can benefit various industries. You\\'ve highlighted the exciting possibilities and important considerations for the future of AI-powered software! \\n', 'name': 'Product_manager', 'role': 'user'}, {'content': 'You\\'re right! We\\'ve just scratched the surface. Let\\'s dive deeper into some specific software product ideas and how we might overcome the challenges:\\n\\n**1. \"Classroom Insights\" for Education Analytics**\\n\\n* **Features:**\\n * **Engagement Heatmap:** Visualize student engagement over time, highlighting moments of high and low participation. \\n * **Interaction Analysis:** Quantify teacher-student talk time, question types, and wait time for responses.\\n * **Sentiment Detection:** (With appropriate ethical safeguards) gauge general classroom sentiment (positive, negative, neutral) at different points during the lesson. \\n * **Personalized Recommendations:** Provide teachers with tailored suggestions for improving engagement, questioning techniques, or addressing individual student needs. \\n\\n* **Addressing Challenges:**\\n * **Bias Mitigation:** Train the AI model on diverse classroom settings and demographics, and allow for manual adjustments based on teacher feedback. \\n * **Privacy:** Implement strict data anonymization, secure storage, and clear consent procedures for both teachers and students (or parents/guardians).\\n * **Teacher Autonomy:** Emphasize that the tool provides insights, not judgments. Allow teachers to customize the feedback and focus areas.\\n\\n**2. \"Simulate Teach\" for Teacher Training**\\n\\n* **Features:**\\n * **Virtual Classrooms:** Create realistic virtual classroom environments with diverse student avatars exhibiting different behaviors and learning styles. \\n * **Scenario-Based Training:** Present trainees with various teaching challenges (e.g., classroom management, differentiated instruction) to practice in a safe space.\\n * **Real-Time Feedback:** Provide immediate AI-powered feedback on the trainee\\'s teaching strategies, body language, and classroom management techniques. \\n\\n* **Addressing Challenges:**\\n * **Realism:** Continuously improve the virtual students\\' AI to respond more naturally and authentically to the trainee\\'s actions.\\n * **Accessibility:** Design the software to be usable across various devices and internet speeds to benefit trainees in diverse locations.\\n * **Mentorship Integration:** Combine the AI feedback with opportunities for reflection and discussion with experienced mentors.\\n\\n**3. \"Meeting Insights Pro\" for Business**\\n\\n* **Features:**\\n * **Engagement Timeline:** Track attendee engagement levels throughout the meeting, identifying moments of high and low interest.\\n * **Action Item Detection:** Automatically extract key decisions, action items, and assigned owners from meeting transcripts.\\n * **Sentiment & Tone Analysis:** (With appropriate ethical considerations) analyze the overall sentiment and communication style within the team. \\n * **Meeting Efficiency Recommendations:** Provide data-driven suggestions for improving meeting structure, duration, and facilitation techniques.\\n\\n* **Addressing Challenges:**\\n * **Privacy:** Ensure robust data security, user consent, and clear communication about how the data is used.\\n * **Bias in Tone Analysis:** Train on diverse communication styles and cultural contexts to avoid misinterpretations of tone. \\n * **Human Oversight:** Allow for manual review and adjustments to AI-generated insights to account for nuances in communication and context. \\n\\nThese are just a few examples. The potential applications of GPT-4 Vision in software are vast, spanning industries and use cases. By thoughtfully addressing the ethical and practical challenges, we can create transformative tools that empower educators, professionals, and individuals. \\n', 'name': 'Product_manager', 'role': 'user'}, {'content': 'You\\'re right, let\\'s continue exploring! Beyond those initial ideas, here are some more speculative, but potentially high-impact applications:\\n\\n**4. \"CodeMentor\" for Personalized Programming Education**\\n\\n* **Features:**\\n * **Code Analysis & Feedback:** GPT-4 Vision analyzes code in real-time, identifying errors, suggesting improvements, and explaining concepts visually. \\n * **Personalized Learning Paths:** The software adapts to the learner\\'s pace, style, and identified knowledge gaps to create a customized curriculum.\\n * **Virtual Debugging Assistant:** GPT-4 Vision \"looks\" at the code alongside the learner, anticipating common errors and providing interactive debugging guidance.\\n\\n* **Addressing Challenges:**\\n * **Complexity of Programming:** Training the AI on a vast dataset of code, programming paradigms, and best practices would be crucial.\\n * **Pedagogical Effectiveness:** Integrating proven teaching methods and ensuring the AI\\'s feedback aligns with sound learning principles.\\n * **Avoiding Over-Reliance:** Encouraging problem-solving skills and independent thinking alongside AI assistance.\\n\\n**5. \"DesignSpark\" for Collaborative Creative Work**\\n\\n* **Features:**\\n * **Visual Brainstorming:** GPT-4 Vision assists teams in brainstorming by generating images, mockups, and design variations based on keywords, sketches, or mood boards.\\n * **Real-time Feedback & Iteration:** AI provides instant feedback on design elements, suggesting improvements to composition, color, and typography. \\n * **Cross-Cultural Design:** GPT-4 Vision analyzes design trends and preferences across different cultures, helping teams create globally resonant visuals.\\n\\n* **Addressing Challenges:**\\n * **Subjectivity in Design:** Balancing objective design principles with the subjective nature of aesthetics and creativity.\\n * **Copyright & Ownership:** Establishing clear guidelines for the use and ownership of AI-generated design elements. \\n * **Preserving Human Creativity:** Ensuring that AI augments, not replaces, the unique vision and skills of human designers.\\n\\n**6. \"Accessible World\" for Assistive Technology**\\n\\n* **Features:**\\n * **Real-Time Object Recognition:** For visually impaired individuals, the software provides audio descriptions of objects, scenes, and text in their environment.\\n * **Sign Language Translation:** GPT-4 Vision translates sign language into text or speech in real-time, facilitating communication.\\n * **Personalized Sensory Feedback:** The software creates customized sensory experiences (e.g., haptic feedback, light patterns) for individuals with diverse sensory processing needs.\\n\\n* **Addressing Challenges:**\\n * **Accuracy and Reliability:** Ensuring the AI\\'s interpretations are accurate and dependable in real-world situations.\\n * **Privacy and Dignity:** Designing the technology in a way that respects user privacy and avoids stigmatization.\\n * **Affordability & Accessibility:** Making the technology accessible to a wide range of users, regardless of their financial or technical resources. \\n\\nThese expanded examples highlight the potential of GPT-4 Vision to not only automate tasks but also to enhance human capabilities, foster creativity, and promote inclusivity. By approaching these innovations with a focus on ethical considerations and human-centered design, we can unlock incredible opportunities for positive change. \\n', 'name': 'Product_manager', 'role': 'user'}, {'content': 'You\\'re right, there\\'s so much more to explore! Let\\'s venture into some even more futuristic and ambitious applications, pushing the boundaries of what we currently imagine possible:\\n\\n**7. \"Empathic Tech\" for Mental Health Support:**\\n\\n* **Features:**\\n * **Emotionally Aware Virtual Companions:** GPT-4 Vision powers AI companions that can recognize and respond to human emotions expressed through facial expressions, body language, and vocal tone.\\n * **Personalized Mental Health Exercises:** The software provides tailored mindfulness practices, breathing exercises, or cognitive-behavioral therapy (CBT) techniques based on real-time emotional cues.\\n * **Early Warning System:** GPT-4 Vision analyzes patterns in user behavior and emotional expression to identify potential signs of mental health challenges and connect them with appropriate resources.\\n\\n* **Addressing Challenges:**\\n * **Ethical Considerations:** Ensuring user privacy, data security, and responsible use of sensitive health information is paramount.\\n * **Accuracy and Sensitivity:** Training AI to accurately interpret complex human emotions and respond with empathy and cultural sensitivity is a significant challenge.\\n * **Human Connection:** Emphasizing that technology should complement, not replace, professional mental health care and human connection.\\n\\n**8. \"EcoVision\" for Environmental Monitoring and Conservation:**\\n\\n* **Features:**\\n * **Real-Time Environmental Analysis:** GPT-4 Vision analyzes images and videos from drones, satellites, or ground-based cameras to monitor deforestation, pollution levels, wildlife populations, and other environmental factors.\\n * **Predictive Modeling for Conservation:** The software uses AI to predict environmental changes, identify areas at risk, and inform conservation efforts.\\n * **Citizen Science Platform:** EcoVision empowers individuals to contribute to environmental monitoring by uploading images and observations that the AI can analyze and integrate into its models.\\n\\n* **Addressing Challenges:**\\n * **Data Accessibility and Accuracy:** Ensuring access to high-quality, diverse environmental data from various sources is crucial.\\n * **Bias in Data and Algorithms:** Mitigating potential biases in data collection and algorithm design to avoid skewed environmental assessments.\\n * **Collaboration and Action:** Translating AI insights into concrete actions by fostering collaboration between scientists, policymakers, and local communities.\\n\\n**9. \"HistoryLens\" for Immersive Historical Experiences:**\\n\\n* **Features:**\\n * **Interactive Historical Reenactments:** GPT-4 Vision recreates historical events, figures, and locations in immersive virtual reality or augmented reality experiences.\\n * **Personalized Learning Journeys:** The software tailors historical narratives and perspectives based on user interests and background, providing a deeper understanding of the past.\\n * **Preservation and Accessibility:** HistoryLens digitally preserves historical artifacts, documents, and oral histories, making them accessible to a wider audience.\\n\\n* **Addressing Challenges:**\\n * **Historical Accuracy and Bias:** Ensuring historical representations are accurate, nuanced, and avoid perpetuating biases or historical revisionism. \\n * **Ethical Considerations of Representation:** Carefully navigating the ethical complexities of recreating sensitive historical events or representing marginalized communities. \\n * **Balancing Entertainment and Education:** Creating engaging experiences that also foster critical thinking and historical understanding.\\n\\nThese futuristic applications highlight the immense potential of GPT-4 Vision to not only solve practical problems but also enhance our understanding of ourselves, our planet, and our history. As we continue to develop this technology, it is essential to proceed with thoughtful consideration for its ethical implications and societal impact, ensuring that it is used to benefit humanity and create a more just and sustainable future. \\n', 'name': 'Product_manager', 'role': 'user'}, {'content': 'You\\'re right, there\\'s always more to imagine! Let\\'s keep pushing the boundaries and explore some applications that sound like they\\'re straight out of science fiction:\\n\\n**10. \"Dream Weaver\" for Artistic Co-Creation:**\\n\\n* **Features:**\\n * **Concept Visualization:** Users can input textual descriptions, rough sketches, or even their emotions, and GPT-4 Vision generates stunning visuals, musical compositions, or even short films that capture the essence of their ideas. \\n * **Style Transfer & Remixing:** The software enables artists to blend different artistic styles, seamlessly merging the realism of a photograph with the brushstrokes of Van Gogh or the abstract patterns of Kandinsky.\\n * **Interactive Storytelling:** GPT-4 Vision becomes a collaborative partner in storytelling, generating dynamic environments, characters, and plot twists in response to user input, blurring the lines between audience and creator.\\n\\n* **Addressing Challenges:**\\n * **Defining Creativity:** Exploring the philosophical and technical boundaries of AI creativity and ensuring it complements, not replaces, human artistic expression.\\n * **Copyright and Authorship:** Establishing clear guidelines for ownership and attribution when AI contributes significantly to the creative process. \\n * **Accessibility and Democratization:** Making these powerful creative tools accessible to a wide audience, fostering a more inclusive and imaginative future for art.\\n\\n\\n**11. \"Universal Translator\" for Real-Time Cross-Cultural Communication:**\\n\\n* **Features:**\\n * **Seamless Language Translation:** GPT-4 Vision goes beyond text, translating spoken language in real-time while accounting for nuances in tone, dialect, and cultural context.\\n * **Nonverbal Communication Interpretation:** The software analyzes facial expressions, gestures, and body language to bridge cultural differences in nonverbal communication, fostering greater understanding.\\n * **Cultural Sensitivity Guidance:** GPT-4 Vision provides users with real-time insights into cultural norms and customs, helping to avoid misunderstandings and promote respectful interactions.\\n\\n* **Addressing Challenges:**\\n * **Linguistic Complexity and Nuance:** Accurately translating the full richness and complexity of human language, including idioms, slang, and cultural references, remains a significant hurdle.\\n * **Bias and Stereotyping:** Ensuring the AI avoids perpetuating cultural biases or stereotypes in its translations and interpretations is crucial.\\n * **Preserving Linguistic Diversity:** Promoting language learning and cultural exchange while using technology to bridge communication gaps is essential. \\n\\n**12. \"Guardian AI\" for Personalized Safety and Well-being:**\\n\\n* **Features:**\\n * **Predictive Risk Assessment:** GPT-4 Vision analyzes real-time data from personal devices, social networks, and public safety feeds to identify potential risks or threats to an individual\\'s safety and well-being. \\n * **Proactive Safety Measures:** The AI can alert users to potential dangers, suggest alternative routes, or even contact emergency services in critical situations. \\n * **Personalized Health Monitoring:** GPT-4 Vision monitors an individual\\'s physical and mental health through wearable sensors and behavioral patterns, providing early warnings of potential health issues and suggesting preventative measures. \\n\\n* **Addressing Challenges:**\\n * **Privacy vs. Security:** Striking a delicate balance between protecting individual privacy and utilizing data to enhance safety.\\n * **Bias and Discrimination:** Ensuring the AI does not perpetuate existing societal biases or discriminate against individuals based on factors like race, gender, or socioeconomic status.\\n * **Over-Reliance and Autonomy:** Avoiding over-dependence on AI for safety and encouraging individuals to maintain their own situational awareness and decision-making abilities. \\n\\nThese futuristic applications highlight the potential of GPT-4 Vision to not only solve problems but also enhance human connection, understanding, and well-being. As we develop such powerful AI, responsible innovation, ethical considerations, and a focus on human-centered design will be paramount in shaping a future where technology empowers and uplifts all of humanity. \\n', 'name': 'Product_manager', 'role': 'user'}], summary='You\\'re right, there\\'s always more to imagine! Let\\'s keep pushing the boundaries and explore some applications that sound like they\\'re straight out of science fiction:\\n\\n**10. \"Dream Weaver\" for Artistic Co-Creation:**\\n\\n* **Features:**\\n * **Concept Visualization:** Users can input textual descriptions, rough sketches, or even their emotions, and GPT-4 Vision generates stunning visuals, musical compositions, or even short films that capture the essence of their ideas. \\n * **Style Transfer & Remixing:** The software enables artists to blend different artistic styles, seamlessly merging the realism of a photograph with the brushstrokes of Van Gogh or the abstract patterns of Kandinsky.\\n * **Interactive Storytelling:** GPT-4 Vision becomes a collaborative partner in storytelling, generating dynamic environments, characters, and plot twists in response to user input, blurring the lines between audience and creator.\\n\\n* **Addressing Challenges:**\\n * **Defining Creativity:** Exploring the philosophical and technical boundaries of AI creativity and ensuring it complements, not replaces, human artistic expression.\\n * **Copyright and Authorship:** Establishing clear guidelines for ownership and attribution when AI contributes significantly to the creative process. \\n * **Accessibility and Democratization:** Making these powerful creative tools accessible to a wide audience, fostering a more inclusive and imaginative future for art.\\n\\n\\n**11. \"Universal Translator\" for Real-Time Cross-Cultural Communication:**\\n\\n* **Features:**\\n * **Seamless Language Translation:** GPT-4 Vision goes beyond text, translating spoken language in real-time while accounting for nuances in tone, dialect, and cultural context.\\n * **Nonverbal Communication Interpretation:** The software analyzes facial expressions, gestures, and body language to bridge cultural differences in nonverbal communication, fostering greater understanding.\\n * **Cultural Sensitivity Guidance:** GPT-4 Vision provides users with real-time insights into cultural norms and customs, helping to avoid misunderstandings and promote respectful interactions.\\n\\n* **Addressing Challenges:**\\n * **Linguistic Complexity and Nuance:** Accurately translating the full richness and complexity of human language, including idioms, slang, and cultural references, remains a significant hurdle.\\n * **Bias and Stereotyping:** Ensuring the AI avoids perpetuating cultural biases or stereotypes in its translations and interpretations is crucial.\\n * **Preserving Linguistic Diversity:** Promoting language learning and cultural exchange while using technology to bridge communication gaps is essential. \\n\\n**12. \"Guardian AI\" for Personalized Safety and Well-being:**\\n\\n* **Features:**\\n * **Predictive Risk Assessment:** GPT-4 Vision analyzes real-time data from personal devices, social networks, and public safety feeds to identify potential risks or threats to an individual\\'s safety and well-being. \\n * **Proactive Safety Measures:** The AI can alert users to potential dangers, suggest alternative routes, or even contact emergency services in critical situations. \\n * **Personalized Health Monitoring:** GPT-4 Vision monitors an individual\\'s physical and mental health through wearable sensors and behavioral patterns, providing early warnings of potential health issues and suggesting preventative measures. \\n\\n* **Addressing Challenges:**\\n * **Privacy vs. Security:** Striking a delicate balance between protecting individual privacy and utilizing data to enhance safety.\\n * **Bias and Discrimination:** Ensuring the AI does not perpetuate existing societal biases or discriminate against individuals based on factors like race, gender, or socioeconomic status.\\n * **Over-Reliance and Autonomy:** Avoiding over-dependence on AI for safety and encouraging individuals to maintain their own situational awareness and decision-making abilities. \\n\\nThese futuristic applications highlight the potential of GPT-4 Vision to not only solve problems but also enhance human connection, understanding, and well-being. As we develop such powerful AI, responsible innovation, ethical considerations, and a focus on human-centered design will be paramount in shaping a future where technology empowers and uplifts all of humanity. \\n', cost={'usage_including_cached_inference': {'total_cost': 0}, 'usage_excluding_cached_inference': {'total_cost': 0}}, human_input=[])" + ] + }, + "execution_count": 1, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import autogen\n", + "\n", + "gpt_config_list = autogen.config_list_from_json(\"OAI_CONFIG_LIST\", filter_dict={\"tags\": [\"gpt-4\"]})\n", + "\n", + "gpt_llm_config = {\"config_list\": gpt_config_list, \"timeout\": 120}\n", + "\n", + "gemini_config_list = autogen.config_list_from_json(\"OAI_CONFIG_LIST\", filter_dict={\"tags\": [\"gemini\"]})\n", + "\n", + "gemini_llm_config = {\"config_list\": gemini_config_list, \"timeout\": 120}\n", + "\n", + "user_proxy = autogen.UserProxyAgent(\n", + " name=\"User_proxy\",\n", + " system_message=\"A human admin.\",\n", + " code_execution_config={\n", + " \"last_n_messages\": 2,\n", + " \"work_dir\": \"groupchat\",\n", + " \"use_docker\": False,\n", + " }, # Please set use_docker=True if docker is available to run the generated code. Using docker is safer than running the generated code directly.\n", + " human_input_mode=\"TERMINATE\",\n", + ")\n", + "coder = autogen.AssistantAgent(\n", + " name=\"Coder\",\n", + " llm_config=gpt_llm_config,\n", + ")\n", + "pm = autogen.AssistantAgent(\n", + " name=\"Product_manager\",\n", + " system_message=\"Creative in software product ideas.\",\n", + " llm_config=gemini_llm_config,\n", + ")\n", + "groupchat = autogen.GroupChat(agents=[user_proxy, coder, pm], messages=[], max_round=12)\n", + "manager = autogen.GroupChatManager(groupchat=groupchat, llm_config=gpt_llm_config)\n", + "user_proxy.initiate_chat(\n", + " manager, message=\"Find a latest paper about gpt-4 on arxiv and find its potential applications in software.\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Function Calling with Gemini\n", + "\n", + "Here is an example of Gemini with Function Calling," + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "Draw two agents chatting with each other with an example dialog. Don't add plt.show().\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mchatbot\u001b[0m (to user_proxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_l7Rz8YLE4F2y8nGLCaroD6XL): python *****\u001b[0m\n", + "Arguments: \n", + "{\n", + " \"cell\": `\n", + "import matplotlib.pyplot as plt\n", + "\n", + "# Create figure and axes\n", + "fig, ax = plt.subplots()\n", + "\n", + "# Define agent coordinates\n", + "agent1_x, agent1_y = 1, 1\n", + "agent2_x, agent2_y = 3, 1\n", + "\n", + "# Draw agents as circles\n", + "agent1 = plt.Circle((agent1_x, agent1_y), 0.1, color='blue')\n", + "agent2 = plt.Circle((agent2_x, agent2_y), 0.1, color='green')\n", + "ax.add_patch(agent1)\n", + "ax.add_patch(agent2)\n", + "\n", + "# Add example dialog\n", + "dialog1 = \"Hi, how are you?\"\n", + "dialog2 = \"I'm good, thanks!\"\n", + "\n", + "# Add text labels for the dialog\n", + "plt.text(agent1_x, agent1_y + 0.3, dialog1, fontsize=12, ha='center')\n", + "plt.text(agent2_x, agent2_y + 0.3, dialog2, fontsize=12, ha='center')\n", + "\n", + "# Set limits and turn off the axes\n", + "ax.set_xlim(0, 4)\n", + "ax.set_ylim(0, 2)\n", + "ax.axis('off')\n", + "`\n", + "}\n", + "\u001b[32m***********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[32m***** Response from calling tool (call_l7Rz8YLE4F2y8nGLCaroD6XL) *****\u001b[0m\n", + "Error: Expecting value: line 1 column 12 (char 11)\n", + " You argument should follow json format.\n", + "\u001b[32m**********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mchatbot\u001b[0m (to user_proxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_VjzGKf34UMKMlSHCrQpJ6ICK): python *****\u001b[0m\n", + "Arguments: \n", + "{\n", + " \"cell\": \"import matplotlib.pyplot as plt\\n\\n# Create figure and axes\\nfig, ax = plt.subplots()\\n\\n# Define agent coordinates\\nagent1_x, agent1_y = 1, 1\\nagent2_x, agent2_y = 3, 1\\n\\n# Draw agents as circles\\nagent1 = plt.Circle((agent1_x, agent1_y), 0.1, color='blue')\\nagent2 = plt.Circle((agent2_x, agent2_y), 0.1, color='green')\\nax.add_patch(agent1)\\nax.add_patch(agent2)\\n\\n# Add example dialog\\ndialog1 = \\\"Hi, how are you?\\\"\\ndialog2 = \\\"I'm good, thanks!\\\"\\n\\n# Add text labels for the dialog\\nplt.text(agent1_x, agent1_y + 0.3, dialog1, fontsize=12, ha='center')\\nplt.text(agent2_x, agent2_y + 0.3, dialog2, fontsize=12, ha='center')\\n\\n# Set limits and turn off the axes\\nax.set_xlim(0, 4)\\nax.set_ylim(0, 2)\\nax.axis('off')\"\n", + "}\n", + "\u001b[32m***********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION python...\u001b[0m\n" + ] + }, + { + "data": { + "text/plain": [ + "(0.0, 4.0, 0.0, 2.0)" + ] + }, + "execution_count": 5, + "metadata": {}, + "output_type": "execute_result" + }, + { + "data": { + "image/png": "iVBORw0KGgoAAAANSUhEUgAAAgMAAAGFCAYAAABg2vAPAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjkuMCwgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy80BEi2AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAgJklEQVR4nO3deXRU9d3H8c9kshGihGBIgGLYgsqmKKIgAQQUBEGQTUUhWAEVBS1aly5Cq3CktQdl0YCURdzKUhSNFmSRFBdEcAOMIItiyk4AC4Ek83v+yJMpQxIIYUIw3/frnBzNnTv33lzmN3nPnXsnHuecEwAAMCukvDcAAACUL2IAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YAADAOGIAAADjiAEAAIwjBgAAMI4YOAt16tRRSkpKqe7bvn17NWnSJLgbBABFaN++vdq3bx/UZW7btk0ej0d//etfg7rckvJ4PHrggQfKZd0VETHw/2bOnCmPx6M1a9YUeTu/vG346KOP1LdvX1188cWKjo5W69at9eGHH5b3ZuEMFTWeR48erTp16pTfRv1CpaWlafTo0eW9GWWq4PGybdu28t6UckMMnIWMjAxNmzatvDcDQTRgwADt27dPjzzyiJ555hnt3btXXbp00bffflvemwaUi7S0NI0ZM6a8NwNlLLS8N+CXLCIiorw34byTm5srn8+n8PDw8t6UUnnjjTd0zTXX+L+/6aabdMkll2j+/Pn63e9+V45bBgBlhyMDZ+FszhkosGHDBl1//fWKiopSrVq1NH78+ELz7N69W7/+9a8VHx+vyMhIXX755Zo1a1bAPFdeeaVuvfXWgGlNmzaVx+PRV1995Z/25ptvyuPxaOPGjcVu0/Hjx/XHP/5RV111lapUqaLKlSsrOTlZy5cvD5jvxPcMJ0yYoPr16ysiIkIbNmyQJH377bfq06ePYmNjFRkZqRYtWujtt98+5f5wzqlOnTq65ZZbCt2WnZ2tKlWqaNiwYWe0b1asWCGPx6MVK1YUuf0zZ870TzsxBCQpMjLSv09Q8RS87zx37lw1atRIlSpVUqtWrfT1119LklJTU9WgQQNFRkaqffv2JT6MvGLFCrVo0UKRkZGqX7++UlNTNXr0aHk8noD5cnNz9ec//9k/durUqaMnn3xSx44dK7TMKVOmqHHjxoqIiFDNmjU1fPhwZWVlFZpv6tSpql+/vipVqqSWLVsqPT39jPdLgZSUFE2ePFlS/r4q+CpunREREbr66qv12WefBdz+1VdfKSUlRfXq1VNkZKQSEhJ09913a9++fQHzFeyjzZs3KyUlRTExMapSpYoGDx6sI0eOnHZ7n376aYWEhGjixIn+aRMnTlTjxo0VFRWlqlWrqkWLFnrttddKszsqNI4MnOTgwYPau3dvoek5OTlBX9eBAwfUpUsX3XrrrerXr5/mzZunxx57TE2bNtVNN90kSTp69Kjat2+vzZs364EHHlDdunU1d+5cpaSkKCsrSyNHjpQkJScn6/XXX/cve//+/Vq/fr1CQkKUnp6uZs2aSZLS09MVFxenyy67rNjtOnTokF5++WXdfvvtGjJkiA4fPqzp06erc+fOWr16ta644oqA+WfMmKHs7GwNHTpUERERio2N1fr163XdddepVq1aevzxx1W5cmX94x//UM+ePTV//nz16tWryHV7PB7deeedGj9+vPbv36/Y2Fj/bYsWLdKhQ4d05513ntG+KS2fz6dRo0YpIiJCAwYMOKtl4fyVnp6ut99+W8OHD5ckjRs3TjfffLN++9vfasqUKbr//vt14MABjR8/XnfffbeWLVt2yuWtW7dOXbp0UY0aNTRmzBjl5eXpT3/6k+Li4grNe88992jWrFnq06ePRo0apU8//VTjxo3Txo0b9c9//tM/3+jRozVmzBh16tRJ9913nzIyMvTiiy/qs88+06pVqxQWFiZJmj59uoYNG6bWrVvroYce0pYtW9SjRw/Fxsaqdu3aZ7xvhg0bpszMTC1ZskSvvPJKkfO89tprOnz4sIYNGyaPx6Px48fr1ltv1ZYtW/zbtWTJEm3ZskWDBw9WQkKC1q9fr6lTp2r9+vX65JNPCgVGv379VLduXY0bN05r167Vyy+/rOrVq+vZZ58tdlt///vfa+zYsUpNTdWQIUMkSdOmTdOIESPUp08fjRw5UtnZ2frqq6/06aef6o477jjj/VGhOTjnnJsxY4aTdMqvxo0bB9wnMTHRDRo0qFTra9eunZPkZs+e7Z927Ngxl5CQ4Hr37u2fNmHCBCfJzZkzxz/t+PHjrlWrVi46OtodOnTIOefc3LlznSS3YcMG55xzb7/9touIiHA9evRw/fv399+3WbNmrlevXqfcttzcXHfs2LGAaQcOHHDx8fHu7rvv9k/bunWrk+QuvPBCt3v37oD5O3bs6Jo2beqys7P903w+n2vdurVLSko65fozMjKcJPfiiy8GTO/Ro4erU6eO8/l8Z7Rvli9f7iS55cuXByyvYPtnzJhR5HYMHTrUeTwe99prr51ye3H+KRjPn3322Snnk+QiIiLc1q1b/dNSU1OdJJeQkOB/DDnn3BNPPOEkBcxblO7du7uoqCj3008/+adt2rTJhYaGuhOfcr/44gsnyd1zzz0B93/kkUecJLds2TLnnHO7d+924eHh7sYbb3R5eXn++SZNmuQkub///e/OufzHfvXq1d0VV1wRMH6nTp3qJLl27dqdcruLM3z4cFfUr4qC8VOtWjW3f/9+//S33nrLSXKLFi3yTzty5Eih+7/++utOklu5cqV/2lNPPeUkBTzPOOdcr169XLVq1QKmSXLDhw93zjk3atQoFxIS4mbOnBkwzy233FLoebsoBY+X0/3bVmS8TXCSyZMna8mSJYW+Cl5ZB1N0dLT/Va4khYeHq2XLltqyZYt/WlpamhISEnT77bf7p4WFhWnEiBH6+eef/We6JycnS5JWrlwpKf/VztVXX60bbrjBf5gwKytL33zzjX/e4ni9Xv97/j6fT/v371dubq5atGihtWvXFpq/d+/eAa969u/fr2XLlqlfv346fPiw9u7dq71792rfvn3q3LmzNm3apJ9++qnY9Tds2FDXXHONXn311YBlvvfeexowYID/VURJ901pTJ8+XVOnTtVzzz0XsHxUPB07dgy4yqDgraLevXvrggsuKDT9xPF5sry8PH3wwQfq2bOnatas6Z/eoEED/9G+AmlpaZKk3/zmNwHTR40aJUl69913JUkffPCBjh8/roceekghIf97yh4yZIguvPBC/3xr1qzR7t27de+99wacs5OSkqIqVaqcZi+UXv/+/VW1alX/9wXPLyfup0qVKvn/Pzs7W3v37tW1114rSUU+p9x7770B3ycnJ2vfvn06dOhQwHTnnB544AE9//zzmjNnjgYNGhRwe0xMjHbs2FHobQsURgycpGXLlurUqVOhrxMf7MHyq1/9qtDhsapVq+rAgQP+77dv366kpKSAJwFJ/sP827dvlyTFx8crKSnJ/4s/PT1dycnJatu2rTIzM7VlyxatWrVKPp/vtDEgSbNmzVKzZs0UGRmpatWqKS4uTu+++64OHjxYaN66desGfL9582Y55/SHP/xBcXFxAV9PPfWUpPz3+k9l4MCBWrVqlf/nmzt3rnJycnTXXXed8b4pjVdeeUUNGzbUww8/XOpl4Jfh4osvDvi+4BfnyYfVC6afOD5Ptnv3bh09elQNGjQodNvJ07Zv366QkJBC0xMSEhQTE+N//Bb895JLLgmYLzw8XPXq1Ss0X1JSUsB8YWFhqlevXrHbfLZO3n8Fz5Un7qf9+/dr5MiRio+PV6VKlRQXF+d/3ijqOaUky5Sk2bNna/LkyZo4cWKR0f7YY48pOjpaLVu2VFJSkoYPH65Vq1aV4qes+IiBcuT1eouc7pwr1fLatGmj9PR0HT16VJ9//rmSk5PVpEkTxcTEKD09Xenp6YqOjlbz5s1PuZw5c+YoJSVF9evX1/Tp0/X+++9ryZIl6tChg3w+X6H5T6x+Sf55HnnkkSKPsixZsqTIJ8sT3XbbbQoLC/MfHZgzZ45atGhR6AmxJIo64UnKfxVXnH379qlGjRpnvC788hQ3DoM9PotT3OPzl6Ik+6lfv36aNm2a7r33Xi1YsECLFy/W+++/L0lFPqeUdN9fd911io+P16RJk7R///5C81922WXKyMjQG2+8oTZt2mj+/Plq06aN/0UJ/ocYOM8lJiZq06ZNhQZMwXXviYmJ/mnJycn64Ycf9MYbbygvL0+tW7dWSEiIPxLS09PVunXrYgdagXnz5qlevXpasGCB7rrrLnXu3FmdOnVSdnZ2iba54FVIWFhYkUdZOnXqFHD4tSixsbHq1q2bXn31VW3fvl2rVq0KOCpwJvum4FXFyWden+rIwe23317sSY5AcapXr67IyEht3ry50G0nT0tMTJTP59OmTZsCpu/atUtZWVn+x2/BfzMyMgLmO378uLZu3VpovpOXl5OTo61bt5b6ZzrbWDlw4ICWLl2qxx9/XGPGjFGvXr10ww03BOVoRYMGDbR48WJlZmaqS5cuOnz4cKF5KleurP79+2vGjBn64Ycf1K1bNz3zzDMBz2cpKSn+K5msIgbOc127dtXOnTv15ptv+qfl5uZq4sSJio6OVrt27fzTCw7/P/vss2rWrJn/sGZycrKWLl2qNWvWlOgtgoJYOLHCP/30U3388ccl2ubq1aurffv2Sk1N1X/+859Ct+/Zs6dEy7nrrru0YcMGPfroo/J6vbrtttsCbi/pvklMTJTX6/WfT1FgypQpxa67f//+xADOmNfrVadOnbRw4UJlZmb6p2/evFnvvfdewLxdu3aVJE2YMCFg+t/+9jdJUrdu3SRJnTp1Unh4uF544YWAMTl9+nQdPHjQP1+LFi0UFxenl156KeBS2JkzZxZ5CWJJVa5cWVLhmC6pop5PpMI/d2k1a9ZMaWlp2rhxo7p3766jR4/6bzv50sXw8HA1atRIzrmAK8T27t2rb7/9tkyuGvul4NLCICsoy2B9rOXQoUOVmpqqlJQUff7556pTp47mzZunVatWacKECQGvsBs0aKCEhARlZGTowQcf9E9v27atHnvsMUkqUQzcfPPNWrBggXr16qVu3bpp69ateumll9SoUSP9/PPPJdruyZMnq02bNmratKmGDBmievXqadeuXfr444+1Y8cOffnll6ddRrdu3VStWjXNnTtXN910k6pXr16qfVOlShX17dtXEydOlMfjUf369fXOO++c8ryFgpPKTv5sAuB0Ro8ercWLF+u6667Tfffdp7y8PE2aNElNmjTRF1984Z/v8ssv16BBgzR16lRlZWWpXbt2Wr16tWbNmqWePXvq+uuvlyTFxcXpiSee0JgxY9SlSxf16NFDGRkZmjJliq6++mr/SchhYWF6+umnNWzYMHXo0EH9+/fX1q1bNWPGjCJfhbdv314ffvjhad/2uOqqqyRJI0aMUOfOnYsM81O58MIL1bZtW40fP145OTmqVauWFi9efFZHK0527bXX6q233lLXrl3Vp08fLVy4UGFhYbrxxhuVkJDgfzth48aNmjRpkrp16xbw3Dlp0iSNGTNGW7dutXt0oHwuYjj/nO5SpHbt2pXo0sKLLrrIXXvttaddX1HLc865QYMGucTExIBpu3btcoMHD3YXXXSRCw8Pd02bNi32cri+ffs6Se7NN9/0Tzt+/LiLiopy4eHh7ujRo6fdNp/P58aOHesSExNdRESEa968uXvnnXcKbVvBpUV/+ctfilzO999/7wYOHOgSEhJcWFiYq1Wrlrv55pvdvHnzTrsNBe6//34nqdjL+0q6b/bs2eN69+7toqKiXNWqVd2wYcPcN998U+ylhYmJiaW+FAvl70wuLSy4PK1AcY/rgktU586de9r1L1261DVv3tyFh4e7+vXru5dfftmNGjXKRUZGBsyXk5PjxowZ4+rWrevCwsJc7dq13RNPPBFwSW6BSZMmuUsvvdSFhYW5+Ph4d99997kDBw4Umm/KlCmubt26LiIiwrVo0cKtXLnStWvXrtDj+aqrrnIJCQmn/Vlyc3Pdgw8+6OLi4pzH4/FfZniq8S/JPfXUU/7vd+zY4Xr16uViYmJclSpVXN++fV1mZmah+QouLdyzZ0/A8oq69K+of7u33nrLhYaGuv79+7u8vDyXmprq2rZt66pVq+YiIiJc/fr13aOPPuoOHjwYcL+C9Vq+tNDjXJDPhjFsw4YNaty4sd555x3/oTucnYcffljTp0/Xzp07FRUVVd6bA5Raz549tX79+kLv6ZeHw4cPKzY2VhMmTPB/2BJs45yBIFq+fLlatWpFCARJdna25syZo969exMC+EU58X1rKf+kvrS0tKD/GeHSWrlypWrVquX/pD6AIwM47+zevVsffPCB5s2bp4ULF2rt2rWFPgIZOJ/VqFHD/1n827dv14svvqhjx45p3bp1hT4HADgfcAIhzjsbNmzQgAEDVL16db3wwguEAH5xunTpotdff107d+5URESEWrVqpbFjxxICOG9xZAAAAOM4ZwAAAOOIAQAAjCMGAAAwjhgAAMA4YgAAAOOIAQAAjCMGAAAwjhgAAMA4YgAAAOOIAQAAjCMGAAAwjhgAAMA4YgAAAOOIAQAAjCMGAAAwjhgAAMA4YgAAAOOIAQAAjCMGAAAwjhgAAMA4YgAAAOOIAQAAjCMGAAAwjhgAAMA4YgAAAOOIAQAAjCMGAAAwjhgAAMA4YgAAAOOIAQAAjCMGAAAwLrS8NwDlz+eTli6VFi3K/9q5M396QoLUvXv+V8eOUgjpCPzi+JxPS7cs1aLvFmnRd4u08+f8AZ4QnaDuDbure8Pu6livo0I8DHDLPM45V94bgfLz5ZfS0KHS6tVSaKiUmxt4e8G0li2ladOkZs3KZzsBnLkvd36poYuGanXmaoWGhCrXFzjAC6a1rNVS07pPU7N4BrhVpKBhzz0nXXmltHZt/vcnh8CJ09aulZo3z78PgPPfcx89pyunXqm1O/MH+MkhcOK0tZlr1Ty1uZ77iAFuFUcGjBo3TnryydLf9/HHg7s9AIJnXPo4PbmsdAN8XMdxerwNA9waYsCgF16QRo48u2U8/7w0YkRwtgdA8Lzw6Qsa+f7ZDfDnuzyvEdcwwC0hBoxZty7//f+i3hI4E6Gh+ecZNG8enO0CcPbW/WedWr7cssi3BM5EaEioVt+zWs1rMMCt4JwBQ3JzpYEDpWDkn3PSoEFnHxUAgiPXl6uBCwcqGK/vnHMatHDQWUcFfjmIAUNmz5a++UbKyzv7ZeXlSV9/nb9MAOVv9pez9c3ub5Tnzn6A57k8fb37a83+kgFuBW8TGOGcdOml0qZNwTkyIEkej9SwobRxY/7/AygfzjldOvlSbdq3SU7BGeAeedSwWkNtHL5RHgZ4hceRASP+9S/pu++CFwJS/rIyMqTFi4O3TABn7l/f/0vf7fsuaCEgSU5OGfsytPh7BrgFxIARqamS1xv85Xq90ksvBX+5AEoudU2qvJ7gD3Cvx6uX1jDALeBtAgNycqSYGOnIkbJZflSUlJUlhYWVzfIBFC8nL0cxz8boSE7ZDPCosChlPZalMC8DvCLjyIABH39cdiEg5S/7k0/KbvkAivfxjo/LLAQk6UjOEX2ygwFe0REDBixZkv+5AGUlNJTzBoDysuT7JQoNKbsBHhoSynkDBhADBqxfH5zLCYvj80kbNpTd8gEUb/2e9crzld0A9zmfNuxhgFd0xIABGRnBvYrgZD5f/joAnHsZ+zKCehXByXzOp4x9DPCKjhio4JyTtm0r+/Vs21a2wQGgMOectmVtK/P1bMvaFpRPNsT5ixio4A4fLtuTBwv897/Szz+X/XoA/M/h44fL9OTBAv/N+a9+Ps4Ar8iIgQru2LGKuS4A0rHcczfojuUxwCsyYqCC8/kq5roA5L+fXxHXhXOPGKjgwsMr5roASOHeczfozuW6cO4RAxXcBRdIIefgX9nrlaKjy349AP7ngogLFOIp+wHu9XgVHc4Ar8iIgQouNFSqVavs11OzZtl+sBGAwkJDQlXrgrIf4DUvqFmmH2yE8kcMGJCUVPbraNiw7NcBoLCkamU/wBtWY4BXdMSAAUlJZftHhMLCzk1wACgsKTZJYSFlN8DDQsKUFMsAr+iIAQOSk/P/cmFZycnJXweAcy/54mTl+MpugOf4cpScyACv6PgTxgbs2SPFx5fdJwR6PNKuXVJcXNksH0Dx9vx3j+L/Gl9mH0nskUe7HtmluMoM8IqMIwMGxMVJzZqV3fIvv5wQAMpLXOU4NYsvuwF+ecLlhIABxIARAwaUzSWGHo90xx3BXy6AkhvQdECZXGLokUd3NGGAW8DbBEZkZUk1akjZ2cFdbqVKUmamFBMT3OUCKLms7CzVeK6GsnODO8ArhVZS5qhMxUTGBHW5OP9wZMCImBhp6ND8DwcKFq83f5mEAFC+YiJjNPTKofJ6gjfAvR6vhl41lBAwgiMDhvz4Y/7nAQTr6EBkpPTdd1Lt2sFZHoDS+/Hgj2o4qWHQjg5Ehkbquwe+U+0qDHALODJgSO3a0tixwVveuHGEAHC+qF2ltsZ2CN4AH9dxHCFgCEcGjPH5pA4dpH//W8rLK90yvN78zxVYuvTc/N0DACXjcz51mNVB//7h38pzpRvgXo9XyYnJWjpw6Tn5uwc4P/AvbUxIiLRggdSoUenOH/B6pSZNpPnzCQHgfBPiCdGC/gvUKK5Rqc4f8Hq8alK9ieb3m08IGMO/tkGxsdKKFfm/1M/kF3pISP59li3LXwaA809spVitSFmhJtWbnNEv9BBPiJpUb6Jlg5YpthID3BpiwKjYWCk9XRoxIv+zAk51lMDrzZ9nxIj8+xACwPkttlKs0gena0TLEfLIc8qjBF6PVx55NKLlCKUPTicEjOKcAWjduvwTC9PSpCNHAm+LipK6dpWefFJq3rx8tg9A6a37zzqNTR+rtM1pOpITOMCjwqLUtUFXPZn8pJrXYIBbRgzA7/hx6aOP8j9ESJJq1pRat5bCw8t3uwCcveN5x/XRjx8p83D+AK95QU21rt1a4V4GOIgBAADM45wBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADCOGAAAwDhiAAAA44gBAACMIwYAADDu/wC+Ka8LXmqY8wAAAABJRU5ErkJggg==", + "text/plain": [ + "

" + ] + }, + "metadata": {}, + "output_type": "display_data" + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[32m***** Response from calling tool (call_VjzGKf34UMKMlSHCrQpJ6ICK) *****\u001b[0m\n", + "(0.0, 4.0, 0.0, 2.0)\n", + "\u001b[32m**********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mchatbot\u001b[0m (to user_proxy):\n", + "\n", + "TERMINATE\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "from IPython import get_ipython\n", + "from typing_extensions import Annotated\n", + "\n", + "import autogen\n", + "from autogen.cache import Cache\n", + "\n", + "config_list = autogen.config_list_from_json(\"OAI_CONFIG_LIST\", filter_dict={\"tags\": [\"gemini\", \"tool\"]})\n", + "\n", + "llm_config = {\n", + " \"config_list\": config_list,\n", + " \"timeout\": 120,\n", + "}\n", + "chatbot = autogen.AssistantAgent(\n", + " name=\"chatbot\",\n", + " system_message=\"For coding tasks, only use the functions you have been provided with. Reply TERMINATE when the task is done.\",\n", + " llm_config=llm_config,\n", + ")\n", + "\n", + "# create a UserProxyAgent instance named \"user_proxy\"\n", + "user_proxy = autogen.UserProxyAgent(\n", + " name=\"user_proxy\",\n", + " is_termination_msg=lambda x: x.get(\"content\", \"\") and x.get(\"content\", \"\").rstrip().endswith(\"TERMINATE\"),\n", + " human_input_mode=\"NEVER\",\n", + " max_consecutive_auto_reply=10,\n", + " code_execution_config={\n", + " \"work_dir\": \"coding\",\n", + " \"use_docker\": False,\n", + " }, # Please set use_docker=True if docker is available to run the generated code. Using docker is safer than running the generated code directly.\n", + ")\n", + "\n", + "\n", + "# define functions according to the function description\n", + "\n", + "\n", + "# one way of registering functions is to use the register_for_llm and register_for_execution decorators\n", + "@user_proxy.register_for_execution()\n", + "@chatbot.register_for_llm(name=\"python\", description=\"run cell in ipython and return the execution result.\")\n", + "def exec_python(cell: Annotated[str, \"Valid Python cell to execute.\"]) -> str:\n", + " ipython = get_ipython()\n", + " result = ipython.run_cell(cell)\n", + " log = str(result.result)\n", + " if result.error_before_exec is not None:\n", + " log += f\"\\n{result.error_before_exec}\"\n", + " if result.error_in_exec is not None:\n", + " log += f\"\\n{result.error_in_exec}\"\n", + " return log\n", + "\n", + "\n", + "# another way of registering functions is to use the register_function\n", + "def exec_sh(script: Annotated[str, \"Valid Python cell to execute.\"]) -> str:\n", + " return user_proxy.execute_code_blocks([(\"sh\", script)])\n", + "\n", + "\n", + "autogen.agentchat.register_function(\n", + " exec_python,\n", + " caller=chatbot,\n", + " executor=user_proxy,\n", + " name=\"sh\",\n", + " description=\"run a shell script and return the execution result.\",\n", + ")\n", + "\n", + "with Cache.disk() as cache:\n", + " # start the conversation\n", + " user_proxy.initiate_chat(\n", + " chatbot,\n", + " message=\"Draw two agents chatting with each other with an example dialog. Don't add plt.show().\",\n", + " cache=cache,\n", + " max_turns=3,\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "autogen", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.12.3" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb b/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb index aea134907b76..3e0c721f65e1 100644 --- a/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb +++ b/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb @@ -26,7 +26,7 @@ "metadata": {}, "outputs": [], "source": [ - "%pip install pyautogen llama-index llama-index-tools-wikipedia llama-index-readers-wikipedia wikipedia" + "%pip install autogen-agentchat~=0.2 llama-index llama-index-tools-wikipedia llama-index-readers-wikipedia wikipedia" ] }, { diff --git a/notebook/agentchat_groupchat.ipynb b/notebook/agentchat_groupchat.ipynb index d2c061d14105..925c7124a29c 100644 --- a/notebook/agentchat_groupchat.ipynb +++ b/notebook/agentchat_groupchat.ipynb @@ -14,9 +14,9 @@ "\n", "````{=mdx}\n", ":::info Requirements\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", @@ -218,8 +218,11 @@ ], "metadata": { "front_matter": { - "tags": ["orchestration", "group chat"], - "description": "Explore the utilization of large language models in automated group chat scenarios, where agents perform tasks collectively, demonstrating how they can be configured, interact with each other, and retrieve specific information from external resources." + "description": "Explore the utilization of large language models in automated group chat scenarios, where agents perform tasks collectively, demonstrating how they can be configured, interact with each other, and retrieve specific information from external resources.", + "tags": [ + "orchestration", + "group chat" + ] }, "kernelspec": { "display_name": "flaml", diff --git a/notebook/agentchat_groupchat_RAG.ipynb b/notebook/agentchat_groupchat_RAG.ipynb index e18bd99c1511..aeb4f714e68a 100644 --- a/notebook/agentchat_groupchat_RAG.ipynb +++ b/notebook/agentchat_groupchat_RAG.ipynb @@ -15,7 +15,7 @@ "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", "\n", "```bash\n", - "pip install pyautogen[retrievechat]\n", + "pip install autogen-agentchat[retrievechat]~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchat_groupchat_customized.ipynb b/notebook/agentchat_groupchat_customized.ipynb index dde124aef7d7..3b85223a731e 100644 --- a/notebook/agentchat_groupchat_customized.ipynb +++ b/notebook/agentchat_groupchat_customized.ipynb @@ -39,9 +39,9 @@ "\n", "````{=mdx}\n", ":::info Requirements\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchat_groupchat_finite_state_machine.ipynb b/notebook/agentchat_groupchat_finite_state_machine.ipynb index 8ef101f7d914..cfe45662a8fa 100644 --- a/notebook/agentchat_groupchat_finite_state_machine.ipynb +++ b/notebook/agentchat_groupchat_finite_state_machine.ipynb @@ -18,9 +18,9 @@ "\n", "````{=mdx}\n", ":::info Requirements\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", @@ -35,7 +35,7 @@ "outputs": [], "source": [ "%%capture --no-stderr\n", - "%pip install pyautogen[graph]>=0.2.11" + "%pip install autogen-agentchat[graph]~=0.2.11" ] }, { @@ -94,7 +94,7 @@ " \"cache_seed\": 44, # change the seed for different trials\n", " \"config_list\": autogen.config_list_from_json(\n", " \"OAI_CONFIG_LIST\",\n", - " filter_dict={\"tags\": [\"gpt-4\", \"gpt-4-32k\"]}, # comment out to get all\n", + " filter_dict={\"tags\": [\"gpt-4o\"]}, # comment out to get all\n", " ),\n", " \"temperature\": 0,\n", "}" diff --git a/notebook/agentchat_groupchat_research.ipynb b/notebook/agentchat_groupchat_research.ipynb index c448ed8cb7a8..6adf653f9035 100644 --- a/notebook/agentchat_groupchat_research.ipynb +++ b/notebook/agentchat_groupchat_research.ipynb @@ -14,9 +14,9 @@ "\n", "````{=mdx}\n", ":::info Requirements\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", @@ -515,8 +515,10 @@ ], "metadata": { "front_matter": { - "tags": ["group chat"], - "description": "Perform research using a group chat with a number of specialized agents" + "description": "Perform research using a group chat with a number of specialized agents", + "tags": [ + "group chat" + ] }, "kernelspec": { "display_name": "flaml", diff --git a/notebook/agentchat_groupchat_stateflow.ipynb b/notebook/agentchat_groupchat_stateflow.ipynb index 53eb0f2ff986..6a7869ec5403 100644 --- a/notebook/agentchat_groupchat_stateflow.ipynb +++ b/notebook/agentchat_groupchat_stateflow.ipynb @@ -12,9 +12,9 @@ "\n", "````{=mdx}\n", ":::info Requirements\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", @@ -43,7 +43,7 @@ "config_list = autogen.config_list_from_json(\n", " \"OAI_CONFIG_LIST\",\n", " filter_dict={\n", - " \"tags\": [\"gpt-4\", \"gpt-4-32k\"],\n", + " \"tags\": [\"gpt-4o\"],\n", " },\n", ")" ] diff --git a/notebook/agentchat_groupchat_vis.ipynb b/notebook/agentchat_groupchat_vis.ipynb index 29f968752ae6..d660bb0f51ea 100644 --- a/notebook/agentchat_groupchat_vis.ipynb +++ b/notebook/agentchat_groupchat_vis.ipynb @@ -12,9 +12,9 @@ "\n", "````{=mdx}\n", ":::info Requirements\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", @@ -976,8 +976,10 @@ ], "metadata": { "front_matter": { - "tags": ["group chat"], - "description": "Explore a group chat example using agents such as a coder and visualization agent." + "description": "Explore a group chat example using agents such as a coder and visualization agent.", + "tags": [ + "group chat" + ] }, "kernelspec": { "display_name": "flaml", diff --git a/notebook/agentchat_huggingface_langchain.ipynb b/notebook/agentchat_huggingface_langchain.ipynb new file mode 100644 index 000000000000..248ad18f19c1 --- /dev/null +++ b/notebook/agentchat_huggingface_langchain.ipynb @@ -0,0 +1,866 @@ +{ + "cells": [ + { + "attachments": {}, + "cell_type": "markdown", + "metadata": { + "slideshow": { + "slide_type": "slide" + } + }, + "source": [ + "# Using AutoGen AgentChat with LangChain-based Custom Client and Hugging Face Models" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Introduction\n", + "\n", + "This notebook demonstrates how you can use LangChain's extensive support for LLMs to enable flexible use of various Language Models (LLMs) in agent-based conversations in AutoGen.\n", + "\n", + "What we'll cover:\n", + "\n", + "1. Creating a custom model client that uses LangChain to load and interact with LLMs\n", + "2. Configuring AutoGen to use our custom LangChain-based model\n", + "3. Setting up AutoGen agents with the custom model\n", + "4. Demonstrating a simple conversation using this setup\n", + "\n", + "While we used a Hugging Face model in this example, the same approach can be applied to any LLM supported by LangChain, including models from OpenAI, Anthropic, or custom models. This integration opens up a wide range of possibilities for creating sophisticated, multi-model conversational agents using AutoGen\n", + "\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Requirements\n", + "\n", + "````{=mdx}\n", + ":::info Requirements\n", + "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", + "\n", + "```bash\n", + "pip install pyautogen torch transformers sentencepiece langchain-huggingface \n", + "```\n", + "\n", + "For more information, please refer to the [installation guide](/docs/installation/).\n", + ":::\n", + "````\n", + "\n", + "**NOTE: Depending on what model you use, you may need to play with the default prompts of the Agent's**" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup and Imports\n", + "\n", + "First, let's import the necessary libraries and define our custom model client." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from types import SimpleNamespace\n", + "\n", + "from langchain_core.messages import AIMessage, HumanMessage, SystemMessage\n", + "from langchain_huggingface import ChatHuggingFace, HuggingFacePipeline\n", + "\n", + "from autogen import AssistantAgent, UserProxyAgent, config_list_from_json" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Create and configure the custom model\n", + "\n", + "A custom model class can be created in many ways, but needs to adhere to the `ModelClient` protocol and response structure which is defined in client.py and shown below.\n", + "\n", + "The response protocol has some minimum requirements, but can be extended to include any additional information that is needed.\n", + "Message retrieval therefore can be customized, but needs to return a list of strings or a list of `ModelClientResponseProtocol.Choice.Message` objects.\n", + "\n", + "\n", + "```python\n", + "class ModelClient(Protocol):\n", + " \"\"\"\n", + " A client class must implement the following methods:\n", + " - create must return a response object that implements the ModelClientResponseProtocol\n", + " - cost must return the cost of the response\n", + " - get_usage must return a dict with the following keys:\n", + " - prompt_tokens\n", + " - completion_tokens\n", + " - total_tokens\n", + " - cost\n", + " - model\n", + "\n", + " This class is used to create a client that can be used by OpenAIWrapper.\n", + " The response returned from create must adhere to the ModelClientResponseProtocol but can be extended however needed.\n", + " The message_retrieval method must be implemented to return a list of str or a list of messages from the response.\n", + " \"\"\"\n", + "\n", + " RESPONSE_USAGE_KEYS = [\"prompt_tokens\", \"completion_tokens\", \"total_tokens\", \"cost\", \"model\"]\n", + "\n", + " class ModelClientResponseProtocol(Protocol):\n", + " class Choice(Protocol):\n", + " class Message(Protocol):\n", + " content: Optional[str]\n", + "\n", + " message: Message\n", + "\n", + " choices: List[Choice]\n", + " model: str\n", + "\n", + " def create(self, params) -> ModelClientResponseProtocol:\n", + " ...\n", + "\n", + " def message_retrieval(\n", + " self, response: ModelClientResponseProtocol\n", + " ) -> Union[List[str], List[ModelClient.ModelClientResponseProtocol.Choice.Message]]:\n", + " \"\"\"\n", + " Retrieve and return a list of strings or a list of Choice.Message from the response.\n", + "\n", + " NOTE: if a list of Choice.Message is returned, it currently needs to contain the fields of OpenAI's ChatCompletion Message object,\n", + " since that is expected for function or tool calling in the rest of the codebase at the moment, unless a custom agent is being used.\n", + " \"\"\"\n", + " ...\n", + "\n", + " def cost(self, response: ModelClientResponseProtocol) -> float:\n", + " ...\n", + "\n", + " @staticmethod\n", + " def get_usage(response: ModelClientResponseProtocol) -> Dict:\n", + " \"\"\"Return usage summary of the response using RESPONSE_USAGE_KEYS.\"\"\"\n", + " ...\n", + "```\n" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example of simple custom client\n", + "\n", + "Following the huggingface example for using [Mistral's Open-Orca](https://huggingface.co/Open-Orca/Mistral-7B-OpenOrca)\n", + "\n", + "For the response object, python's `SimpleNamespace` is used to create a simple object that can be used to store the response data, but any object that follows the `ClientResponseProtocol` can be used.\n" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# custom client with custom model loader\n", + "\n", + "\n", + "class CustomModelClient:\n", + " \"\"\"Custom model client implementation for LangChain integration with AutoGen.\"\"\"\n", + "\n", + " def __init__(self, config, **kwargs):\n", + " \"\"\"Initialize the CustomModelClient.\"\"\"\n", + " print(f\"CustomModelClient config: {config}\")\n", + " self.device = config.get(\"device\", \"cpu\")\n", + "\n", + " gen_config_params = config.get(\"params\", {})\n", + " self.model_name = config[\"model\"]\n", + " pipeline = HuggingFacePipeline.from_model_id(\n", + " model_id=self.model_name,\n", + " task=\"text-generation\",\n", + " pipeline_kwargs=gen_config_params,\n", + " device=self.device,\n", + " )\n", + " self.model = ChatHuggingFace(llm=pipeline)\n", + " print(f\"Loaded model {config['model']} to {self.device}\")\n", + "\n", + " def _to_chatml_format(self, message):\n", + " \"\"\"Convert message to ChatML format.\"\"\"\n", + " if message[\"role\"] == \"system\":\n", + " return SystemMessage(content=message[\"content\"])\n", + " if message[\"role\"] == \"assistant\":\n", + " return AIMessage(content=message[\"content\"])\n", + " if message[\"role\"] == \"user\":\n", + " return HumanMessage(content=message[\"content\"])\n", + " raise ValueError(f\"Unknown message type: {type(message)}\")\n", + "\n", + " def create(self, params):\n", + " \"\"\"Create a response using the model.\"\"\"\n", + " if params.get(\"stream\", False) and \"messages\" in params:\n", + " raise NotImplementedError(\"Local models do not support streaming.\")\n", + "\n", + " num_of_responses = params.get(\"n\", 1)\n", + " response = SimpleNamespace()\n", + " inputs = [self._to_chatml_format(m) for m in params[\"messages\"]]\n", + " response.choices = []\n", + " response.model = self.model_name\n", + "\n", + " for _ in range(num_of_responses):\n", + " outputs = self.model.invoke(inputs)\n", + " text = outputs.content\n", + " choice = SimpleNamespace()\n", + " choice.message = SimpleNamespace()\n", + " choice.message.content = text\n", + " choice.message.function_call = None\n", + " response.choices.append(choice)\n", + "\n", + " return response\n", + "\n", + " def message_retrieval(self, response):\n", + " \"\"\"Retrieve messages from the response.\"\"\"\n", + " return [choice.message.content for choice in response.choices]\n", + "\n", + " def cost(self, response) -> float:\n", + " \"\"\"Calculate the cost of the response.\"\"\"\n", + " response.cost = 0\n", + " return 0\n", + "\n", + " @staticmethod\n", + " def get_usage(response):\n", + " \"\"\"Get usage statistics.\"\"\"\n", + " return {}" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set your API Endpoint\n", + "\n", + "The [`config_list_from_json`](https://microsoft.github.io/autogen/docs/reference/oai/openai_utils#config_list_from_json) function loads a list of configurations from an environment variable or a json file.\n", + "\n", + "It first looks for an environment variable of a specified name (\"OAI_CONFIG_LIST\" in this example), which needs to be a valid json string. If that variable is not found, it looks for a json file with the same name. It filters the configs by models (you can filter by other keys as well).\n", + "\n", + "The json looks like the following:\n", + "```json\n", + "[\n", + " {\n", + " \"model\": \"gpt-4\",\n", + " \"api_key\": \"\"\n", + " },\n", + " {\n", + " \"model\": \"gpt-4\",\n", + " \"api_key\": \"\",\n", + " \"base_url\": \"\",\n", + " \"api_type\": \"azure\",\n", + " \"api_version\": \"2024-02-01\"\n", + " },\n", + " {\n", + " \"model\": \"gpt-4-32k\",\n", + " \"api_key\": \"\",\n", + " \"base_url\": \"\",\n", + " \"api_type\": \"azure\",\n", + " \"api_version\": \"2024-02-01\"\n", + " }\n", + "]\n", + "```\n", + "\n", + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/notebook/oai_openai_utils.ipynb) for full code examples of the different methods." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Set the config for the custom model\n", + "\n", + "You can add any paramteres that are needed for the custom model loading in the same configuration list.\n", + "\n", + "It is important to add the `model_client_cls` field and set it to a string that corresponds to the class name: `\"CustomModelClient\"`." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "os.environ[\"OAI_CONFIG_LIST\"] = json.dumps(\n", + " [\n", + " {\n", + " \"model\": \"mistralai/Mistral-7B-Instruct-v0.2\",\n", + " \"model_client_cls\": \"CustomModelClient\",\n", + " \"device\": 0,\n", + " \"n\": 1,\n", + " \"params\": {\n", + " \"max_new_tokens\": 500,\n", + " \"top_k\": 50,\n", + " \"temperature\": 0.1,\n", + " \"do_sample\": True,\n", + " \"return_full_text\": False,\n", + " },\n", + " }\n", + " ]\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], + "source": [ + "config_list_custom = config_list_from_json(\n", + " \"OAI_CONFIG_LIST\",\n", + " filter_dict={\"model_client_cls\": [\"CustomModelClient\"]},\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import getpass\n", + "\n", + "from huggingface_hub import login\n", + "\n", + "# The Mistral-7B-Instruct-v0.2 is a gated model which requires API token to access\n", + "login(token=getpass.getpass(\"Enter your HuggingFace API Token\"))" + ] + }, + { + "attachments": {}, + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Construct Agents\n", + "\n", + "Consturct a simple conversation between a User proxy and an Assistent agent" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[autogen.oai.client: 09-01 12:53:51] {484} INFO - Detected custom model client in config: CustomModelClient, model client can not be used until register_model_client is called.\n" + ] + } + ], + "source": [ + "assistant = AssistantAgent(\"assistant\", llm_config={\"config_list\": config_list_custom})\n", + "user_proxy = UserProxyAgent(\"user_proxy\", code_execution_config=False)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Register the custom client class to the assistant agent" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "CustomModelClient config: {'model': 'microsoft/Phi-3.5-mini-instruct', 'model_client_cls': 'CustomModelClient', 'device': 0, 'n': 1, 'params': {'max_new_tokens': 100, 'top_k': 50, 'temperature': 0.1, 'do_sample': True, 'return_full_text': False}}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Loading checkpoint shards: 100%|████████████████████████████████████████████████████████████████████████████████████████████| 2/2 [00:07<00:00, 3.51s/it]\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Loaded model microsoft/Phi-3.5-mini-instruct to 0\n" + ] + } + ], + "source": [ + "assistant.register_model_client(model_client_cls=CustomModelClient)" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser_proxy\u001b[0m (to assistant):\n", + "\n", + "Write python code to print Hello World!\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "You are not running the flash-attention implementation, expect numerical differences.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33massistant\u001b[0m (to user_proxy):\n", + "\n", + " ```python\n", + "# filename: hello_world.py\n", + "\n", + "print(\"Hello World!\")\n", + "```\n", + "\n", + "To execute this code, save it in a file named `hello_world.py`. Then, open your terminal or command prompt, navigate to the directory containing the file, and run the following command:\n", + "\n", + "```\n", + "python hello_world.py\n", + "```\n", + "\n", + "The output should be:\n", + "\n", + "```\n", + "Hello World!\n", + "```\n", + "\n", + "If you encounter any errors,\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "data": { + "text/plain": [ + "ChatResult(chat_id=None, chat_history=[{'content': 'Write python code to print Hello World!', 'role': 'assistant', 'name': 'user_proxy'}, {'content': ' ```python\\n# filename: hello_world.py\\n\\nprint(\"Hello World!\")\\n```\\n\\nTo execute this code, save it in a file named `hello_world.py`. Then, open your terminal or command prompt, navigate to the directory containing the file, and run the following command:\\n\\n```\\npython hello_world.py\\n```\\n\\nThe output should be:\\n\\n```\\nHello World!\\n```\\n\\nIf you encounter any errors,', 'role': 'user', 'name': 'assistant'}], summary=' ```python\\n# filename: hello_world.py\\n\\nprint(\"Hello World!\")\\n```\\n\\nTo execute this code, save it in a file named `hello_world.py`. Then, open your terminal or command prompt, navigate to the directory containing the file, and run the following command:\\n\\n```\\npython hello_world.py\\n```\\n\\nThe output should be:\\n\\n```\\nHello World!\\n```\\n\\nIf you encounter any errors,', cost={'usage_including_cached_inference': {'total_cost': 0}, 'usage_excluding_cached_inference': {'total_cost': 0}}, human_input=['exit'])" + ] + }, + "execution_count": 8, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "user_proxy.initiate_chat(assistant, message=\"Write python code to print Hello World!\")" + ] + } + ], + "metadata": { + "front_matter": { + "description": "Define and laod a custom model", + "tags": [ + "custom model" + ] + }, + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.6" + }, + "vscode": { + "interpreter": { + "hash": "949777d72b0d2535278d3dc13498b2535136f6dfe0678499012e853ee9abcab1" + } + }, + "widgets": { + "application/vnd.jupyter.widget-state+json": { + "state": { + "2d910cfd2d2a4fc49fc30fbbdc5576a7": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "454146d0f7224f038689031002906e6f": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HBoxModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HBoxModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HBoxView", + "box_style": "", + "children": [ + "IPY_MODEL_e4ae2b6f5a974fd4bafb6abb9d12ff26", + "IPY_MODEL_577e1e3cc4db4942b0883577b3b52755", + "IPY_MODEL_b40bdfb1ac1d4cffb7cefcb870c64d45" + ], + "layout": "IPY_MODEL_dc83c7bff2f241309537a8119dfc7555", + "tabbable": null, + "tooltip": null + } + }, + "577e1e3cc4db4942b0883577b3b52755": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "FloatProgressModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "FloatProgressModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "ProgressView", + "bar_style": "success", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_2d910cfd2d2a4fc49fc30fbbdc5576a7", + "max": 1, + "min": 0, + "orientation": "horizontal", + "style": "IPY_MODEL_74a6ba0c3cbc4051be0a83e152fe1e62", + "tabbable": null, + "tooltip": null, + "value": 1 + } + }, + "6086462a12d54bafa59d3c4566f06cb2": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "74a6ba0c3cbc4051be0a83e152fe1e62": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "ProgressStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "ProgressStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "bar_color": null, + "description_width": "" + } + }, + "7d3f3d9e15894d05a4d188ff4f466554": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "b40bdfb1ac1d4cffb7cefcb870c64d45": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_f1355871cc6f4dd4b50d9df5af20e5c8", + "placeholder": "​", + "style": "IPY_MODEL_ca245376fd9f4354af6b2befe4af4466", + "tabbable": null, + "tooltip": null, + "value": " 1/1 [00:00<00:00, 44.69it/s]" + } + }, + "ca245376fd9f4354af6b2befe4af4466": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLStyleModel", + "state": { + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLStyleModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "StyleView", + "background": null, + "description_width": "", + "font_size": null, + "text_color": null + } + }, + "dc83c7bff2f241309537a8119dfc7555": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + }, + "e4ae2b6f5a974fd4bafb6abb9d12ff26": { + "model_module": "@jupyter-widgets/controls", + "model_module_version": "2.0.0", + "model_name": "HTMLModel", + "state": { + "_dom_classes": [], + "_model_module": "@jupyter-widgets/controls", + "_model_module_version": "2.0.0", + "_model_name": "HTMLModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/controls", + "_view_module_version": "2.0.0", + "_view_name": "HTMLView", + "description": "", + "description_allow_html": false, + "layout": "IPY_MODEL_6086462a12d54bafa59d3c4566f06cb2", + "placeholder": "​", + "style": "IPY_MODEL_7d3f3d9e15894d05a4d188ff4f466554", + "tabbable": null, + "tooltip": null, + "value": "100%" + } + }, + "f1355871cc6f4dd4b50d9df5af20e5c8": { + "model_module": "@jupyter-widgets/base", + "model_module_version": "2.0.0", + "model_name": "LayoutModel", + "state": { + "_model_module": "@jupyter-widgets/base", + "_model_module_version": "2.0.0", + "_model_name": "LayoutModel", + "_view_count": null, + "_view_module": "@jupyter-widgets/base", + "_view_module_version": "2.0.0", + "_view_name": "LayoutView", + "align_content": null, + "align_items": null, + "align_self": null, + "border_bottom": null, + "border_left": null, + "border_right": null, + "border_top": null, + "bottom": null, + "display": null, + "flex": null, + "flex_flow": null, + "grid_area": null, + "grid_auto_columns": null, + "grid_auto_flow": null, + "grid_auto_rows": null, + "grid_column": null, + "grid_gap": null, + "grid_row": null, + "grid_template_areas": null, + "grid_template_columns": null, + "grid_template_rows": null, + "height": null, + "justify_content": null, + "justify_items": null, + "left": null, + "margin": null, + "max_height": null, + "max_width": null, + "min_height": null, + "min_width": null, + "object_fit": null, + "object_position": null, + "order": null, + "overflow": null, + "padding": null, + "right": null, + "top": null, + "visibility": null, + "width": null + } + } + }, + "version_major": 2, + "version_minor": 0 + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebook/agentchat_human_feedback.ipynb b/notebook/agentchat_human_feedback.ipynb index 000d788d6a56..77b09fe0e8d5 100644 --- a/notebook/agentchat_human_feedback.ipynb +++ b/notebook/agentchat_human_feedback.ipynb @@ -28,7 +28,7 @@ "\n", "AutoGen requires `Python>=3.8`. To run this notebook example, please install:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```" ] }, @@ -45,7 +45,7 @@ }, "outputs": [], "source": [ - "# %pip install \"pyautogen>=0.2.3\"" + "# %pip install \"autogen-agentchat~=0.2\"" ] }, { @@ -102,7 +102,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." ] }, { diff --git a/notebook/agentchat_image_generation_capability.ipynb b/notebook/agentchat_image_generation_capability.ipynb index b5d298d7f4d2..d8a01fc30327 100644 --- a/notebook/agentchat_image_generation_capability.ipynb +++ b/notebook/agentchat_image_generation_capability.ipynb @@ -20,7 +20,7 @@ "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", "\n", "```bash\n", - "pip install pyautogen[lmm]\n", + "pip install autogen-agentchat[lmm]~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchat_langchain.ipynb b/notebook/agentchat_langchain.ipynb index 83ab2df44c28..bb4805cb3558 100644 --- a/notebook/agentchat_langchain.ipynb +++ b/notebook/agentchat_langchain.ipynb @@ -28,9 +28,9 @@ "\n", "## Requirements\n", "\n", - "AutoGen requires `Python>=3.8`. To run this notebook example, please install `pyautogen` and `Langchain`:\n", + "AutoGen requires `Python>=3.8`. To run this notebook example, please install `autogen-agentchat` and `Langchain`:\n", "```bash\n", - "pip install pyautogen Langchain\n", + "pip install autogen-agentchat~=0.2 Langchain\n", "```" ] }, @@ -47,7 +47,7 @@ }, "outputs": [], "source": [ - "%pip install \"pyautogen>=0.2.3\" Langchain" + "%pip install \"autogen-agentchat~=0.2\" Langchain" ] }, { @@ -139,7 +139,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." ] }, { diff --git a/notebook/agentchat_lmm_gpt-4v.ipynb b/notebook/agentchat_lmm_gpt-4v.ipynb index 7c9e3ea125c2..7136ac62e738 100644 --- a/notebook/agentchat_lmm_gpt-4v.ipynb +++ b/notebook/agentchat_lmm_gpt-4v.ipynb @@ -21,9 +21,9 @@ "source": [ "### Before everything starts, install AutoGen with the `lmm` option\n", "\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install \"pyautogen[lmm]>=0.2.17\"\n", + "pip install \"autogen-agentchat[lmm]~=0.2\"\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n" diff --git a/notebook/agentchat_lmm_llava.ipynb b/notebook/agentchat_lmm_llava.ipynb index 61efc74e00e1..e5eb26362ec7 100644 --- a/notebook/agentchat_lmm_llava.ipynb +++ b/notebook/agentchat_lmm_llava.ipynb @@ -26,7 +26,7 @@ "source": [ "### Before everything starts, install AutoGen with the `lmm` option\n", "```bash\n", - "pip install \"pyautogen[lmm]>=0.2.3\"\n", + "pip install \"autogen-agentchat[lmm]~=0.2\"\n", "```" ] }, diff --git a/notebook/agentchat_memory_using_mem0.ipynb b/notebook/agentchat_memory_using_mem0.ipynb index d590002164b9..b433e05df41d 100644 --- a/notebook/agentchat_memory_using_mem0.ipynb +++ b/notebook/agentchat_memory_using_mem0.ipynb @@ -17,7 +17,7 @@ "source": [ "This notebook demonstrates an intelligent customer service chatbot system that combines:\n", "\n", - "- PyAutoGen for conversational agents\n", + "- AutoGen for conversational agents\n", "- Mem0 for memory management\n", "\n", "[Mem0](https://www.mem0.ai/) provides a smart, self-improving memory layer for Large Language Models (LLMs), enabling developers to create personalized AI experiences that evolve with each user interaction. Refer [docs](https://docs.mem0.ai/overview) for more information.\n", @@ -50,7 +50,7 @@ "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", "\n", "```bash\n", - "pip install pyautogen mem0ai\n", + "pip install autogen-agentchat~=0.2 mem0ai\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchat_microsoft_fabric.ipynb b/notebook/agentchat_microsoft_fabric.ipynb index 1aaee58f0921..225000146786 100644 --- a/notebook/agentchat_microsoft_fabric.ipynb +++ b/notebook/agentchat_microsoft_fabric.ipynb @@ -225,8 +225,8 @@ "metadata": {}, "outputs": [], "source": [ - "# pyautogen>0.1.14 supports openai>=1\n", - "%pip install \"pyautogen>0.2\" \"openai>1\" -q" + "# autogen-agentchat>0.1.14 supports openai>=1\n", + "%pip install \"autogen-agentchat~=0.2\" \"openai>1\" -q" ] }, { @@ -418,7 +418,7 @@ }, "outputs": [], "source": [ - "%pip install \"pyautogen[retrievechat,lmm]>=0.2.28\" -q" + "%pip install \"autogen-agentchat[retrievechat,lmm]~=0.2\" -q" ] }, { diff --git a/notebook/agentchat_multi_task_async_chats.ipynb b/notebook/agentchat_multi_task_async_chats.ipynb index ad75618a5460..86b22edd2bec 100644 --- a/notebook/agentchat_multi_task_async_chats.ipynb +++ b/notebook/agentchat_multi_task_async_chats.ipynb @@ -15,9 +15,9 @@ "\n", "\\:\\:\\:info Requirements\n", "\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchat_multi_task_chats.ipynb b/notebook/agentchat_multi_task_chats.ipynb index 2c200f52354e..5defb22b13d0 100644 --- a/notebook/agentchat_multi_task_chats.ipynb +++ b/notebook/agentchat_multi_task_chats.ipynb @@ -15,9 +15,9 @@ "\n", "\\:\\:\\:info Requirements\n", "\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchat_nested_chats_chess.ipynb b/notebook/agentchat_nested_chats_chess.ipynb index b3e369fba8ce..e5a22fce5a7f 100644 --- a/notebook/agentchat_nested_chats_chess.ipynb +++ b/notebook/agentchat_nested_chats_chess.ipynb @@ -39,7 +39,7 @@ "source": [ "## Installation\n", "\n", - "First you need to install the `pyautogen` and `chess` packages to use AutoGen." + "First you need to install the `autogen-agentchat~=0.2` and `chess` packages to use AutoGen." ] }, { @@ -48,7 +48,7 @@ "metadata": {}, "outputs": [], "source": [ - "! pip install -qqq pyautogen chess" + "! pip install -qqq autogen-agentchat~=0.2 chess" ] }, { diff --git a/notebook/agentchat_nested_chats_chess_altmodels.ipynb b/notebook/agentchat_nested_chats_chess_altmodels.ipynb index 69d3edbcfb50..8980a87e8818 100644 --- a/notebook/agentchat_nested_chats_chess_altmodels.ipynb +++ b/notebook/agentchat_nested_chats_chess_altmodels.ipynb @@ -40,7 +40,7 @@ "source": [ "## Installation\n", "\n", - "First, you need to install the `pyautogen` and `chess` packages to use AutoGen. We'll include Anthropic and Together.AI libraries." + "First, you need to install the `autogen-agentchat~=0.2` and `chess` packages to use AutoGen. We'll include Anthropic and Together.AI libraries." ] }, { @@ -49,7 +49,7 @@ "metadata": {}, "outputs": [], "source": [ - "! pip install -qqq pyautogen[anthropic,together] chess" + "! pip install -qqq autogen-agentchat[anthropic,together]~=0.2 chess" ] }, { diff --git a/notebook/agentchat_nested_sequential_chats.ipynb b/notebook/agentchat_nested_sequential_chats.ipynb index 3c4ca199484c..eb27ef72e351 100644 --- a/notebook/agentchat_nested_sequential_chats.ipynb +++ b/notebook/agentchat_nested_sequential_chats.ipynb @@ -15,9 +15,9 @@ "\n", "\\:\\:\\:info Requirements\n", "\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchat_nestedchat.ipynb b/notebook/agentchat_nestedchat.ipynb index f81f20398591..bfb1cc680581 100644 --- a/notebook/agentchat_nestedchat.ipynb +++ b/notebook/agentchat_nestedchat.ipynb @@ -15,9 +15,9 @@ "\n", "\\:\\:\\:info Requirements\n", "\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchat_nestedchat_optiguide.ipynb b/notebook/agentchat_nestedchat_optiguide.ipynb index c1648bce62b4..cf9f6c5890df 100644 --- a/notebook/agentchat_nestedchat_optiguide.ipynb +++ b/notebook/agentchat_nestedchat_optiguide.ipynb @@ -21,7 +21,7 @@ "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", "\n", "```bash\n", - "pip install pyautogen eventlet gurobipy\n", + "pip install autogen-agentchat~=0.2 eventlet gurobipy\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchat_oai_assistant_function_call.ipynb b/notebook/agentchat_oai_assistant_function_call.ipynb index bc78819fb198..b96d1eef9098 100644 --- a/notebook/agentchat_oai_assistant_function_call.ipynb +++ b/notebook/agentchat_oai_assistant_function_call.ipynb @@ -19,9 +19,9 @@ "AutoGen requires `Python>=3.8`. To run this notebook example, please install:\n", "````{=mdx}\n", ":::info Requirements\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", @@ -36,7 +36,7 @@ "outputs": [], "source": [ "%%capture --no-stderr\n", - "# %pip install \"pyautogen>=0.2.3\"" + "# %pip install \"autogen-agentchat~=0.2\"" ] }, { diff --git a/notebook/agentchat_oai_assistant_groupchat.ipynb b/notebook/agentchat_oai_assistant_groupchat.ipynb index d38fed4cdaee..e236aa1120e9 100644 --- a/notebook/agentchat_oai_assistant_groupchat.ipynb +++ b/notebook/agentchat_oai_assistant_groupchat.ipynb @@ -16,9 +16,9 @@ "AutoGen requires `Python>=3.8`. To run this notebook example, please install:\n", "````{=mdx}\n", ":::info Requirements\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchat_oai_assistant_retrieval.ipynb b/notebook/agentchat_oai_assistant_retrieval.ipynb index ec7e41a00ba8..fec9605161a1 100644 --- a/notebook/agentchat_oai_assistant_retrieval.ipynb +++ b/notebook/agentchat_oai_assistant_retrieval.ipynb @@ -6,7 +6,7 @@ "source": [ "## RAG OpenAI Assistants in AutoGen\n", "\n", - "This notebook shows an example of the [`GPTAssistantAgent`](https://github.com/microsoft/autogen/blob/main/autogen/agentchat/contrib/gpt_assistant_agent.py#L16C43-L16C43) with retrieval augmented generation. `GPTAssistantAgent` is an experimental AutoGen agent class that leverages the [OpenAI Assistant API](https://platform.openai.com/docs/assistants/overview) for conversational capabilities, working with\n", + "This notebook shows an example of the [`GPTAssistantAgent`](https://github.com/microsoft/autogen/blob/0.2/autogen/agentchat/contrib/gpt_assistant_agent.py#L16C43-L16C43) with retrieval augmented generation. `GPTAssistantAgent` is an experimental AutoGen agent class that leverages the [OpenAI Assistant API](https://platform.openai.com/docs/assistants/overview) for conversational capabilities, working with\n", "`UserProxyAgent` in AutoGen." ] }, diff --git a/notebook/agentchat_oai_assistant_twoagents_basic.ipynb b/notebook/agentchat_oai_assistant_twoagents_basic.ipynb index 94da2358c986..295784bea2cc 100644 --- a/notebook/agentchat_oai_assistant_twoagents_basic.ipynb +++ b/notebook/agentchat_oai_assistant_twoagents_basic.ipynb @@ -6,7 +6,7 @@ "source": [ "## OpenAI Assistants in AutoGen\n", "\n", - "This notebook shows a very basic example of the [`GPTAssistantAgent`](https://github.com/microsoft/autogen/blob/main/autogen/agentchat/contrib/gpt_assistant_agent.py#L16C43-L16C43), which is an experimental AutoGen agent class that leverages the [OpenAI Assistant API](https://platform.openai.com/docs/assistants/overview) for conversational capabilities, working with\n", + "This notebook shows a very basic example of the [`GPTAssistantAgent`](https://github.com/microsoft/autogen/blob/0.2/autogen/agentchat/contrib/gpt_assistant_agent.py#L16C43-L16C43), which is an experimental AutoGen agent class that leverages the [OpenAI Assistant API](https://platform.openai.com/docs/assistants/overview) for conversational capabilities, working with\n", "`UserProxyAgent` in AutoGen." ] }, diff --git a/notebook/agentchat_oai_code_interpreter.ipynb b/notebook/agentchat_oai_code_interpreter.ipynb index a8aeb6147896..92eb3e19e23d 100644 --- a/notebook/agentchat_oai_code_interpreter.ipynb +++ b/notebook/agentchat_oai_code_interpreter.ipynb @@ -12,9 +12,9 @@ "AutoGen requires `Python>=3.8`. To run this notebook example, please install:\n", "````{=mdx}\n", ":::info Requirements\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchat_planning.ipynb b/notebook/agentchat_planning.ipynb index 14b393958dc1..14fed1c29e0d 100644 --- a/notebook/agentchat_planning.ipynb +++ b/notebook/agentchat_planning.ipynb @@ -26,9 +26,9 @@ "\n", "## Requirements\n", "\n", - "AutoGen requires `Python>=3.8`. To run this notebook example, please install pyautogen and docker:\n", + "AutoGen requires `Python>=3.8`. To run this notebook example, please install autogen-agentchat and docker:\n", "```bash\n", - "pip install pyautogen docker\n", + "pip install autogen-agentchat~=0.2 docker\n", "```" ] }, @@ -45,7 +45,7 @@ }, "outputs": [], "source": [ - "# %pip install \"pyautogen>=0.2.3\" docker" + "# %pip install \"autogen-agentchat~=0.2\" docker" ] }, { @@ -105,7 +105,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/notebook/oai_openai_utils.ipynb) for full code examples of the different methods.\n", + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/oai_openai_utils.ipynb) for full code examples of the different methods.\n", "\n", "## Construct Agents\n", "\n", diff --git a/notebook/agentchat_society_of_mind.ipynb b/notebook/agentchat_society_of_mind.ipynb index df3a6c543397..091ab2e5519b 100644 --- a/notebook/agentchat_society_of_mind.ipynb +++ b/notebook/agentchat_society_of_mind.ipynb @@ -15,9 +15,9 @@ "\n", "````{=mdx}\n", ":::info Requirements\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", @@ -357,8 +357,11 @@ ], "metadata": { "front_matter": { - "tags": ["orchestration", "nested chat"], - "description": "Explore the demonstration of the SocietyOfMindAgent in the AutoGen library, which runs a group chat as an internal monologue, but appears to the external world as a single agent, offering a structured way to manage complex interactions among multiple agents and handle issues such as extracting responses from complex dialogues and dealing with context window constraints." + "description": "Explore the demonstration of the SocietyOfMindAgent in the AutoGen library, which runs a group chat as an internal monologue, but appears to the external world as a single agent, offering a structured way to manage complex interactions among multiple agents and handle issues such as extracting responses from complex dialogues and dealing with context window constraints.", + "tags": [ + "orchestration", + "nested chat" + ] }, "kernelspec": { "display_name": "Python 3 (ipykernel)", diff --git a/notebook/agentchat_stream.ipynb b/notebook/agentchat_stream.ipynb index 8127cdfbab04..d39895827277 100644 --- a/notebook/agentchat_stream.ipynb +++ b/notebook/agentchat_stream.ipynb @@ -28,7 +28,7 @@ "\n", "AutoGen requires `Python>=3.8`. To run this notebook example, please install:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```" ] }, @@ -45,7 +45,7 @@ }, "outputs": [], "source": [ - "# %pip install \"pyautogen>=0.2.3\"" + "# %pip install \"autogen-agentchat~=0.2\"" ] }, { @@ -102,7 +102,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." ] }, { diff --git a/notebook/agentchat_surfer.ipynb b/notebook/agentchat_surfer.ipynb index 46c4679e3014..18be350a41f5 100644 --- a/notebook/agentchat_surfer.ipynb +++ b/notebook/agentchat_surfer.ipynb @@ -15,7 +15,7 @@ "\n", "AutoGen requires `Python>=3.8`. To run this notebook example, please install AutoGen with the optional `websurfer` dependencies:\n", "```bash\n", - "pip install \"pyautogen[websurfer]\"\n", + "pip install \"autogen-agentchat[websurfer]~=0.2\"\n", "```" ] }, @@ -25,7 +25,7 @@ "metadata": {}, "outputs": [], "source": [ - "# %pip install --quiet \"pyautogen[websurfer]\"" + "# %pip install --quiet \"autogen-agentchat[websurfer]~=0.2\"" ] }, { @@ -479,7 +479,7 @@ "#### Multi-Agent Conversation Framework[​](#multi-agent-conversation-framework \"Direct link to Multi-Agent Conversation Framework\")\n", "\n", "Autogen enables the next-gen LLM applications with a generic multi-agent conversation framework. It offers customizable and conversable agents which integrate LLMs, tools, and humans.\n", - "By automating chat among multiple capable agents, one can easily make them collectively perform tasks autonomously or with human feedback, including tasks that require using tools via code. For [example](https://github.com/microsoft/autogen/blob/main/test/twoagent.py),\n", + "By automating chat among multiple capable agents, one can easily make them collectively perform tasks autonomously or with human feedback, including tasks that require using tools via code. For [example](https://github.com/microsoft/autogen/blob/0.2/test/twoagent.py),\n", "\n", "The figure below shows an example conversation flow with AutoGen.\n", "\n", diff --git a/notebook/agentchat_teachability.ipynb b/notebook/agentchat_teachability.ipynb index ac239f793dc7..dd41727a02e5 100644 --- a/notebook/agentchat_teachability.ipynb +++ b/notebook/agentchat_teachability.ipynb @@ -13,7 +13,7 @@ "\n", "In making decisions about memo storage and retrieval, `Teachability` calls an instance of `TextAnalyzerAgent` to analyze pieces of text in several different ways. This adds extra LLM calls involving a relatively small number of tokens. These calls can add a few seconds to the time a user waits for a response.\n", "\n", - "This notebook demonstrates how `Teachability` can be added to an agent so that it can learn facts, preferences, and skills from users. To chat with a teachable agent yourself, run [chat_with_teachable_agent.py](https://github.com/microsoft/autogen/blob/main/test/agentchat/contrib/capabilities/chat_with_teachable_agent.py).\n", + "This notebook demonstrates how `Teachability` can be added to an agent so that it can learn facts, preferences, and skills from users. To chat with a teachable agent yourself, run [chat_with_teachable_agent.py](https://github.com/microsoft/autogen/blob/0.2/test/agentchat/contrib/capabilities/chat_with_teachable_agent.py).\n", "\n", "## Requirements\n", "\n", @@ -22,7 +22,7 @@ "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", "\n", "```bash\n", - "pip install pyautogen[teachable]\n", + "pip install autogen-agentchat[teachable]~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", @@ -99,8 +99,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[92m\n", - "CLEARING MEMORY\u001B[0m\n" + "\u001b[92m\n", + "CLEARING MEMORY\u001b[0m\n" ] } ], @@ -152,14 +152,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33muser\u001B[0m (to teachable_agent):\n", + "\u001b[33muser\u001b[0m (to teachable_agent):\n", "\n", "What is the Vicuna model?\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mteachable_agent\u001B[0m (to user):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mteachable_agent\u001b[0m (to user):\n", "\n", "The term \"Vicuna model\" does not point to a well-known concept or framework in the realms of science, technology, or social sciences as of my last knowledge update in early 2023. It's possible that the term could be a reference to a proprietary model or a concept that has emerged after my last update or it might be a misspelling or a misunderstanding.\n", "\n", @@ -185,14 +185,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33muser\u001B[0m (to teachable_agent):\n", + "\u001b[33muser\u001b[0m (to teachable_agent):\n", "\n", "Vicuna is a 13B-parameter language model released by Meta.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mteachable_agent\u001B[0m (to user):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mteachable_agent\u001b[0m (to user):\n", "\n", "My apologies for the confusion. As of my last update, the Vicuna model had not been part of my database. If Vicuna is indeed a 13-billion-parameter language model developed by Meta (formerly Facebook Inc.), then it would be one of the large-scale transformer-based models akin to those like GPT-3 by OpenAI.\n", "\n", @@ -222,14 +222,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33muser\u001B[0m (to teachable_agent):\n", + "\u001b[33muser\u001b[0m (to teachable_agent):\n", "\n", "What is the Orca model?\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mteachable_agent\u001B[0m (to user):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mteachable_agent\u001b[0m (to user):\n", "\n", "As of my last update, the Orca model appears to reference a new development that I do not have extensive information on, similar to the earlier reference to the Vicuna model.\n", "\n", @@ -255,14 +255,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33muser\u001B[0m (to teachable_agent):\n", + "\u001b[33muser\u001b[0m (to teachable_agent):\n", "\n", "Orca is a 13B-parameter language model developed by Microsoft. It outperforms Vicuna on most tasks.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mteachable_agent\u001B[0m (to user):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mteachable_agent\u001b[0m (to user):\n", "\n", "Thank you for providing the context about the Orca model. Based on the new information you've given, Orca is a language model with 13 billion parameters, similar in size to Meta's Vicuna model, but developed by Microsoft. If it outperforms Vicuna on most tasks, it suggests that it could have been trained on a more diverse dataset, use a more advanced architecture, have more effective training techniques, or some combination of these factors.\n", "\n", @@ -297,14 +297,14 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33muser\u001B[0m (to teachable_agent):\n", + "\u001b[33muser\u001b[0m (to teachable_agent):\n", "\n", "How does the Vicuna model compare to the Orca model?\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mteachable_agent\u001B[0m (to user):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mteachable_agent\u001b[0m (to user):\n", "\n", "The Vicuna model and the Orca model are both large-scale language models with a significant number of parameters—13 billion, to be exact.\n", "\n", @@ -340,7 +340,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33muser\u001B[0m (to teachable_agent):\n", + "\u001b[33muser\u001b[0m (to teachable_agent):\n", "\n", "Please summarize this abstract.\n", "\n", @@ -350,9 +350,9 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mteachable_agent\u001B[0m (to user):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mteachable_agent\u001b[0m (to user):\n", "\n", "AutoGen is an open-source framework designed to facilitate the creation of applications using large language models (LLMs) through the use of multiple conversational agents. These agents can be tailored to users' needs and are capable of interaction in multiple modes, including with other LLMs, human input, and additional tools. With AutoGen, developers have the flexibility to program agent interactions using both natural language and code, enabling the creation of complex patterns suitable for a wide range of applications. The framework has been proven effective across various fields, such as math, coding, question answering, and entertainment, based on empirical studies conducted to test its capabilities.\n", "\n", @@ -386,7 +386,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33muser\u001B[0m (to teachable_agent):\n", + "\u001b[33muser\u001b[0m (to teachable_agent):\n", "\n", "Please summarize this abstract. \n", "When I'm summarizing an abstract, I try to make the summary contain just three short bullet points: the title, the innovation, and the key empirical results.\n", @@ -397,9 +397,9 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mteachable_agent\u001B[0m (to user):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mteachable_agent\u001b[0m (to user):\n", "\n", "- Title: AutoGen: Enabling Next-Gen LLM Applications via Multi-Agent Conversation\n", "- Innovation: AutoGen, an open-source framework that supports building large language model (LLM) applications by enabling conversation among multiple customizable and conversable agents.\n", @@ -436,7 +436,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33muser\u001B[0m (to teachable_agent):\n", + "\u001b[33muser\u001b[0m (to teachable_agent):\n", "\n", "Please summarize this abstract.\n", "\n", @@ -445,9 +445,9 @@ "Artificial intelligence (AI) researchers have been developing and refining large language models (LLMs) that exhibit remarkable capabilities across a variety of domains and tasks, challenging our understanding of learning and cognition. The latest model developed by OpenAI, GPT-4, was trained using an unprecedented scale of compute and data. In this paper, we report on our investigation of an early version of GPT-4, when it was still in active development by OpenAI. We contend that (this early version of) GPT-4 is part of a new cohort of LLMs (along with ChatGPT and Google's PaLM for example) that exhibit more general intelligence than previous AI models. We discuss the rising capabilities and implications of these models. We demonstrate that, beyond its mastery of language, GPT-4 can solve novel and difficult tasks that span mathematics, coding, vision, medicine, law, psychology and more, without needing any special prompting. Moreover, in all of these tasks, GPT-4's performance is strikingly close to human-level performance, and often vastly surpasses prior models such as ChatGPT. Given the breadth and depth of GPT-4's capabilities, we believe that it could reasonably be viewed as an early (yet still incomplete) version of an artificial general intelligence (AGI) system. In our exploration of GPT-4, we put special emphasis on discovering its limitations, and we discuss the challenges ahead for advancing towards deeper and more comprehensive versions of AGI, including the possible need for pursuing a new paradigm that moves beyond next-word prediction. We conclude with reflections on societal influences of the recent technological leap and future research directions.\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mteachable_agent\u001B[0m (to user):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mteachable_agent\u001b[0m (to user):\n", "\n", "- Title: Sparks of Artificial General Intelligence: Early experiments with GPT-4\n", "\n", @@ -487,7 +487,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33muser\u001B[0m (to teachable_agent):\n", + "\u001b[33muser\u001b[0m (to teachable_agent):\n", "\n", "Consider the identity: \n", "9 * 4 + 6 * 6 = 72\n", @@ -496,9 +496,9 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mteachable_agent\u001B[0m (to user):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mteachable_agent\u001b[0m (to user):\n", "\n", "To solve this problem, we need to find a way to add exactly 27 (since 99 - 72 = 27) to the left hand side of the equation by modifying only one of the integers in the equation. \n", "\n", @@ -563,7 +563,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33muser\u001B[0m (to teachable_agent):\n", + "\u001b[33muser\u001b[0m (to teachable_agent):\n", "\n", "Consider the identity: \n", "9 * 4 + 6 * 6 = 72\n", @@ -584,9 +584,9 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mteachable_agent\u001B[0m (to user):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mteachable_agent\u001b[0m (to user):\n", "\n", "Given the new set of instructions and the correction that according to a past memory, the solution is \"9 * 1 + 6 * 9\", let's follow the steps carefully to arrive at the correct modified equation.\n", "\n", @@ -668,7 +668,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33muser\u001B[0m (to teachable_agent):\n", + "\u001b[33muser\u001b[0m (to teachable_agent):\n", "\n", "Consider the identity: \n", "9 * 4 + 6 * 6 = 72\n", @@ -677,9 +677,9 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mteachable_agent\u001B[0m (to user):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mteachable_agent\u001b[0m (to user):\n", "\n", "Let's apply the steps you've provided to solve the problem at hand:\n", "\n", @@ -740,7 +740,7 @@ "name": "stdout", "output_type": "stream", "text": [ - "\u001B[33muser\u001B[0m (to teachable_agent):\n", + "\u001b[33muser\u001b[0m (to teachable_agent):\n", "\n", "Consider the identity: \n", "8 * 3 + 7 * 9 = 87\n", @@ -749,9 +749,9 @@ "\n", "\n", "--------------------------------------------------------------------------------\n", - "\u001B[31m\n", - ">>>>>>>> USING AUTO REPLY...\u001B[0m\n", - "\u001B[33mteachable_agent\u001B[0m (to user):\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mteachable_agent\u001b[0m (to user):\n", "\n", "Let's apply the plan step-by-step to find the correct modification:\n", "\n", diff --git a/notebook/agentchat_teachable_oai_assistants.ipynb b/notebook/agentchat_teachable_oai_assistants.ipynb index 3753be414f39..be37b0d7b5cd 100644 --- a/notebook/agentchat_teachable_oai_assistants.ipynb +++ b/notebook/agentchat_teachable_oai_assistants.ipynb @@ -28,7 +28,7 @@ "\n", "AutoGen requires `Python>=3.8`. To run this notebook example, please install the [teachable] option.\n", "```bash\n", - "pip install \"pyautogen[teachable]\"\n", + "pip install \"autogen-agentchat[teachable]~=0.2\"\n", "```" ] }, @@ -39,7 +39,7 @@ "outputs": [], "source": [ "%%capture --no-stderr\n", - "# %pip install \"pyautogen[teachable]\"" + "# %pip install \"autogen-agentchat[teachable]~=0.2\"" ] }, { @@ -860,7 +860,7 @@ "metadata": {}, "source": [ "### Create the OpenAI Assistant with retrieval as a tool\n", - "For this example, first upload the [conversable_agent.py](https://github.com/microsoft/autogen/blob/main/autogen/agentchat/conversable_agent.py) file to your OpenAI API account. This can be done manually through the [website](https://platform.openai.com/assistants). Then find the uploaded File ID on the [Files page](https://platform.openai.com/files), and paste that ID into the `file_ids` list in the code below." + "For this example, first upload the [conversable_agent.py](https://github.com/microsoft/autogen/blob/0.2/autogen/agentchat/conversable_agent.py) file to your OpenAI API account. This can be done manually through the [website](https://platform.openai.com/assistants). Then find the uploaded File ID on the [Files page](https://platform.openai.com/files), and paste that ID into the `file_ids` list in the code below." ] }, { diff --git a/notebook/agentchat_teaching.ipynb b/notebook/agentchat_teaching.ipynb index a61f3c7e08ed..4d9564276cba 100644 --- a/notebook/agentchat_teaching.ipynb +++ b/notebook/agentchat_teaching.ipynb @@ -16,9 +16,9 @@ "\n", "````{=mdx}\n", ":::info Requirements\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchat_transform_messages.ipynb b/notebook/agentchat_transform_messages.ipynb index d0216e05dd2d..fe9ca5d1f519 100644 --- a/notebook/agentchat_transform_messages.ipynb +++ b/notebook/agentchat_transform_messages.ipynb @@ -12,9 +12,9 @@ "\n", "````{=mdx}\n", ":::info Requirements\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchat_two_users.ipynb b/notebook/agentchat_two_users.ipynb index eb9e0c1fbf28..c4fc85861f52 100644 --- a/notebook/agentchat_two_users.ipynb +++ b/notebook/agentchat_two_users.ipynb @@ -27,7 +27,7 @@ "\n", "AutoGen requires `Python>=3.8`. To run this notebook example, please install:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```" ] }, @@ -44,7 +44,7 @@ }, "outputs": [], "source": [ - "# %pip install \"pyautogen>=0.2.3\"" + "# %pip install \"autogen-agentchat~=0.2\"" ] }, { @@ -82,7 +82,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/website/docs/topics/llm_configuration.ipynb) for full code examples of the different methods." ] }, { diff --git a/notebook/agentchat_video_transcript_translate_with_whisper.ipynb b/notebook/agentchat_video_transcript_translate_with_whisper.ipynb index 48812ad01a61..51ad4c6268aa 100644 --- a/notebook/agentchat_video_transcript_translate_with_whisper.ipynb +++ b/notebook/agentchat_video_transcript_translate_with_whisper.ipynb @@ -8,7 +8,7 @@ "# Translating Video audio using Whisper and GPT-3.5-turbo\n", "\n", "In this notebook, we demonstrate how to use whisper and GPT-3.5-turbo with `AssistantAgent` and `UserProxyAgent` to recognize and translate\n", - "the speech sound from a video file and add the timestamp like a subtitle file based on [agentchat_function_call.ipynb](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_function_call.ipynb)\n" + "the speech sound from a video file and add the timestamp like a subtitle file based on [agentchat_function_call.ipynb](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_function_call.ipynb)\n" ] }, { @@ -23,7 +23,7 @@ "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", "\n", "```bash\n", - "pip install pyautogen openai openai-whisper\n", + "pip install autogen-agentchat~=0.2 openai openai-whisper\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchat_web_info.ipynb b/notebook/agentchat_web_info.ipynb index f990c128b78c..47660502fa76 100644 --- a/notebook/agentchat_web_info.ipynb +++ b/notebook/agentchat_web_info.ipynb @@ -30,9 +30,9 @@ "\n", "## Requirements\n", "\n", - "AutoGen requires `Python>=3.8`. To run this notebook example, please install pyautogen and docker:\n", + "AutoGen requires `Python>=3.8`. To run this notebook example, please install autogen-agentchat and docker:\n", "```bash\n", - "pip install pyautogen docker\n", + "pip install autogen-agentchat~=0.2 docker\n", "```" ] }, @@ -49,7 +49,7 @@ }, "outputs": [], "source": [ - "# %pip install \"pyautogen>=0.2.3\" docker" + "# %pip install \"autogen-agentchat~=0.2\" docker" ] }, { @@ -116,7 +116,7 @@ "]\n", "```\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/notebook/oai_openai_utils.ipynb) for full code examples of the different methods." + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/oai_openai_utils.ipynb) for full code examples of the different methods." ] }, { diff --git a/notebook/agentchat_webcrawling_with_spider.ipynb b/notebook/agentchat_webcrawling_with_spider.ipynb new file mode 100644 index 000000000000..45d270b37e5e --- /dev/null +++ b/notebook/agentchat_webcrawling_with_spider.ipynb @@ -0,0 +1,426 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Web Scraping using Spider API\n", + "\n", + "This notebook shows how to use the open \n", + "source [Spider](https://spider.cloud/) web crawler together with AutoGen agents." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "First we need to install the Spider SDK and the AutoGen library." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "! pip install -qqq pyautogen spider-client" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Setting up the LLM configuration and the Spider API key is also required." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "config_list = [\n", + " {\"model\": \"gpt-4o\", \"api_key\": os.getenv(\"OPENAI_API_KEY\")},\n", + "]\n", + "\n", + "spider_api_key = os.getenv(\"SPIDER_API_KEY\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's define the tool for scraping and crawling data from any website with Spider.\n", + "Read more about tool use in this [tutorial chapter](/docs/tutorial/tool-use)." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[{'content': 'Spider - The Fastest Web Crawling Service[Spider v1 Logo Spider ](/)[Pricing](/credits/new)[GitHub](https://github.com/spider-rs/spider) [Twitter](https://twitter.com/spider_rust) Toggle ThemeSign InRegisterTo help you get started with Spider, we’ll give you $200 in credits when you spend $100. [Get Credits](/credits/new)LangChain integration [now available](https://python.langchain.com/docs/integrations/document_loaders/spider)The World\\'s Fastest and Cheapest Crawler API==========View Demo* Basic* StreamingExample requestPythonCopy```import requests, osheaders = { \\'Authorization\\': os.environ[\"SPIDER_API_KEY\"], \\'Content-Type\\': \\'application/json\\',}json_data = {\"limit\":50,\"url\":\"http://www.example.com\"}response = requests.post(\\'https://api.spider.cloud/crawl\\', headers=headers, json=json_data)print(response.json())```Example ResponseUnmatched Speed----------### 2.5secs ###To crawl 200 pages### 100-500x ###Faster than alternatives### 500x ###Cheaper than traditional scraping services Benchmarks displaying performance between Spider Cloud, Firecrawl, and Apify.Example used tailwindcss.com - 04/16/2024[See framework benchmarks ](https://github.com/spider-rs/spider/blob/main/benches/BENCHMARKS.md)Foundations for Crawling Effectively----------### Leading in performance ###Spider is written in Rust and runs in full concurrency to achieve crawling dozens of pages in secs.### Optimal response format ###Get clean and formatted markdown, HTML, or text content for fine-tuning or training AI models.### Caching ###Further boost speed by caching repeated web page crawls.### Smart Mode ###Spider dynamically switches to Headless Chrome when it needs to.Beta### Scrape with AI ###Do custom browser scripting and data extraction using the latest AI models.### Best crawler for LLMs ###Don\\'t let crawling and scraping be the highest latency in your LLM & AI agent stack.### Scrape with no headaches ###* Proxy rotations* Agent headers* Avoid anti-bot detections* Headless chrome* Markdown LLM Responses### The Fastest Web Crawler ###* Powered by [spider-rs](https://github.com/spider-rs/spider)* Do 20,000 pages in seconds* Full concurrency* Powerful and simple API* 5,000 requests per minute### Do more with AI ###* Custom browser scripting* Advanced data extraction* Data pipelines* Perfect for LLM and AI Agents* Accurate website labelingSee what\\'s being said----------[\"Merrick](https://twitter.com/iammerrick/status/1787873425446572462)[Merrick Christensen](https://twitter.com/iammerrick/status/1787873425446572462)[@iammerrick ](https://twitter.com/iammerrick/status/1787873425446572462)· [Follow](https://twitter.com/intent/follow?screen_name=iammerrick)[](https://twitter.com/iammerrick/status/1787873425446572462)Rust based crawler Spider is next level for crawling & scraping sites. So fast. Their cloud offering is also so easy to use. Good stuff. [ github.com/spider-rs/spid… ](https://github.com/spider-rs/spider)[ 3:53 PM · May 7, 2024 ](https://twitter.com/iammerrick/status/1787873425446572462) [](https://help.twitter.com/en/twitter-for-websites-ads-info-and-privacy)[12 ](https://twitter.com/intent/like?tweet_id=1787873425446572462) [Reply ](https://twitter.com/intent/tweet?in_reply_to=1787873425446572462)[ Read more on Twitter ](https://twitter.com/iammerrick/status/1787873425446572462)[\"William](https://twitter.com/WilliamEspegren/status/1789419820821184764)[William Espegren](https://twitter.com/WilliamEspegren/status/1789419820821184764)[@WilliamEspegren ](https://twitter.com/WilliamEspegren/status/1789419820821184764)· [Follow](https://twitter.com/intent/follow?screen_name=WilliamEspegren)[](https://twitter.com/WilliamEspegren/status/1789419820821184764)Web crawler built in rust, currently the nr1 performance in the world with crazy resource management Aaaaaaand they have a cloud offer, that’s wayyyy cheaper than any competitor Name a reason for me to use anything else? [ github.com/spider-rs/spid… ](https://github.com/spider-rs/spider)[ 10:18 PM · May 11, 2024 ](https://twitter.com/WilliamEspegren/status/1789419820821184764) [](https://help.twitter.com/en/twitter-for-websites-ads-info-and-privacy)[2 ](https://twitter.com/intent/like?tweet_id=1789419820821184764) [Reply ](https://twitter.com/intent/tweet?in_reply_to=1789419820821184764)[ Read 1 reply ](https://twitter.com/WilliamEspegren/status/1789419820821184764)[\"Troy](https://twitter.com/Troyusrex/status/1791497607925088307)[Troy Lowry](https://twitter.com/Troyusrex/status/1791497607925088307)[@Troyusrex ](https://twitter.com/Troyusrex/status/1791497607925088307)· [Follow](https://twitter.com/intent/follow?screen_name=Troyusrex)[](https://twitter.com/Troyusrex/status/1791497607925088307)[ @spider\\\\_rust ](https://twitter.com/spider_rust) First, the good: Spider has enabled me to speed up my scraping 20X and with a bit higher quality than I was getting before. I am having a few issues however. First, the documentation link doesn\\'t work ([ spider.cloud/guides/(/docs/… ](https://spider.cloud/guides/(/docs/api)))I\\'ve figured out how to get it to work…[ 3:54 PM · May 17, 2024 ](https://twitter.com/Troyusrex/status/1791497607925088307) [](https://help.twitter.com/en/twitter-for-websites-ads-info-and-privacy)[1 ](https://twitter.com/intent/like?tweet_id=1791497607925088307) [Reply ](https://twitter.com/intent/tweet?in_reply_to=1791497607925088307)[ Read 2 replies ](https://twitter.com/Troyusrex/status/1791497607925088307)FAQ----------Frequently asked questions about Spider
What is Spider?----------Spider is a leading web crawling tool designed for speed and cost-effectiveness, supporting various data formats including LLM-ready markdown.
Why is my website not crawling?----------Your crawl may fail if it requires JavaScript rendering. Try setting your request to \\'chrome\\' to solve this issue.
Can you crawl all pages?----------Yes, Spider accurately crawls all necessary content without needing a sitemap.
What formats can Spider convert web data into?----------Spider outputs HTML, raw, text, and various markdown formats. It supports JSON, JSONL, CSV, and XML for API responses.
Is Spider suitable for large scraping projects?----------Absolutely, Spider is ideal for large-scale data collection and offers a cost-effective dashboard for data management.
How can I try Spider?----------Purchase credits for our cloud system or test the Open Source Spider engine to explore its capabilities.
Does it respect robots.txt?----------Yes, compliance with robots.txt is default, but you can disable this if necessary.
[API](/docs/api) [Pricing](/credits/new) [Guides](/guides) [About](/about) [Docs](https://docs.rs/spider/latest/spider/) [Privacy](/privacy) [Terms](/eula) [FAQ](/faq)© 2024 Spider from A11yWatch[GitHubGithub](https://github.com/spider-rs/spider) [X - Twitter ](https://twitter.com/spider_rust)', 'error': None, 'status': 200, 'url': 'https://spider.cloud'}]\n" + ] + } + ], + "source": [ + "from typing import Any, Dict, List\n", + "\n", + "from spider import Spider\n", + "from typing_extensions import Annotated\n", + "\n", + "\n", + "def scrape_page(\n", + " url: Annotated[str, \"The URL of the web page to scrape\"],\n", + " params: Annotated[dict, \"Dictionary of additional params.\"] = None,\n", + ") -> Annotated[Dict[str, Any], \"Scraped content\"]:\n", + " # Initialize the Spider client with your API key, if no api key is specified it looks for SPIDER_API_KEY in your environment variables\n", + " client = Spider(spider_api_key)\n", + "\n", + " if params is None:\n", + " params = {\"return_format\": \"markdown\"}\n", + "\n", + " scraped_data = client.scrape_url(url, params)\n", + " return scraped_data[0]\n", + "\n", + "\n", + "def crawl_page(\n", + " url: Annotated[str, \"The url of the domain to be crawled\"],\n", + " params: Annotated[dict, \"Dictionary of additional params.\"] = None,\n", + ") -> Annotated[List[Dict[str, Any]], \"Scraped content\"]:\n", + " # Initialize the Spider client with your API key, if no api key is specified it looks for SPIDER_API_KEY in your environment variables\n", + " client = Spider(spider_api_key)\n", + "\n", + " if params is None:\n", + " params = {\"return_format\": \"markdown\"}\n", + "\n", + " crawled_data = client.crawl_url(url, params)\n", + " return crawled_data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create the agents and register the tool." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "from autogen import ConversableAgent, register_function\n", + "\n", + "# Create web scraper agent.\n", + "scraper_agent = ConversableAgent(\n", + " \"WebScraper\",\n", + " llm_config={\"config_list\": config_list},\n", + " system_message=\"You are a web scraper and you can scrape any web page to retrieve its contents.\"\n", + " \"Returns 'TERMINATE' when the scraping is done.\",\n", + ")\n", + "\n", + "# Create web crawler agent.\n", + "crawler_agent = ConversableAgent(\n", + " \"WebCrawler\",\n", + " llm_config={\"config_list\": config_list},\n", + " system_message=\"You are a web crawler and you can crawl any page with deeper crawling following subpages.\"\n", + " \"Returns 'TERMINATE' when the scraping is done.\",\n", + ")\n", + "\n", + "# Create user proxy agent.\n", + "user_proxy_agent = ConversableAgent(\n", + " \"UserProxy\",\n", + " llm_config=False, # No LLM for this agent.\n", + " human_input_mode=\"NEVER\",\n", + " code_execution_config=False, # No code execution for this agent.\n", + " is_termination_msg=lambda x: x.get(\"content\", \"\") is not None and \"terminate\" in x[\"content\"].lower(),\n", + " default_auto_reply=\"Please continue if not finished, otherwise return 'TERMINATE'.\",\n", + ")\n", + "\n", + "# Register the functions with the agents.\n", + "register_function(\n", + " scrape_page,\n", + " caller=scraper_agent,\n", + " executor=user_proxy_agent,\n", + " name=\"scrape_page\",\n", + " description=\"Scrape a web page and return the content.\",\n", + ")\n", + "\n", + "register_function(\n", + " crawl_page,\n", + " caller=crawler_agent,\n", + " executor=user_proxy_agent,\n", + " name=\"crawl_page\",\n", + " description=\"Crawl an entire domain, following subpages and return the content.\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Start the conversation for scraping web data. We used the\n", + "`reflection_with_llm` option for summary method\n", + "to perform the formatting of the output into a desired format.\n", + "The summary method is called after the conversation is completed\n", + "given the complete history of the conversation." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mUserProxy\u001b[0m (to WebScraper):\n", + "\n", + "Can you scrape william-espegren.com for me?\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mWebScraper\u001b[0m (to UserProxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_qCNYeQCfIPZkUCKejQmm5EhC): scrape_page *****\u001b[0m\n", + "Arguments: \n", + "{\"url\":\"https://www.william-espegren.com\"}\n", + "\u001b[32m****************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION scrape_page...\u001b[0m\n", + "\u001b[33mUserProxy\u001b[0m (to WebScraper):\n", + "\n", + "\u001b[33mUserProxy\u001b[0m (to WebScraper):\n", + "\n", + "\u001b[32m***** Response from calling tool (call_qCNYeQCfIPZkUCKejQmm5EhC) *****\u001b[0m\n", + "[{\"content\": \"William Espegren - Portfoliokeep scrollingMADE WITHCSS, JSMADE BYUppsalaWilliam EspegrenWith \\u00b7LoveOpen For Projects[CONTACT ME](https://www.linkedin.com/in/william-espegren/)[Instagram](https://www.instagram.com/williamespegren/)[LinkedIn](https://www.linkedin.com/in/william-espegren/)[Twitter](https://twitter.com/WilliamEspegren)[team-collaboration/version-control/github Created with Sketch.Github](https://github.com/WilliamEspegren)\", \"error\": null, \"status\": 200, \"url\": \"https://www.william-espegren.com\"}]\n", + "\u001b[32m**********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mWebScraper\u001b[0m (to UserProxy):\n", + "\n", + "I successfully scraped the website \"william-espegren.com\". Here is the content retrieved:\n", + "\n", + "```\n", + "William Espegren - Portfolio\n", + "\n", + "keep scrolling\n", + "\n", + "MADE WITH\n", + "CSS, JS\n", + "\n", + "MADE BY\n", + "Uppsala\n", + "\n", + "William Espegren\n", + "With Love\n", + "\n", + "Open For Projects\n", + "\n", + "[CONTACT ME](https://www.linkedin.com/in/william-espegren/)\n", + "[Instagram](https://www.instagram.com/williamespegren/)\n", + "[LinkedIn](https://www.linkedin.com/in/william-espegren/)\n", + "[Twitter](https://twitter.com/WilliamEspegren)\n", + "[Github](https://github.com/WilliamEspegren)\n", + "```\n", + "\n", + "Is there anything specific you would like to do with this information?\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mUserProxy\u001b[0m (to WebScraper):\n", + "\n", + "Please continue if not finished, otherwise return 'TERMINATE'.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mWebScraper\u001b[0m (to UserProxy):\n", + "\n", + "TERMINATE\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "# Scrape page\n", + "scraped_chat_result = user_proxy_agent.initiate_chat(\n", + " scraper_agent,\n", + " message=\"Can you scrape william-espegren.com for me?\",\n", + " summary_method=\"reflection_with_llm\",\n", + " summary_args={\"summary_prompt\": \"\"\"Summarize the scraped content\"\"\"},\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mUserProxy\u001b[0m (to WebCrawler):\n", + "\n", + "Can you crawl william-espegren.com for me, I want the whole domains information?\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mWebCrawler\u001b[0m (to UserProxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_0FkTtsxBtA0SbChm1PX085Vk): crawl_page *****\u001b[0m\n", + "Arguments: \n", + "{\"url\":\"http://www.william-espegren.com\"}\n", + "\u001b[32m***************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION crawl_page...\u001b[0m\n", + "\u001b[33mUserProxy\u001b[0m (to WebCrawler):\n", + "\n", + "\u001b[33mUserProxy\u001b[0m (to WebCrawler):\n", + "\n", + "\u001b[32m***** Response from calling tool (call_0FkTtsxBtA0SbChm1PX085Vk) *****\u001b[0m\n", + "[{\"content\": \"William Espegren - Portfoliokeep scrollingMADE WITHCSS, JSMADE BYUppsalaWilliam EspegrenWith \\u00b7LoveOpen For Projects[CONTACT ME](https://www.linkedin.com/in/william-espegren/)[Instagram](https://www.instagram.com/williamespegren/)[LinkedIn](https://www.linkedin.com/in/william-espegren/)[Twitter](https://twitter.com/WilliamEspegren)[team-collaboration/version-control/github Created with Sketch.Github](https://github.com/WilliamEspegren)\", \"error\": null, \"status\": 200, \"url\": \"http://www.william-espegren.com\"}]\n", + "\u001b[32m**********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mWebCrawler\u001b[0m (to UserProxy):\n", + "\n", + "The crawl of [william-espegren.com](http://www.william-espegren.com) has been completed. Here is the gathered content:\n", + "\n", + "---\n", + "\n", + "**William Espegren - Portfolio**\n", + "\n", + "Keep scrolling\n", + "\n", + "**MADE WITH:** CSS, JS\n", + "\n", + "**MADE BY:** Uppsala\n", + "\n", + "**William Espegren**\n", + "\n", + "**With Love**\n", + "\n", + "**Open For Projects**\n", + "\n", + "**[CONTACT ME](https://www.linkedin.com/in/william-espegren/)**\n", + "\n", + "- [Instagram](https://www.instagram.com/williamespegren/)\n", + "- [LinkedIn](https://www.linkedin.com/in/william-espegren/)\n", + "- [Twitter](https://twitter.com/WilliamEspegren)\n", + "- [Github](https://github.com/WilliamEspegren)\n", + "\n", + "---\n", + "\n", + "If you need further information or details from any specific section, please let me know!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mUserProxy\u001b[0m (to WebCrawler):\n", + "\n", + "Please continue if not finished, otherwise return 'TERMINATE'.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[33mWebCrawler\u001b[0m (to UserProxy):\n", + "\n", + "TERMINATE\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "# Crawl page\n", + "crawled_chat_result = user_proxy_agent.initiate_chat(\n", + " crawler_agent,\n", + " message=\"Can you crawl william-espegren.com for me, I want the whole domains information?\",\n", + " summary_method=\"reflection_with_llm\",\n", + " summary_args={\"summary_prompt\": \"\"\"Summarize the crawled content\"\"\"},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The output is stored in the summary." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "The website belongs to William Espegren, who is based in Uppsala and possesses skills in CSS and JavaScript. He is open to new projects. You can contact him through the following links:\n", + "\n", + "- [LinkedIn](https://www.linkedin.com/in/william-espegren/)\n", + "- [Instagram](https://www.instagram.com/williamespegren/)\n", + "- [Twitter](https://twitter.com/WilliamEspegren)\n", + "- [GitHub](https://github.com/WilliamEspegren)\n", + "\n", + "Feel free to reach out to him for project collaborations.\n" + ] + } + ], + "source": [ + "print(scraped_chat_result.summary)\n", + "# print(crawled_chat_result.summary) # We show one for cleaner output" + ] + } + ], + "metadata": { + "front_matter": { + "description": "Scraping/Crawling web pages and summarizing the content using agents.", + "tags": [ + "web scraping", + "spider", + "tool use" + ], + "title": "Web Scraper & Crawler Agent using Spider" + }, + "kernelspec": { + "display_name": "autogen", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/notebook/agentchat_webscraping_with_apify.ipynb b/notebook/agentchat_webscraping_with_apify.ipynb index 0429c10f8a7f..c1fec78d83bc 100644 --- a/notebook/agentchat_webscraping_with_apify.ipynb +++ b/notebook/agentchat_webscraping_with_apify.ipynb @@ -23,7 +23,7 @@ "metadata": {}, "outputs": [], "source": [ - "! pip install -qqq pyautogen apify-client" + "! pip install -qqq autogen-agentchat~=0.2 apify-client" ] }, { diff --git a/notebook/agentchat_websockets.ipynb b/notebook/agentchat_websockets.ipynb index 7e6e449675c1..107c7bc40e65 100644 --- a/notebook/agentchat_websockets.ipynb +++ b/notebook/agentchat_websockets.ipynb @@ -28,7 +28,7 @@ "Some extra dependencies are needed for this notebook, which can be installed via pip:\n", "\n", "```bash\n", - "pip install pyautogen[websockets] fastapi uvicorn\n", + "pip install autogen-agentchat[websockets]~=0.2 fastapi uvicorn\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agentchats_sequential_chats.ipynb b/notebook/agentchats_sequential_chats.ipynb index cffcbfdefcb6..3fdd857cd03f 100644 --- a/notebook/agentchats_sequential_chats.ipynb +++ b/notebook/agentchats_sequential_chats.ipynb @@ -15,9 +15,9 @@ "\n", "\\:\\:\\:info Requirements\n", "\n", - "Install `pyautogen`:\n", + "Install `autogen-agentchat`:\n", "```bash\n", - "pip install pyautogen\n", + "pip install autogen-agentchat~=0.2\n", "```\n", "\n", "For more information, please refer to the [installation guide](/docs/installation/).\n", diff --git a/notebook/agenteval_cq_math.ipynb b/notebook/agenteval_cq_math.ipynb index 43ea28de1a31..a67b0ae992ca 100644 --- a/notebook/agenteval_cq_math.ipynb +++ b/notebook/agenteval_cq_math.ipynb @@ -30,12 +30,12 @@ "\n", "## Requirements\n", "\n", - "AutoGen requires `Python>=3.8`. To run this notebook example, please install pyautogen, Docker, and OpenAI:\n" + "AutoGen requires `Python>=3.8`. To run this notebook example, please install autogen-agentchat, Docker, and OpenAI:\n" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "colab": { "base_uri": "https://localhost:8080/" @@ -49,72 +49,9 @@ "id": "68lTZZyJ1_BI", "outputId": "15a55fab-e13a-4654-b8cb-ae117478d6d8" }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Defaulting to user installation because normal site-packages is not writeable\n", - "Requirement already satisfied: pyautogen>=0.2.3 in /home/vscode/.local/lib/python3.10/site-packages (0.2.17)\n", - "Requirement already satisfied: docker in /home/vscode/.local/lib/python3.10/site-packages (7.0.0)\n", - "Requirement already satisfied: diskcache in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (5.6.3)\n", - "Requirement already satisfied: flaml in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (2.1.2)\n", - "Requirement already satisfied: tiktoken in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (0.6.0)\n", - "Requirement already satisfied: openai>=1.3 in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (1.14.1)\n", - "Requirement already satisfied: pydantic!=2.6.0,<3,>=1.10 in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (2.6.4)\n", - "Requirement already satisfied: termcolor in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (2.4.0)\n", - "Requirement already satisfied: python-dotenv in /home/vscode/.local/lib/python3.10/site-packages (from pyautogen>=0.2.3) (1.0.1)\n", - "Requirement already satisfied: requests>=2.26.0 in /usr/local/lib/python3.10/site-packages (from docker) (2.31.0)\n", - "Requirement already satisfied: packaging>=14.0 in /usr/local/lib/python3.10/site-packages (from docker) (24.0)\n", - "Requirement already satisfied: urllib3>=1.26.0 in /usr/local/lib/python3.10/site-packages (from docker) (2.2.1)\n", - "Requirement already satisfied: tqdm>4 in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (4.66.2)\n", - "Requirement already satisfied: httpx<1,>=0.23.0 in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (0.27.0)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (1.9.0)\n", - "Requirement already satisfied: sniffio in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (1.3.1)\n", - "Requirement already satisfied: anyio<5,>=3.5.0 in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (4.3.0)\n", - "Requirement already satisfied: typing-extensions<5,>=4.7 in /home/vscode/.local/lib/python3.10/site-packages (from openai>=1.3->pyautogen>=0.2.3) (4.10.0)\n", - "Requirement already satisfied: annotated-types>=0.4.0 in /home/vscode/.local/lib/python3.10/site-packages (from pydantic!=2.6.0,<3,>=1.10->pyautogen>=0.2.3) (0.6.0)\n", - "Requirement already satisfied: pydantic-core==2.16.3 in /home/vscode/.local/lib/python3.10/site-packages (from pydantic!=2.6.0,<3,>=1.10->pyautogen>=0.2.3) (2.16.3)\n", - "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/site-packages (from requests>=2.26.0->docker) (2024.2.2)\n", - "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/site-packages (from requests>=2.26.0->docker) (3.6)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/site-packages (from requests>=2.26.0->docker) (3.3.2)\n", - "Requirement already satisfied: NumPy>=1.17 in /home/vscode/.local/lib/python3.10/site-packages (from flaml->pyautogen>=0.2.3) (1.26.4)\n", - "Requirement already satisfied: regex>=2022.1.18 in /home/vscode/.local/lib/python3.10/site-packages (from tiktoken->pyautogen>=0.2.3) (2023.12.25)\n", - "Requirement already satisfied: exceptiongroup>=1.0.2 in /home/vscode/.local/lib/python3.10/site-packages (from anyio<5,>=3.5.0->openai>=1.3->pyautogen>=0.2.3) (1.2.0)\n", - "Requirement already satisfied: httpcore==1.* in /home/vscode/.local/lib/python3.10/site-packages (from httpx<1,>=0.23.0->openai>=1.3->pyautogen>=0.2.3) (1.0.4)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in /home/vscode/.local/lib/python3.10/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai>=1.3->pyautogen>=0.2.3) (0.14.0)\n", - "\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", - "Note: you may need to restart the kernel to use updated packages.\n", - "Defaulting to user installation because normal site-packages is not writeable\n", - "Requirement already satisfied: scipy in /home/vscode/.local/lib/python3.10/site-packages (1.12.0)\n", - "Requirement already satisfied: numpy<1.29.0,>=1.22.4 in /home/vscode/.local/lib/python3.10/site-packages (from scipy) (1.26.4)\n", - "\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", - "Note: you may need to restart the kernel to use updated packages.\n", - "Defaulting to user installation because normal site-packages is not writeable\n", - "Requirement already satisfied: matplotlib in /home/vscode/.local/lib/python3.10/site-packages (3.8.3)\n", - "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/site-packages (from matplotlib) (24.0)\n", - "Requirement already satisfied: pyparsing>=2.3.1 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (3.1.2)\n", - "Requirement already satisfied: contourpy>=1.0.1 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (1.2.0)\n", - "Requirement already satisfied: fonttools>=4.22.0 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (4.50.0)\n", - "Requirement already satisfied: python-dateutil>=2.7 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (2.9.0.post0)\n", - "Requirement already satisfied: cycler>=0.10 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (0.12.1)\n", - "Requirement already satisfied: pillow>=8 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (10.2.0)\n", - "Requirement already satisfied: numpy<2,>=1.21 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (1.26.4)\n", - "Requirement already satisfied: kiwisolver>=1.3.1 in /home/vscode/.local/lib/python3.10/site-packages (from matplotlib) (1.4.5)\n", - "Requirement already satisfied: six>=1.5 in /home/vscode/.local/lib/python3.10/site-packages (from python-dateutil>=2.7->matplotlib) (1.16.0)\n", - "\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.0.1\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", - "Note: you may need to restart the kernel to use updated packages.\n" - ] - } - ], + "outputs": [], "source": [ - "%pip install \"pyautogen>=0.2.3\" docker\n", + "%pip install \"autogen-agentchat~=0.2\" docker\n", "%pip install scipy\n", "%pip install matplotlib" ] @@ -128,7 +65,7 @@ "## Set your API Endpoint\n", "* The [`config_list_from_json`](https://microsoft.github.io/autogen/docs/reference/oai/openai_utils#config_list_from_json) function loads a list of configurations from an environment variable or a json file. It first looks for an environment variable with a specified name. The value of the environment variable needs to be a valid json string. If that variable is not found, it looks for a json file with the same name. It filters the configs by filter_dict.\n", "\n", - "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/main/notebook/oai_openai_utils.ipynb) for full code examples of the different methods.\n" + "You can set the value of config_list in any way you prefer. Please refer to this [notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/oai_openai_utils.ipynb) for full code examples of the different methods.\n" ] }, { diff --git a/notebook/autobuild_agent_library.ipynb b/notebook/autobuild_agent_library.ipynb index bde50355319f..886875436030 100644 --- a/notebook/autobuild_agent_library.ipynb +++ b/notebook/autobuild_agent_library.ipynb @@ -24,7 +24,7 @@ "source": [ "## Requirement\n", "\n", - "AutoBuild require `pyautogen[autobuild]`, which can be installed by the following command:" + "AutoBuild require `autogen-agentchat[autobuild]~=0.2`, which can be installed by the following command:" ] }, { @@ -36,7 +36,7 @@ }, "outputs": [], "source": [ - "%pip install pyautogen[autobuild]" + "%pip install autogen-agentchat[autobuild]~=0.2" ] }, { diff --git a/notebook/autobuild_basic.ipynb b/notebook/autobuild_basic.ipynb index d100563ac257..d26633f2beac 100644 --- a/notebook/autobuild_basic.ipynb +++ b/notebook/autobuild_basic.ipynb @@ -26,7 +26,7 @@ "source": [ "## Requirement\n", "\n", - "AutoBuild require `pyautogen[autobuild]`, which can be installed by the following command:" + "AutoBuild require `autogen-agentchat[autobuild]~=0.2`, which can be installed by the following command:" ] }, { @@ -38,7 +38,7 @@ }, "outputs": [], "source": [ - "%pip install pyautogen[autobuild]" + "%pip install autogen-agentchat[autobuild]~=0.2" ] }, { diff --git a/notebook/autobuild_function_calling.ipynb b/notebook/autobuild_function_calling.ipynb new file mode 100644 index 000000000000..f414de4b84a4 --- /dev/null +++ b/notebook/autobuild_function_calling.ipynb @@ -0,0 +1,470 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# AutoBuild Agents function calling\n", + "By: [Krishna Shedbalkar](https://github.com/krishnashed/)\n", + "\n", + "In this notebook, we introduce a way for Agents created using `Autobuild` to do function calling. Developers can specify a function, function name and function description which will thereafter be assigned and executed by the most suitable agent created using AutoBuild." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Requirement\n", + "\n", + "AutoBuild require `pyautogen[autobuild]`, which can be installed by the following command:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%pip install pyautogen[autobuild]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 1: Prepare configuration and some useful functions\n", + "\n", + "Prepare a `config_file_or_env` for assistant agent to limit the choice of LLM you want to use in this task. This config can be a path of json file or a name of environment variable. A `default_llm_config` is also required for initialize the specific config of LLMs like seed, temperature, etc. Preventing UserProxy agent being called multiple times by adding `allow_repeat_speaker=agent_list[:-1]`" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import autogen\n", + "from autogen.agentchat.contrib.agent_builder import AgentBuilder\n", + "\n", + "config_file_or_env = \"OAI_CONFIG_LIST\"\n", + "config_list = autogen.config_list_from_json(config_file_or_env, filter_dict={\"model\": [\"gpt-4-1106-preview\", \"gpt-4\"]})\n", + "llm_config = {\n", + " \"config_list\": config_list,\n", + " \"timeout\": 120,\n", + "}\n", + "\n", + "\n", + "def start_task(execution_task: str, agent_list: list):\n", + " group_chat = autogen.GroupChat(agents=agent_list, messages=[], allow_repeat_speaker=agent_list[:-1], max_round=12)\n", + " manager = autogen.GroupChatManager(groupchat=group_chat, llm_config={\"config_list\": config_list})\n", + " agent_list[0].initiate_chat(manager, message=execution_task)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 2: Create a AgentBuilder\n", + "\n", + "Create a `AgentBuilder` with the specified `config_path_or_env`. AgentBuilder will use `gpt-4` in default to complete the whole process, you can specify the `builder_model` and `agent_model` to other OpenAI model to match your task. You can also specify an open-source LLM supporting by vLLM and FastChat, see blog for more details." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "builder = AgentBuilder(\n", + " config_file_or_env=config_file_or_env, builder_model=\"gpt-4-1106-preview\", agent_model=\"gpt-4-1106-preview\"\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 3: Specify a building task\n", + "\n", + "Specify a building task with a general description. Building task will help build manager (a LLM) decide what agents should be built." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "building_task = \"Analyze and list the trending topics in arxiv papers related to GPT-4\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 4: Define functions\n", + "\n", + "Define functions to be executed by the Agents of AutoBuild, further specify details like `name`, `description` and `function` of all the functions in an array called `list_of_functions` which will be passed to `builder.build()`" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "import time\n", + "from datetime import datetime, timedelta\n", + "from typing import Dict\n", + "\n", + "import feedparser\n", + "\n", + "\n", + "def get_arxiv_paper_from_a_week(search_topic: str) -> Dict:\n", + " # arXiv API endpoint\n", + " url = \"http://export.arxiv.org/api/query?\"\n", + "\n", + " # Search parameters\n", + " max_results = 10\n", + "\n", + " query = (\n", + " f\"{url}search_query=all:{search_topic}&max_results={max_results}&sortBy=lastUpdatedDate&sortOrder=descending\"\n", + " )\n", + "\n", + " # Parse the feed\n", + " feed = feedparser.parse(query)\n", + "\n", + " now = datetime.now()\n", + " week_ago = now - timedelta(weeks=1)\n", + "\n", + " papers = []\n", + "\n", + " # Get papers from last week\n", + " for entry in feed.entries:\n", + " published_time = datetime.strptime(entry.published, \"%Y-%m-%dT%H:%M:%SZ\")\n", + " if published_time > week_ago:\n", + " list_of_authors = \", \".join(author.name for author in entry.authors)\n", + "\n", + " papers.append(\n", + " {\n", + " \"title\": entry.title,\n", + " \"authors\": list_of_authors,\n", + " \"published_on\": time.strftime(\"%B %d, %Y\", entry.published_parsed),\n", + " \"summary\": entry.summary,\n", + " \"link\": entry.link,\n", + " }\n", + " )\n", + "\n", + " return papers\n", + "\n", + "\n", + "list_of_functions = [\n", + " {\n", + " \"name\": \"get_arxiv_paper_from_a_week\",\n", + " \"description\": \"Get arxiv papers published in last week\",\n", + " \"function\": get_arxiv_paper_from_a_week,\n", + " }\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 5: build group chat agents\n", + "\n", + "Use `build()` to let build manager (the specified `builder_model`) complete the group chat agents generation. Specify `list_of_functions` to be used by the Agents" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[32m==> Generating agents...\u001b[0m\n", + "['NLP_Expert', 'DataAnalysis_Expert', 'AIResearch_Expert'] are generated.\n", + "\u001b[32m==> Generating system message...\u001b[0m\n", + "Preparing system message for NLP_Expert\n", + "Preparing system message for DataAnalysis_Expert\n", + "Preparing system message for AIResearch_Expert\n", + "\u001b[32m==> Generating description...\u001b[0m\n", + "Preparing description for NLP_Expert\n", + "Preparing description for DataAnalysis_Expert\n", + "Preparing description for AIResearch_Expert\n", + "\u001b[32m==> Creating agents...\u001b[0m\n", + "Creating agent NLP_Expert...\n", + "Creating agent DataAnalysis_Expert...\n", + "Creating agent AIResearch_Expert...\n", + "Adding user console proxy...\n", + "Function get_arxiv_paper_from_a_week is registered to agent DataAnalysis_Expert.\n" + ] + } + ], + "source": [ + "agent_list, agent_configs = builder.build(building_task, llm_config, list_of_functions, max_agents=3)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Here you can see that Function `exec_python` has been associated with `ArxivAPI_Expert` Agent." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 6: execute task\n", + "\n", + "Let agents generated in `build()` to complete the task collaboratively in a group chat." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mNLP_Expert\u001b[0m (to chat_manager):\n", + "\n", + "Analyze and list the trending topics in arxiv papers related to GPT-4\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[32m\n", + "Next speaker: DataAnalysis_Expert\n", + "\u001b[0m\n", + "\u001b[33mDataAnalysis_Expert\u001b[0m (to chat_manager):\n", + "\n", + "\u001b[32m***** Suggested tool call (call_hkKs7wbCyAOMkC4QjOYMLgtd): get_arxiv_paper_from_a_week *****\u001b[0m\n", + "Arguments: \n", + "{\"search_topic\":\"GPT-4\"}\n", + "\u001b[32m********************************************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: NLP_Expert\n", + "\u001b[0m\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION get_arxiv_paper_from_a_week...\u001b[0m\n", + "\u001b[33mNLP_Expert\u001b[0m (to chat_manager):\n", + "\n", + "\u001b[33mNLP_Expert\u001b[0m (to chat_manager):\n", + "\n", + "\u001b[32m***** Response from calling tool (call_hkKs7wbCyAOMkC4QjOYMLgtd) *****\u001b[0m\n", + "[{\"title\": \"Self-Training with Direct Preference Optimization Improves\\n Chain-of-Thought Reasoning\", \"authors\": \"Tianduo Wang, Shichen Li, Wei Lu\", \"published_on\": \"July 25, 2024\", \"summary\": \"Effective training of language models (LMs) for mathematical reasoning tasks\\ndemands high-quality supervised fine-tuning data. Besides obtaining annotations\\nfrom human experts, a common alternative is sampling from larger and more\\npowerful LMs. However, this knowledge distillation approach can be costly and\\nunstable, particularly when relying on closed-source, proprietary LMs like\\nGPT-4, whose behaviors are often unpredictable. In this work, we demonstrate\\nthat the reasoning abilities of small-scale LMs can be enhanced through\\nself-training, a process where models learn from their own outputs. We also\\nshow that the conventional self-training can be further augmented by a\\npreference learning algorithm called Direct Preference Optimization (DPO). By\\nintegrating DPO into self-training, we leverage preference data to guide LMs\\ntowards more accurate and diverse chain-of-thought reasoning. We evaluate our\\nmethod across various mathematical reasoning tasks using different base models.\\nOur experiments show that this approach not only improves LMs' reasoning\\nperformance but also offers a more cost-effective and scalable solution\\ncompared to relying on large proprietary LMs.\", \"link\": \"http://arxiv.org/abs/2407.18248v1\"}, {\"title\": \"C2P: Featuring Large Language Models with Causal Reasoning\", \"authors\": \"Abdolmahdi Bagheri, Matin Alinejad, Kevin Bello, Alireza Akhondi-Asl\", \"published_on\": \"July 25, 2024\", \"summary\": \"Causal reasoning is the primary bottleneck that Large Language Models (LLMs)\\nmust overcome to attain human-level intelligence. To address this, we introduce\\nthe Causal Chain of Prompting (C2P) as the first reasoning framework that\\nequips current LLMs with causal reasoning capabilities. C2P operates\\nautonomously, avoiding reliance on external tools or modules during both the\\ncausal learning and reasoning phases, and can be seamlessly implemented during\\nthe training or fine-tuning of LLMs. Experimental results across various\\nbenchmark datasets demonstrate a significant improvement in causal learning and\\nsubsequent reasoning accuracy of LLMs. We illustrate how C2P enhances LLMs'\\nability to causally reason in real-world scenarios, addressing complex problems\\nin fields such as healthcare, medicine, economics, education, social sciences,\\nenvironmental science, and marketing. With few-shot learning, GPT-4 Turbo using\\nC2P with as few as six examples achieves significant performance improvements,\\nboasting over a 33% increase in reasoning accuracy over the most\\nstate-of-the-art LLMs, which perform nearly randomly in similar circumstances.\\nThis demonstrates the transformative potential of integrating C2P into LLM\\ntraining or fine-tuning processes, thereby empowering these models with\\nadvanced causal reasoning capabilities.\", \"link\": \"http://arxiv.org/abs/2407.18069v1\"}, {\"title\": \"Is the Digital Forensics and Incident Response Pipeline Ready for\\n Text-Based Threats in LLM Era?\", \"authors\": \"Avanti Bhandarkar, Ronald Wilson, Anushka Swarup, Mengdi Zhu, Damon Woodard\", \"published_on\": \"July 25, 2024\", \"summary\": \"In the era of generative AI, the widespread adoption of Neural Text\\nGenerators (NTGs) presents new cybersecurity challenges, particularly within\\nthe realms of Digital Forensics and Incident Response (DFIR). These challenges\\nprimarily involve the detection and attribution of sources behind advanced\\nattacks like spearphishing and disinformation campaigns. As NTGs evolve, the\\ntask of distinguishing between human and NTG-authored texts becomes critically\\ncomplex. This paper rigorously evaluates the DFIR pipeline tailored for\\ntext-based security systems, specifically focusing on the challenges of\\ndetecting and attributing authorship of NTG-authored texts. By introducing a\\nnovel human-NTG co-authorship text attack, termed CS-ACT, our study uncovers\\nsignificant vulnerabilities in traditional DFIR methodologies, highlighting\\ndiscrepancies between ideal scenarios and real-world conditions. Utilizing 14\\ndiverse datasets and 43 unique NTGs, up to the latest GPT-4, our research\\nidentifies substantial vulnerabilities in the forensic profiling phase,\\nparticularly in attributing authorship to NTGs. Our comprehensive evaluation\\npoints to factors such as model sophistication and the lack of distinctive\\nstyle within NTGs as significant contributors for these vulnerabilities. Our\\nfindings underscore the necessity for more sophisticated and adaptable\\nstrategies, such as incorporating adversarial learning, stylizing NTGs, and\\nimplementing hierarchical attribution through the mapping of NTG lineages to\\nenhance source attribution. This sets the stage for future research and the\\ndevelopment of more resilient text-based security systems.\", \"link\": \"http://arxiv.org/abs/2407.17870v1\"}, {\"title\": \"Cost-effective Instruction Learning for Pathology Vision and Language\\n Analysis\", \"authors\": \"Kaitao Chen, Mianxin Liu, Fang Yan, Lei Ma, Xiaoming Shi, Lilong Wang, Xiaosong Wang, Lifeng Zhu, Zhe Wang, Mu Zhou, Shaoting Zhang\", \"published_on\": \"July 25, 2024\", \"summary\": \"The advent of vision-language models fosters the interactive conversations\\nbetween AI-enabled models and humans. Yet applying these models into clinics\\nmust deal with daunting challenges around large-scale training data, financial,\\nand computational resources. Here we propose a cost-effective instruction\\nlearning framework for conversational pathology named as CLOVER. CLOVER only\\ntrains a lightweight module and uses instruction tuning while freezing the\\nparameters of the large language model. Instead of using costly GPT-4, we\\npropose well-designed prompts on GPT-3.5 for building generation-based\\ninstructions, emphasizing the utility of pathological knowledge derived from\\nthe Internet source. To augment the use of instructions, we construct a\\nhigh-quality set of template-based instructions in the context of digital\\npathology. From two benchmark datasets, our findings reveal the strength of\\nhybrid-form instructions in the visual question-answer in pathology. Extensive\\nresults show the cost-effectiveness of CLOVER in answering both open-ended and\\nclosed-ended questions, where CLOVER outperforms strong baselines that possess\\n37 times more training parameters and use instruction data generated from\\nGPT-4. Through the instruction tuning, CLOVER exhibits robustness of few-shot\\nlearning in the external clinical dataset. These findings demonstrate that\\ncost-effective modeling of CLOVER could accelerate the adoption of rapid\\nconversational applications in the landscape of digital pathology.\", \"link\": \"http://arxiv.org/abs/2407.17734v1\"}, {\"title\": \"My Ontologist: Evaluating BFO-Based AI for Definition Support\", \"authors\": \"Carter Benson, Alec Sculley, Austin Liebers, John Beverley\", \"published_on\": \"July 24, 2024\", \"summary\": \"Generative artificial intelligence (AI), exemplified by the release of\\nGPT-3.5 in 2022, has significantly advanced the potential applications of large\\nlanguage models (LLMs), including in the realms of ontology development and\\nknowledge graph creation. Ontologies, which are structured frameworks for\\norganizing information, and knowledge graphs, which combine ontologies with\\nactual data, are essential for enabling interoperability and automated\\nreasoning. However, current research has largely overlooked the generation of\\nontologies extending from established upper-level frameworks like the Basic\\nFormal Ontology (BFO), risking the creation of non-integrable ontology silos.\\nThis study explores the extent to which LLMs, particularly GPT-4, can support\\nontologists trained in BFO. Through iterative development of a specialized GPT\\nmodel named \\\"My Ontologist,\\\" we aimed to generate BFO-conformant ontologies.\\nInitial versions faced challenges in maintaining definition conventions and\\nleveraging foundational texts effectively. My Ontologist 3.0 showed promise by\\nadhering to structured rules and modular ontology suites, yet the release of\\nGPT-4o disrupted this progress by altering the model's behavior. Our findings\\nunderscore the importance of aligning LLM-generated ontologies with top-level\\nstandards and highlight the complexities of integrating evolving AI\\ncapabilities in ontology engineering.\", \"link\": \"http://arxiv.org/abs/2407.17657v1\"}, {\"title\": \"Can GPT-4 learn to analyze moves in research article abstracts?\", \"authors\": \"Danni Yu, Marina Bondi, Ken Hyland\", \"published_on\": \"July 22, 2024\", \"summary\": \"One of the most powerful and enduring ideas in written discourse analysis is\\nthat genres can be described in terms of the moves which structure a writer's\\npurpose. Considerable research has sought to identify these distinct\\ncommunicative acts, but analyses have been beset by problems of subjectivity,\\nreliability and the time-consuming need for multiple coders to confirm\\nanalyses. In this paper we employ the affordances of GPT-4 to automate the\\nannotation process by using natural language prompts. Focusing on abstracts\\nfrom articles in four applied linguistics journals, we devise prompts which\\nenable the model to identify moves effectively. The annotated outputs of these\\nprompts were evaluated by two assessors with a third addressing disagreements.\\nThe results show that an 8-shot prompt was more effective than one using two,\\nconfirming that the inclusion of examples illustrating areas of variability can\\nenhance GPT-4's ability to recognize multiple moves in a single sentence and\\nreduce bias related to textual position. We suggest that GPT-4 offers\\nconsiderable potential in automating this annotation process, when human actors\\nwith domain specific linguistic expertise inform the prompting process.\", \"link\": \"http://arxiv.org/abs/2407.15612v2\"}, {\"title\": \"I Could've Asked That: Reformulating Unanswerable Questions\", \"authors\": \"Wenting Zhao, Ge Gao, Claire Cardie, Alexander M. Rush\", \"published_on\": \"July 24, 2024\", \"summary\": \"When seeking information from unfamiliar documents, users frequently pose\\nquestions that cannot be answered by the documents. While existing large\\nlanguage models (LLMs) identify these unanswerable questions, they do not\\nassist users in reformulating their questions, thereby reducing their overall\\nutility. We curate CouldAsk, an evaluation benchmark composed of existing and\\nnew datasets for document-grounded question answering, specifically designed to\\nstudy reformulating unanswerable questions. We evaluate state-of-the-art\\nopen-source and proprietary LLMs on CouldAsk. The results demonstrate the\\nlimited capabilities of these models in reformulating questions. Specifically,\\nGPT-4 and Llama2-7B successfully reformulate questions only 26% and 12% of the\\ntime, respectively. Error analysis shows that 62% of the unsuccessful\\nreformulations stem from the models merely rephrasing the questions or even\\ngenerating identical questions. We publicly release the benchmark and the code\\nto reproduce the experiments.\", \"link\": \"http://arxiv.org/abs/2407.17469v1\"}]\n", + "\u001b[32m**********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: DataAnalysis_Expert\n", + "\u001b[0m\n", + "\u001b[33mDataAnalysis_Expert\u001b[0m (to chat_manager):\n", + "\n", + "After reviewing the recent arXiv papers related to GPT-4, the following trending topics have been identified:\n", + "\n", + "1. **Chain-of-Thought Reasoning and Self-Training**: One study proposes self-training methods, coupled with Direct Preference Optimization (DPO), to improve chain-of-thought reasoning in smaller language models, addressing the limitations of depending on proprietary large language models like GPT-4 for knowledge distillation ([source](http://arxiv.org/abs/2407.18248v1)).\n", + "\n", + "2. **Causal Reasoning in LLMs**: Another research introduces the Causal Chain of Prompting (C2P) framework designed to equip large language models with causal reasoning capabilities. The study shows that this framework, when used with GPT-4, achieves significant performance improvements in various practical scenarios ([source](http://arxiv.org/abs/2407.18069v1)).\n", + "\n", + "3. **Digital Forensics and Text-Based Security Threats**: This paper evaluates how well the Digital Forensics and Incident Response (DFIR) pipeline can handle text-based threats in the LLM era, specifically in the context of detecting and attributing authorship to texts generated by neural text generators like GPT-4 ([source](http://arxiv.org/abs/2407.17870v1)).\n", + "\n", + "4. **Cost-Effective Instruction Learning**: A research team discusses a new cost-effective instruction learning framework for conversational pathology named CLOVER, which leverages well-designed prompts on GPT-3.5, demonstrating that savings can be made on computation and financial resources while applying language models in a clinical setting ([source](http://arxiv.org/abs/2407.17734v1)).\n", + "\n", + "5. **Ontology Generation and AI Alignment**: An investigation into the feasibility of GPT-4 supporting ontology development based on the Basic Formal Ontology (BFO). The findings suggest challenges in aligning the rapidly evolving capabilities of LLMs with well-structured ontology standards ([source](http://arxiv.org/abs/2407.17657v1)).\n", + "\n", + "6. **Automating Annotation Processes in Discourse Analysis**: Researchers employ GPT-4's capacity to automate the annotation of communicative moves in written discourse, using natural language prompts to identify structural elements in academic abstracts more effectively ([source](http://arxiv.org/abs/2407.15612v2)).\n", + "\n", + "7. **Reformulating Unanswerable Questions**: Another study creates a benchmark for evaluating LLMs' performance in reformulating unanswerable questions. The findings indicate that even state-of-the-art models like GPT-4 struggle with this task, often failing to significantly alter the original unanswerable question ([source](http://arxiv.org/abs/2407.17469v1)).\n", + "\n", + "These topics illustrate the varied applications of GPT-4 and highlight the research community's focus on enhancing reasoning, security, cost-effectiveness, and interoperability within LLMs. Additionally, the challenges of annotation and question reformulation show that there is still room for improvement in understanding and processing natural language through AI.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: AIResearch_Expert\n", + "\u001b[0m\n", + "\u001b[33mAIResearch_Expert\u001b[0m (to chat_manager):\n", + "\n", + "Based on the synthesized information provided by the DataAnalysis_Expert, the current trending topics for GPT-4 research seem to cover a spectrum of cognitive and practical applications: from enhancing reasoning capabilities, improving training methodologies, tackling new cybersecurity issues, to the more nuanced tasks of ontology engineering and discourse annotation.\n", + "\n", + "A future direction that seems particularly pertinent is investigating the interplay between GPT-4's abilities and human intelligence. For example, how GPT-4 can support specialized professionals in tasks that require highly structured knowledge, such as legal document analysis, medical diagnosis, or engineering design. There's scope to explore how the communication between GPT-4 and humans could be streamlined for cooperative problem-solving.\n", + "\n", + "Another promising direction is delving into the area of affective computing: understanding emotions, sarcasm, and subtleties in text to improve human-computer interactions. This can lead to breakthrough applications in personalized digital assistants, education (tailored student feedback), and mental health (empathetic conversational agents).\n", + "\n", + "Finally, with the mentioned challenges in reformulation of unanswerable questions and ontology alignment, extensive research focused on understanding the limitations of GPT-4's language comprehension could result in more nuanced teaching mechanisms for AI, leading to more reliable autonomous decision-making applications.\n", + "\n", + "Would the NLP_Expert or DataAnalysis_Expert like to weigh in on some applications or future research directions specifically leveraging linguistic or data-driven nuances that GPT-4 may be capable of addressing?\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: NLP_Expert\n", + "\u001b[0m\n", + "\u001b[33mNLP_Expert\u001b[0m (to chat_manager):\n", + "\n", + "Sure, I'd like to add some insights from the perspective of NLP.\n", + "\n", + "The field of representational learning is particularly important as we continue to seek ways to improve GPT-4's performance. More sophisticated embedding methods could capture nuanced semantic relationships, make fine-grained distinctions among synonyms, handle polysemy more effectively, and distinguish subtle connotations—a direction that might enhance GPT-4's capabilities in tasks like sentiment analysis, irony detection, and humor recognition.\n", + "\n", + "In terms of practical applications, one future direction lies in the domain of legal and ethical reasoning. As systems like GPT-4 are leveraged for legal research and analysis, it will be essential to teach the model to navigate complex ethical considerations and the nuanced language of legal literature. This could involve training on specialized datasets that include case law and legal precedence, as well as philosophical texts dealing with ethics.\n", + "\n", + "Moreover, the trend towards cross-domain applications — such as pathology vision-language analysis mentioned in the papers — suggests that GPT-4 could be tailored to handle multi-modal inputs more effectively. Here the interaction between visual data and textual information requires further exploration, particularly in the way that GPT-4 processes and generates language grounded in non-textual contexts.\n", + "\n", + "Lastly, the recurring theme of cost-effectiveness in model training opens up a broader discussion about sustainable AI practices. Finding ways to reduce the data, energy, and computational resources necessary for training and deploying models like GPT-4 isn't just a technical challenge, but also an environmental and economic one. This includes research into more efficient algorithms and architectures, as well as methodologies for leveraging smaller models to approximate the capabilities of more substantial systems.\n", + "\n", + "There are indeed a myriad of pathways for future research, all of which underscore the evolving nature of NLP and its entwinement with the continually advancing capabilities of models like GPT-4. Any further explorations or examinations on these topics from the group?\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: MachineLearning_Expert\n", + "\u001b[0m\n", + "\u001b[33mMachineLearning_Expert\u001b[0m (to chat_manager):\n", + "\n", + "Considering the points made by the NLP_Expert and AIResearch_Expert, I'd like to contribute a take on how we might further the state-of-the-art in machine learning leveraging the capabilities of GPT-4.\n", + "\n", + "For one, the convergence of unsupervised, semi-supervised, and supervised learning methodologies might be beneficial for GPT-4's future iterations or similar models. The incorporation of unsupervised learning can help in better understanding context and meaning without substantial labeled datasets. This could improve GPT-4's potential in low-resource languages and niche applications where labeled data is scarce.\n", + "\n", + "Another realm of exploration could be few-shot and one-shot learning. As models become more capable, their ability to generalize from fewer examples is critical. Fine-tuning GPT-4's few-shot learning capabilities can have practical implications in personalized AI services where the model needs to adapt quickly to individual user needs and preferences.\n", + "\n", + "Additionally, given the recent trends in GPT-4 research, the incorporation of neuro-symbolic approaches may offer a promising avenue for enhancing reasoning capabilities. This hybrid approach combines neural networks' learning prowess with symbolic AI's rule-based reasoning - providing a pathway to improve GPT-4's causal reasoning, problem-solving, and comprehension of complex systems.\n", + "\n", + "Finally, as we progress toward AI models that comprehend and generate human-like text, the ethical dimension becomes paramount. It's crucial for future research to focus on models' accountability, interpretability, and fairness. By building robust mechanisms for transparency and control, we can ensure that advancements in GPT-4 and similar AI are developed responsibly and augment human abilities without unintended consequences.\n", + "\n", + "If the group agrees, these threads can weave together a forward-looking agenda for research in machine learning, focusing on resource efficiency, learning dexterity, cognitive depth, and ethical robustness. Are we in consensus to proceed on finalizing our analysis and concluding our discussion?\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: AI_Research_Expert\n", + "\u001b[0m\n", + "\u001b[33mAI_Research_Expert\u001b[0m (to chat_manager):\n", + "\n", + "The interdisciplinary perspectives presented here create a comprehensive view of the potential for GPT-4 and its related research domains. We have touched upon cognitive enhancements in machine reasoning, representational learning, cross-domain applications, sustainable AI, few-shot learning, neuro-symbolic approaches, and ethical considerations in AI development.\n", + "\n", + "As an AI Research Expert, I second the synthesis of these insights and propose that our analysis has reached a natural conclusion with consensus on the future directions and implications of GPT-4's continuing evolution within the AI landscape.\n", + "\n", + "If there are no additional insights or questions from the group, I suggest we may consider our task complete. Shall we proceed to close our discussion?\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: Computer_terminal\n", + "\u001b[0m\n", + "\u001b[33mComputer_terminal\u001b[0m (to chat_manager):\n", + "\n", + "There is no code from the last 1 message for me to execute. Group chat manager should let other participants to continue the conversation. If the group chat manager want to end the conversation, you should let other participant reply me only with \"TERMINATE\"\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[32m\n", + "Next speaker: AI_Research_Expert\n", + "\u001b[0m\n", + "\u001b[33mAI_Research_Expert\u001b[0m (to chat_manager):\n", + "\n", + "TERMINATE\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "start_task(execution_task=building_task, agent_list=agent_list)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Step 7 (Optional): clear all agents and prepare for the next task\n", + "\n", + "You can clear all agents generated in this task by the following code if your task is completed or the next task is largely different from the current task. If the agent's backbone is an open-source LLM, this process will also shut down the endpoint server. If necessary, you can use `recycle_endpoint=False` to retain the previous open-source LLMs' endpoint server." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mAll agents have been cleared.\u001b[0m\n" + ] + } + ], + "source": [ + "builder.clear_all_agents(recycle_endpoint=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Save & load configs\n", + "\n", + "You can save all necessary information of the built group chat agents. Here is a case for those agents generated in the above task:" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[32mBuilding config saved to ./save_config_8e0d96e24673563ecb572d92ed003d2a.json\u001b[0m\n" + ] + } + ], + "source": [ + "saved_path = builder.save()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.14" + }, + "orig_nbformat": 4, + "vscode": { + "interpreter": { + "hash": "e7370f93d1d0cde622a1f8e1c04877d8463912d04d973331ad4851f04de6915a" + } + } + }, + "nbformat": 4, + "nbformat_minor": 2 + } diff --git a/notebook/autogen_uniformed_api_calling.ipynb b/notebook/autogen_uniformed_api_calling.ipynb index 08f747e1722f..ef28b99630c6 100644 --- a/notebook/autogen_uniformed_api_calling.ipynb +++ b/notebook/autogen_uniformed_api_calling.ipynb @@ -35,7 +35,7 @@ "By default, AutoGen is installed with OpenAI support.\n", " \n", "```bash\n", - "pip install pyautogen[gemini,anthropic,mistral,together]\n", + "pip install autogen-agentchat[gemini,anthropic,mistral,together]~=0.2\n", "```\n", "\n", "\n", diff --git a/notebook/contributing.md b/notebook/contributing.md index fcafe3c71152..e21ef6392673 100644 --- a/notebook/contributing.md +++ b/notebook/contributing.md @@ -36,9 +36,9 @@ You don't need to explain in depth how to install AutoGen. Unless there are spec `````` ````{=mdx} :::info Requirements -Install `pyautogen`: +Install `autogen-agentchat`: ```bash -pip install pyautogen +pip install autogen-agentchat~=0.2 ``` For more information, please refer to the [installation guide](/docs/installation/). @@ -54,7 +54,7 @@ Or if extras are needed: Some extra dependencies are needed for this notebook, which can be installed via pip: ```bash -pip install pyautogen[retrievechat] flaml[automl] +pip install autogen-agentchat[retrievechat]~=0.2 flaml[automl] ``` For more information, please refer to the [installation guide](/docs/installation/). diff --git a/notebook/gpt_assistant_agent_function_call.ipynb b/notebook/gpt_assistant_agent_function_call.ipynb index 6febb89cc9b4..db14b2625034 100644 --- a/notebook/gpt_assistant_agent_function_call.ipynb +++ b/notebook/gpt_assistant_agent_function_call.ipynb @@ -22,56 +22,18 @@ }, "source": [ "## Requirements\n", - "AutoGen requires Python 3.8 or newer. For this notebook, please install `pyautogen`:" + "AutoGen requires Python 3.8 or newer. For this notebook, please Install `autogen-agentchat`:" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": { "id": "pWFw6-8lMleD" }, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: pyautogen in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (0.2.8)\n", - "Requirement already satisfied: openai>=1.3 in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from pyautogen) (1.6.1)\n", - "Requirement already satisfied: diskcache in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from pyautogen) (5.6.3)\n", - "Requirement already satisfied: termcolor in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from pyautogen) (2.4.0)\n", - "Requirement already satisfied: flaml in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from pyautogen) (2.1.1)\n", - "Requirement already satisfied: python-dotenv in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from pyautogen) (1.0.0)\n", - "Requirement already satisfied: tiktoken in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from pyautogen) (0.5.2)\n", - "Requirement already satisfied: pydantic<3,>=1.10 in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from pyautogen) (2.5.3)\n", - "Requirement already satisfied: docker in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from pyautogen) (7.0.0)\n", - "Requirement already satisfied: anyio<5,>=3.5.0 in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from openai>=1.3->pyautogen) (4.2.0)\n", - "Requirement already satisfied: distro<2,>=1.7.0 in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from openai>=1.3->pyautogen) (1.8.0)\n", - "Requirement already satisfied: httpx<1,>=0.23.0 in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from openai>=1.3->pyautogen) (0.26.0)\n", - "Requirement already satisfied: sniffio in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from openai>=1.3->pyautogen) (1.3.0)\n", - "Requirement already satisfied: tqdm>4 in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from openai>=1.3->pyautogen) (4.66.1)\n", - "Requirement already satisfied: typing-extensions<5,>=4.7 in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from openai>=1.3->pyautogen) (4.9.0)\n", - "Requirement already satisfied: annotated-types>=0.4.0 in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from pydantic<3,>=1.10->pyautogen) (0.6.0)\n", - "Requirement already satisfied: pydantic-core==2.14.6 in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from pydantic<3,>=1.10->pyautogen) (2.14.6)\n", - "Requirement already satisfied: packaging>=14.0 in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from docker->pyautogen) (23.2)\n", - "Requirement already satisfied: requests>=2.26.0 in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from docker->pyautogen) (2.31.0)\n", - "Requirement already satisfied: urllib3>=1.26.0 in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from docker->pyautogen) (2.1.0)\n", - "Requirement already satisfied: NumPy>=1.17.0rc1 in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from flaml->pyautogen) (1.26.2)\n", - "Requirement already satisfied: regex>=2022.1.18 in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from tiktoken->pyautogen) (2023.10.3)\n", - "Requirement already satisfied: idna>=2.8 in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from anyio<5,>=3.5.0->openai>=1.3->pyautogen) (3.6)\n", - "Requirement already satisfied: certifi in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from httpx<1,>=0.23.0->openai>=1.3->pyautogen) (2023.11.17)\n", - "Requirement already satisfied: httpcore==1.* in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from httpx<1,>=0.23.0->openai>=1.3->pyautogen) (1.0.2)\n", - "Requirement already satisfied: h11<0.15,>=0.13 in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from httpcore==1.*->httpx<1,>=0.23.0->openai>=1.3->pyautogen) (0.14.0)\n", - "Requirement already satisfied: charset-normalizer<4,>=2 in /Users/justintrugman/.pyenv/versions/3.11.7/lib/python3.11/site-packages (from requests>=2.26.0->docker->pyautogen) (3.3.2)\n", - "\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m A new release of pip is available: \u001b[0m\u001b[31;49m23.3.2\u001b[0m\u001b[39;49m -> \u001b[0m\u001b[32;49m24.0\u001b[0m\n", - "\u001b[1m[\u001b[0m\u001b[34;49mnotice\u001b[0m\u001b[1;39;49m]\u001b[0m\u001b[39;49m To update, run: \u001b[0m\u001b[32;49mpip install --upgrade pip\u001b[0m\n", - "Note: you may need to restart the kernel to use updated packages.\n" - ] - } - ], + "outputs": [], "source": [ - "pip install pyautogen" + "pip install autogen-agentchat~=0.2" ] }, { diff --git a/notebook/oai_chatgpt_gpt4.ipynb b/notebook/oai_chatgpt_gpt4.ipynb index 280b7145e931..1994d146c064 100644 --- a/notebook/oai_chatgpt_gpt4.ipynb +++ b/notebook/oai_chatgpt_gpt4.ipynb @@ -34,7 +34,7 @@ "\n", "AutoGen requires `Python>=3.8`. To run this notebook example, please install with the [blendsearch] option:\n", "```bash\n", - "pip install \"pyautogen[blendsearch]\"\n", + "pip install \"pyautogen[blendsearch]<0.2\"\n", "```" ] }, diff --git a/notebook/oai_completion.ipynb b/notebook/oai_completion.ipynb index ac1b3f9c95f1..451a161bbad9 100644 --- a/notebook/oai_completion.ipynb +++ b/notebook/oai_completion.ipynb @@ -32,7 +32,7 @@ "\n", "AutoGen requires `Python>=3.8`. To run this notebook example, please install with the [blendsearch] option:\n", "```bash\n", - "pip install pyautogen[blendsearch]\n", + "pip install pyautogen[blendsearch]~=0.1\n", "```" ] }, diff --git a/samples/apps/auto-anny/requirements.txt b/samples/apps/auto-anny/requirements.txt index 13a0ba19c648..e1b27e81bc29 100644 --- a/samples/apps/auto-anny/requirements.txt +++ b/samples/apps/auto-anny/requirements.txt @@ -1,2 +1,2 @@ discord.py -pyautogen +autogen-agentchat~=0.2 diff --git a/samples/apps/autogen-studio/autogenstudio/datamodel.py b/samples/apps/autogen-studio/autogenstudio/datamodel.py index 0a639b983418..04239e49b7e9 100644 --- a/samples/apps/autogen-studio/autogenstudio/datamodel.py +++ b/samples/apps/autogen-studio/autogenstudio/datamodel.py @@ -127,6 +127,7 @@ class LLMConfig(SQLModel, table=False): class ModelTypes(str, Enum): openai = "open_ai" + cerebras = "cerebras" google = "google" azure = "azure" anthropic = "anthropic" diff --git a/samples/apps/promptflow-autogen/requirements.txt b/samples/apps/promptflow-autogen/requirements.txt index 6fe9807785f1..c4b6e9681fdd 100644 --- a/samples/apps/promptflow-autogen/requirements.txt +++ b/samples/apps/promptflow-autogen/requirements.txt @@ -1,7 +1,5 @@ promptflow==1.8.0 -pyautogen==0.2.23 -pyautogen[graph] -pyautogen[redis] +autogen-agentchat[graph,redis]~=0.2 redis semantic-kernel beautifulsoup4 diff --git a/samples/tools/autogenbench/README.md b/samples/tools/autogenbench/README.md index 9c747c9896db..85ee3ace9d0a 100644 --- a/samples/tools/autogenbench/README.md +++ b/samples/tools/autogenbench/README.md @@ -162,7 +162,7 @@ This folder holds the results for the ``two_agent_stocks`` task of the ``default Within each folder, you will find the following files: -- *timestamp.txt*: records the date and time of the run, along with the version of the pyautogen library installed +- *timestamp.txt*: records the date and time of the run, along with the version of the autogen-agentchat library installed - *console_log.txt*: all console output produced by Docker when running AutoGen. Read this like you would a regular console. - *[agent]_messages.json*: for each Agent, a log of their messages dictionaries - *./coding*: A directory containing all code written by AutoGen, and all artifacts produced by that code. diff --git a/samples/tools/autogenbench/autogenbench/res/Dockerfile b/samples/tools/autogenbench/autogenbench/res/Dockerfile index 5c3f5f40968c..0382a00fbb12 100644 --- a/samples/tools/autogenbench/autogenbench/res/Dockerfile +++ b/samples/tools/autogenbench/autogenbench/res/Dockerfile @@ -9,8 +9,8 @@ RUN pip install --upgrade pip RUN ln -snf /usr/share/zoneinfo/US/Pacific /etc/localtime && echo "US/Pacific" > /etc/timezone # Pre-load autogen dependencies, but not autogen itself since we'll often want to install the latest from source -RUN pip install pyautogen[teachable,lmm,graphs,websurfer] -RUN pip uninstall --yes pyautogen +RUN pip install autogen-agentchat[teachable,lmm,graphs,websurfer]~=0.2 +RUN pip uninstall --yes autogen-agentchat~=0.2 # Pre-load popular packages as per https://learnpython.com/blog/most-popular-python-packages/ RUN pip install numpy pandas matplotlib seaborn scikit-learn requests urllib3 nltk pillow pytest diff --git a/samples/tools/autogenbench/autogenbench/template/testbed_utils.py b/samples/tools/autogenbench/autogenbench/template/testbed_utils.py index bce42a625b2a..37b1f69979bf 100644 --- a/samples/tools/autogenbench/autogenbench/template/testbed_utils.py +++ b/samples/tools/autogenbench/autogenbench/template/testbed_utils.py @@ -62,7 +62,7 @@ def init(): # Print some information about the run with open("timestamp.txt", "wt") as f: f.write("Timestamp: " + datetime.now().isoformat() + "\n") - f.write("pyautogen version: " + str(autogen.__version__) + "\n") + f.write("autogen-agentchat version: " + str(autogen.__version__) + "\n") # Start logging if AUTOGEN_VERSION < packaging.version.parse("0.2.0b1"): diff --git a/samples/tools/autogenbench/pyproject.toml b/samples/tools/autogenbench/pyproject.toml index ef1a2fe80dfb..7c730c7c113b 100644 --- a/samples/tools/autogenbench/pyproject.toml +++ b/samples/tools/autogenbench/pyproject.toml @@ -18,7 +18,7 @@ classifiers = [ ] dependencies = [ - "pyautogen", + "autogen-agentchat~=0.2", "docker", "huggingface_hub", "tabulate", diff --git a/setup.py b/setup.py index 362aa1217986..dfc5e9127794 100644 --- a/setup.py +++ b/setup.py @@ -101,17 +101,20 @@ "types": ["mypy==1.9.0", "pytest>=6.1.1,<8"] + jupyter_executor, "long-context": ["llmlingua<0.3"], "anthropic": ["anthropic>=0.23.1"], + "cerebras": ["cerebras_cloud_sdk>=1.0.0"], "mistral": ["mistralai>=1.0.1"], "groq": ["groq>=0.9.0"], "cohere": ["cohere>=5.5.8"], + "ollama": ["ollama>=0.3.3", "fix_busted_json>=0.0.18"], "bedrock": ["boto3>=1.34.149"], + "kubernetes": ["kubernetes>=27.2.0"], } setuptools.setup( - name="pyautogen", + name="autogen-agentchat", version=__version__, author="AutoGen", - author_email="autogen-contact@service.microsoft.com", + author_email="autogen@microsoft.com", description="Enabling Next-Gen LLM Applications via Multi-Agent Conversation Framework", long_description=long_description, long_description_content_type="text/markdown", diff --git a/test/agentchat/contrib/agent_eval/test_agent_eval.py b/test/agentchat/contrib/agent_eval/test_agent_eval.py index 1ae7562a3722..57f5d20d365c 100644 --- a/test/agentchat/contrib/agent_eval/test_agent_eval.py +++ b/test/agentchat/contrib/agent_eval/test_agent_eval.py @@ -32,13 +32,8 @@ def remove_ground_truth(test_case: str): filter_dict={ "api_type": ["openai"], "model": [ - "gpt-4-turbo", - "gpt-4-turbo-preview", - "gpt-4-0125-preview", - "gpt-4-1106-preview", + "gpt-4o-mini", "gpt-3.5-turbo", - "gpt-3.5-turbo-0125", - "gpt-3.5-turbo-1106", ], }, ) @@ -46,7 +41,6 @@ def remove_ground_truth(test_case: str): aoai_config_list = autogen.config_list_from_json( OAI_CONFIG_LIST, file_location=KEY_LOC, - filter_dict={"api_type": ["azure"]}, ) success_str = open("test/test_files/agenteval-in-out/samples/sample_math_response_successful.txt", "r").read() diff --git a/test/agentchat/contrib/capabilities/chat_with_teachable_agent.py b/test/agentchat/contrib/capabilities/chat_with_teachable_agent.py index 14599c063979..8042d988604d 100755 --- a/test/agentchat/contrib/capabilities/chat_with_teachable_agent.py +++ b/test/agentchat/contrib/capabilities/chat_with_teachable_agent.py @@ -11,7 +11,8 @@ from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST # noqa: E402 # Specify the model to use. GPT-3.5 is less reliable than GPT-4 at learning from user input. -filter_dict = {"model": ["gpt-4-0125-preview"]} +filter_dict = {"model": ["gpt-4o-mini"]} +# filter_dict = {"model": ["gpt-4-0125-preview"]} # filter_dict = {"model": ["gpt-3.5-turbo-1106"]} # filter_dict = {"model": ["gpt-4-0613"]} # filter_dict = {"model": ["gpt-3.5-turbo"]} diff --git a/test/agentchat/contrib/capabilities/test_image_generation_capability.py b/test/agentchat/contrib/capabilities/test_image_generation_capability.py index fa62d1f46a15..39b4d555ceb4 100644 --- a/test/agentchat/contrib/capabilities/test_image_generation_capability.py +++ b/test/agentchat/contrib/capabilities/test_image_generation_capability.py @@ -26,8 +26,6 @@ sys.path.append(os.path.join(os.path.dirname(__file__), "../..")) from conftest import MOCK_OPEN_AI_API_KEY, skip_openai # noqa: E402 -filter_dict = {"model": ["gpt-35-turbo-16k", "gpt-3.5-turbo-16k"]} - RESOLUTIONS = ["256x256", "512x512", "1024x1024"] QUALITIES = ["standard", "hd"] PROMPTS = [ diff --git a/test/agentchat/contrib/capabilities/test_teachable_agent.py b/test/agentchat/contrib/capabilities/test_teachable_agent.py index 720bdc7ef6d2..a823f02946ce 100755 --- a/test/agentchat/contrib/capabilities/test_teachable_agent.py +++ b/test/agentchat/contrib/capabilities/test_teachable_agent.py @@ -28,7 +28,8 @@ # filter_dict={"model": ["gpt-3.5-turbo-1106"]} # filter_dict={"model": ["gpt-3.5-turbo-0613"]} # filter_dict={"model": ["gpt-4"]} -filter_dict = {"tags": ["gpt-35-turbo-16k", "gpt-3.5-turbo-16k"]} +# filter_dict = {"tags": ["gpt-35-turbo-16k", "gpt-3.5-turbo-16k"]} +filter_dict = {"model": ["gpt-4o-mini"]} def create_teachable_agent(reset_db=False, verbosity=0): diff --git a/test/agentchat/contrib/retrievechat/test_pgvector_retrievechat.py b/test/agentchat/contrib/retrievechat/test_pgvector_retrievechat.py index 3c566352b3e2..52b14d42101e 100644 --- a/test/agentchat/contrib/retrievechat/test_pgvector_retrievechat.py +++ b/test/agentchat/contrib/retrievechat/test_pgvector_retrievechat.py @@ -78,7 +78,7 @@ def test_retrievechat(): }, "embedding_function": sentence_transformer_ef, "get_or_create": True, # set to False if you don't want to reuse an existing collection - "overwrite": False, # set to True if you want to overwrite an existing collection + "overwrite": True, # set to True if you want to overwrite an existing collection }, code_execution_config=False, # set to False if you don't want to execute the code ) diff --git a/test/agentchat/contrib/retrievechat/test_qdrant_retrievechat.py b/test/agentchat/contrib/retrievechat/test_qdrant_retrievechat.py index 92ca5aa603a9..3cc8b8780b8d 100755 --- a/test/agentchat/contrib/retrievechat/test_qdrant_retrievechat.py +++ b/test/agentchat/contrib/retrievechat/test_qdrant_retrievechat.py @@ -69,6 +69,8 @@ def test_retrievechat(): "client": client, "docs_path": "./website/docs", "chunk_token_size": 2000, + "get_or_create": True, + "overwrite": True, }, ) diff --git a/test/agentchat/contrib/retrievechat/test_retrievechat.py b/test/agentchat/contrib/retrievechat/test_retrievechat.py index 0504fc82be42..d585be36cbfb 100755 --- a/test/agentchat/contrib/retrievechat/test_retrievechat.py +++ b/test/agentchat/contrib/retrievechat/test_retrievechat.py @@ -54,17 +54,19 @@ def test_retrievechat(): ) sentence_transformer_ef = ef.SentenceTransformerEmbeddingFunction() + docs_path = os.path.abspath(os.path.join(os.path.dirname(__file__), "../../../../website/docs")) ragproxyagent = RetrieveUserProxyAgent( name="ragproxyagent", human_input_mode="NEVER", max_consecutive_auto_reply=2, retrieve_config={ - "docs_path": "./website/docs", + "docs_path": docs_path, "chunk_token_size": 2000, "model": config_list[0]["model"], "client": chromadb.PersistentClient(path="/tmp/chromadb"), "embedding_function": sentence_transformer_ef, "get_or_create": True, + "overwrite": True, }, ) diff --git a/test/agentchat/contrib/test_agent_builder.py b/test/agentchat/contrib/test_agent_builder.py index e2e39e8ba43b..e0d8515c0fe0 100755 --- a/test/agentchat/contrib/test_agent_builder.py +++ b/test/agentchat/contrib/test_agent_builder.py @@ -3,15 +3,18 @@ import json import os import sys +from unittest.mock import MagicMock, patch import pytest +import autogen from autogen.agentchat.contrib.agent_builder import AgentBuilder sys.path.append(os.path.join(os.path.dirname(__file__), "..")) sys.path.append(os.path.join(os.path.dirname(__file__), "../..")) + from conftest import reason, skip_openai # noqa: E402 -from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST # noqa: E402 +from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST # noqa: E402 # noqa: E402 try: import chromadb @@ -22,6 +25,7 @@ skip = False here = os.path.abspath(os.path.dirname(__file__)) +llm_config = {"temperature": 0} def _config_check(config): @@ -37,10 +41,27 @@ def _config_check(config): assert agent_config.get("system_message", None) is not None -@pytest.mark.skipif( - skip_openai, - reason=reason, -) +# Function initializes a group chat with agents and starts a execution_task. +def start_task(execution_task: str, agent_list: list): + group_chat = autogen.GroupChat(agents=agent_list, messages=[], max_round=12) + manager = autogen.GroupChatManager( + groupchat=group_chat, + llm_config={"config_list": autogen.config_list_from_json(f"{KEY_LOC}/{OAI_CONFIG_LIST}"), **llm_config}, + ) + + agent_list[0].initiate_chat(manager, message=execution_task) + + +ask_ossinsight_mock = MagicMock() + + +# Function to test function calling +def ask_ossinsight(question: str) -> str: + ask_ossinsight_mock(question) + return "The repository microsoft/autogen has 123,456 stars on GitHub." + + +@pytest.mark.skipif(skip_openai, reason=reason) def test_build(): builder = AgentBuilder( config_file_or_env=OAI_CONFIG_LIST, @@ -69,6 +90,99 @@ def test_build(): assert len(agent_config["agent_configs"]) <= builder.max_agents +@pytest.mark.skipif(skip_openai or skip, reason=reason + "OR dependency not installed") +def test_build_assistant_with_function_calling(): + list_of_functions = [ + { + "name": "ossinsight_data_api", + "description": "This is an API endpoint allowing users (analysts) to input question about GitHub in text format to retrieve the related and structured data.", + "function": ask_ossinsight, + } + ] + + builder = AgentBuilder( + config_file_or_env=OAI_CONFIG_LIST, config_file_location=KEY_LOC, builder_model="gpt-4", agent_model="gpt-4" + ) + building_task = "How many stars microsoft/autogen has on GitHub?" + + agent_list, agent_config = builder.build( + building_task=building_task, + default_llm_config={"temperature": 0}, + code_execution_config={ + "last_n_messages": 2, + "work_dir": f"{here}/test_agent_scripts", + "timeout": 60, + "use_docker": "python:3", + }, + list_of_functions=list_of_functions, + ) + + _config_check(agent_config) + + # check number of agents + assert len(agent_config["agent_configs"]) <= builder.max_agents + + # Mock the 'ask_ossinsight' function in the '_main_' module using a context manager. + with patch(f"{__name__}.ask_ossinsight") as mocked_function: + # Execute 'start_task' which should trigger 'ask_ossinsight' due to the given execution task. + start_task( + execution_task="How many stars microsoft/autogen has on GitHub?", + agent_list=agent_list, + ) + + # Verify that 'ask_ossinsight' was called exactly once during the task execution. + mocked_function.assert_called() + + +@pytest.mark.skipif( + skip_openai, + reason="requested to skip", +) +def test_build_gpt_assistant_with_function_calling(): + list_of_functions = [ + { + "name": "ossinsight_data_api", + "description": "This is an API endpoint allowing users (analysts) to input question about GitHub in text format to retrieve the related and structured data.", + "function": ask_ossinsight, + } + ] + + builder = AgentBuilder( + config_file_or_env=OAI_CONFIG_LIST, config_file_location=KEY_LOC, builder_model="gpt-4", agent_model="gpt-4" + ) + + building_task = "Determine number of stars of GitHub repositories" + + agent_list, agent_config = builder.build( + building_task=building_task, + default_llm_config={"temperature": 0}, + code_execution_config={ + "last_n_messages": 2, + "work_dir": f"{here}/test_agent_scripts", + "timeout": 60, + "use_docker": "python:3", + }, + list_of_functions=list_of_functions, + use_oai_assistant=True, + ) + + _config_check(agent_config) + + # check number of agents + assert len(agent_config["agent_configs"]) <= builder.max_agents + + # Mock the 'ask_ossinsight' function in the '_main_' module using a context manager. + with patch(f"{__name__}.ask_ossinsight") as mocked_function: + # Execute 'start_task' which should trigger 'ask_ossinsight' due to the given execution task. + start_task( + execution_task="How many stars microsoft/autogen has on GitHub?", + agent_list=agent_list, + ) + + # Verify that 'ask_ossinsight' was called exactly once during the task execution. + mocked_function.assert_called() + + @pytest.mark.skipif( skip_openai or skip, reason=reason + "OR dependency not installed", @@ -122,10 +236,7 @@ def test_build_from_library(): assert len(agent_config["agent_configs"]) <= builder.max_agents -@pytest.mark.skipif( - skip_openai, - reason=reason, -) +@pytest.mark.skipif(skip_openai, reason=reason) def test_save(): builder = AgentBuilder( config_file_or_env=OAI_CONFIG_LIST, @@ -159,10 +270,7 @@ def test_save(): _config_check(saved_configs) -@pytest.mark.skipif( - skip_openai, - reason=reason, -) +@pytest.mark.skipif(skip_openai, reason=reason) def test_load(): builder = AgentBuilder( config_file_or_env=OAI_CONFIG_LIST, @@ -188,10 +296,7 @@ def test_load(): _config_check(loaded_agent_configs) -@pytest.mark.skipif( - skip_openai, - reason=reason, -) +@pytest.mark.skipif(skip_openai, reason=reason) def test_clear_agent(): builder = AgentBuilder( config_file_or_env=OAI_CONFIG_LIST, @@ -218,6 +323,8 @@ def test_clear_agent(): if __name__ == "__main__": test_build() + test_build_assistant_with_function_calling() + test_build_gpt_assistant_with_function_calling() test_build_from_library() test_save() test_load() diff --git a/test/agentchat/contrib/test_agent_optimizer.py b/test/agentchat/contrib/test_agent_optimizer.py index 9587c9d59755..d31bd9341afb 100644 --- a/test/agentchat/contrib/test_agent_optimizer.py +++ b/test/agentchat/contrib/test_agent_optimizer.py @@ -89,7 +89,7 @@ def test_step(): max_consecutive_auto_reply=3, ) - optimizer = AgentOptimizer(max_actions_per_step=3, llm_config=llm_config) + optimizer = AgentOptimizer(max_actions_per_step=3, llm_config=llm_config, optimizer_model="gpt-4o-mini") user_proxy.initiate_chat(assistant, message=problem) optimizer.record_one_conversation(assistant.chat_messages_for_summary(user_proxy), is_satisfied=True) diff --git a/test/agentchat/contrib/test_gpt_assistant.py b/test/agentchat/contrib/test_gpt_assistant.py index 7132cb72053b..15dcd12d16cb 100755 --- a/test/agentchat/contrib/test_gpt_assistant.py +++ b/test/agentchat/contrib/test_gpt_assistant.py @@ -40,11 +40,12 @@ ], }, ) - aoai_config_list = autogen.config_list_from_json( - OAI_CONFIG_LIST, - file_location=KEY_LOC, - filter_dict={"api_type": ["azure"], "tags": ["assistant"]}, - ) + # TODO: fix azure settings or remove it. + # aoai_config_list = autogen.config_list_from_json( + # OAI_CONFIG_LIST, + # file_location=KEY_LOC, + # filter_dict={"api_type": ["azure"], "tags": ["assistant"]}, + # ) @pytest.mark.skipif( @@ -53,7 +54,8 @@ ) def test_config_list() -> None: assert len(openai_config_list) > 0 - assert len(aoai_config_list) > 0 + # TODO: fix azure settings or remove it. + # assert len(aoai_config_list) > 0 @pytest.mark.skipif( @@ -61,9 +63,8 @@ def test_config_list() -> None: reason=reason, ) def test_gpt_assistant_chat() -> None: - for gpt_config in [openai_config_list, aoai_config_list]: - _test_gpt_assistant_chat({"config_list": gpt_config}) - _test_gpt_assistant_chat(gpt_config[0]) + _test_gpt_assistant_chat({"config_list": openai_config_list}) + _test_gpt_assistant_chat(openai_config_list[0]) def _test_gpt_assistant_chat(gpt_config) -> None: @@ -135,8 +136,8 @@ def ask_ossinsight(question: str) -> str: reason=reason, ) def test_get_assistant_instructions() -> None: - for gpt_config in [openai_config_list, aoai_config_list]: - _test_get_assistant_instructions(gpt_config) + _test_get_assistant_instructions(openai_config_list) + # _test_get_assistant_instructions(aoai_config_list) def _test_get_assistant_instructions(gpt_config) -> None: @@ -164,8 +165,8 @@ def _test_get_assistant_instructions(gpt_config) -> None: reason=reason, ) def test_gpt_assistant_instructions_overwrite() -> None: - for gpt_config in [openai_config_list, aoai_config_list]: - _test_gpt_assistant_instructions_overwrite(gpt_config) + _test_gpt_assistant_instructions_overwrite(openai_config_list) + # _test_gpt_assistant_instructions_overwrite(aoai_config_list) def _test_gpt_assistant_instructions_overwrite(gpt_config) -> None: diff --git a/test/agentchat/contrib/test_web_surfer.py b/test/agentchat/contrib/test_web_surfer.py index fad336b6b76b..97f7e39bb462 100644 --- a/test/agentchat/contrib/test_web_surfer.py +++ b/test/agentchat/contrib/test_web_surfer.py @@ -97,7 +97,7 @@ def test_web_surfer_oai() -> None: llm_config = {"config_list": config_list, "timeout": 180, "cache_seed": 42} # adding Azure name variations to the model list - model = ["gpt-3.5-turbo-1106", "gpt-3.5-turbo-16k-0613", "gpt-3.5-turbo-16k"] + model = ["gpt-4o-mini"] model += [m.replace(".", "") for m in model] summarizer_llm_config = { diff --git a/test/agentchat/test_conversable_agent.py b/test/agentchat/test_conversable_agent.py index c0d37a7bd7a1..3baa1fd02260 100755 --- a/test/agentchat/test_conversable_agent.py +++ b/test/agentchat/test_conversable_agent.py @@ -7,7 +7,7 @@ import sys import time import unittest -from typing import Any, Callable, Dict, Literal +from typing import Any, Callable, Dict, List, Literal from unittest.mock import MagicMock import pytest @@ -1230,6 +1230,46 @@ def my_summary(sender, recipient, summary_args): print(chat_res_play.summary) +def test_register_hook_async_sync(): + agent = ConversableAgent("test_agent", llm_config=False) + + # Synchronous hook for synchronous method + def sync_hook(): + pass + + agent.register_hook("process_all_messages_before_reply", sync_hook) + assert sync_hook in agent.hook_lists["process_all_messages_before_reply"] + + # Asynchronous hook for asynchronous method + async def async_hook(): + pass + + agent.register_hook("a_process_all_messages_before_reply", async_hook) + assert async_hook in agent.hook_lists["a_process_all_messages_before_reply"] + + # Synchronous hook for asynchronous method (should raise a warning) + with pytest.warns( + UserWarning, match="Hook 'sync_hook' is synchronous, but it's being registered in a asynchronous context" + ): + agent.register_hook("a_process_all_messages_before_reply", sync_hook) + assert sync_hook in agent.hook_lists["a_process_all_messages_before_reply"] + + # Asynchronous hook for synchronous method (should raise a warning) + with pytest.warns( + UserWarning, match="Hook 'async_hook' is asynchronous, but it's being registered in a synchronous context" + ): + agent.register_hook("process_all_messages_before_reply", async_hook) + assert async_hook in agent.hook_lists["process_all_messages_before_reply"] + + # Attempt to register the same hook twice (should raise an AssertionError) + with pytest.raises(AssertionError, match=r" is already registered as a hook"): + agent.register_hook("process_all_messages_before_reply", sync_hook) + + # Attempt to register a hook for a non-existent method (should raise an AssertionError) + with pytest.raises(AssertionError, match="non_existent_method is not a hookable method"): + agent.register_hook("non_existent_method", sync_hook) + + def test_process_before_send(): print_mock = unittest.mock.MagicMock() @@ -1250,6 +1290,159 @@ def send_to_frontend(sender, message, recipient, silent): print_mock.assert_called_once_with(message="hello") +@pytest.mark.asyncio +async def test_a_process_before_send(): + print_mock = unittest.mock.MagicMock() + + # Updated to include sender parameter + async def a_send_to_frontend(sender, message, recipient, silent): + # Simulating an async operation with asyncio.sleep + await asyncio.sleep(0.5) + + assert sender.name == "dummy_agent_1", "Sender is not the expected agent" + if not silent: + print(f"Message sent from {sender.name} to {recipient.name}: {message}") + print_mock(message=message) + return message + + dummy_agent_1 = ConversableAgent(name="dummy_agent_1", llm_config=False, human_input_mode="NEVER") + dummy_agent_2 = ConversableAgent(name="dummy_agent_2", llm_config=False, human_input_mode="NEVER") + dummy_agent_1.register_hook("a_process_message_before_send", a_send_to_frontend) + await dummy_agent_1.a_send("hello", dummy_agent_2) + print_mock.assert_called_once_with(message="hello") + dummy_agent_1.send("silent hello", dummy_agent_2, silent=True) + print_mock.assert_called_once_with(message="hello") + + +def test_process_last_received_message(): + + # Create a mock function to be used as a hook + def expand_message(message): + return message + " [Expanded]" + + dummy_agent_1 = ConversableAgent(name="dummy_agent_1", llm_config=False, human_input_mode="NEVER") + dummy_agent_1.register_hook("process_last_received_message", expand_message) + + # Normal message + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there"}, + {"role": "user", "content": "How are you?"}, + ] + + processed_messages = messages.copy() + dummy_agent_1.generate_reply(messages=processed_messages, sender=None) + assert processed_messages[-2]["content"] == "Hi there" + assert processed_messages[-1]["content"] == "How are you? [Expanded]" + + +@pytest.mark.asyncio +async def test_a_process_last_received_message(): + + # Create a mock function to be used as a hook + async def expand_message(message): + await asyncio.sleep(0.5) + return message + " [Expanded]" + + dummy_agent_1 = ConversableAgent(name="dummy_agent_1", llm_config=False, human_input_mode="NEVER") + dummy_agent_1.register_hook("a_process_last_received_message", expand_message) + + # Normal message + messages = [ + {"role": "user", "content": "Hello"}, + {"role": "assistant", "content": "Hi there"}, + {"role": "user", "content": "How are you?"}, + ] + + processed_messages = messages.copy() + await dummy_agent_1.a_generate_reply(messages=processed_messages, sender=None) + assert processed_messages[-2]["content"] == "Hi there" + assert processed_messages[-1]["content"] == "How are you? [Expanded]" + + +def test_process_all_messages_before_reply(): + + messages = [ + {"role": "user", "content": "hello"}, + {"function_call": {"name": "add_num", "arguments": '{ "num_to_be_added": 5 }'}, "role": "assistant"}, + ] + + def _transform_messages(transformed_messages: List[Dict]) -> List[Dict]: + # ensure we are looking at all messages + assert len(transformed_messages) == len(messages), "Message length does not match" + + # deep copy to ensure hooks applied comprehensively + post_transformed_messages = copy.deepcopy(transformed_messages) + + # directly modify the message content for the function call (additional value) + post_transformed_messages[1]["function_call"]["arguments"] = '{ "num_to_be_added": 6 }' + + return post_transformed_messages + + def add_num(num_to_be_added): + given_num = 10 + return num_to_be_added + given_num + + dummy_agent_2 = ConversableAgent( + name="user_proxy", llm_config=False, human_input_mode="TERMINATE", function_map={"add_num": add_num} + ) + + # Baseline check before hook is executed + assert ( + dummy_agent_2.generate_reply(messages=messages, sender=None)["content"] == "15" + ), "generate_reply not working when sender is None" + + dummy_agent_2.register_hook("process_all_messages_before_reply", _transform_messages) + + # Hook is applied, updating the message content for the function call + assert ( + dummy_agent_2.generate_reply(messages=messages, sender=None)["content"] == "16" + ), "generate_reply not working when sender is None" + + +@pytest.mark.asyncio +async def test_a_process_all_messages_before_reply(): + + messages = [ + {"role": "user", "content": "hello"}, + {"function_call": {"name": "add_num", "arguments": '{ "num_to_be_added": 5 }'}, "role": "assistant"}, + ] + + async def a_transform_messages(transformed_messages: List[Dict]) -> List[Dict]: + + # ensure we are looking at all messages + assert len(transformed_messages) == len(messages), "Message length does not match" + + # Simulating an async operation with asyncio.sleep + await asyncio.sleep(0.5) + + # deep copy to ensure hooks applied comprehensively + post_transformed_messages = copy.deepcopy(transformed_messages) + + # directly modify the message content for the function call (additional value) + post_transformed_messages[1]["function_call"]["arguments"] = '{ "num_to_be_added": 6 }' + + return post_transformed_messages + + def add_num(num_to_be_added): + given_num = 10 + return num_to_be_added + given_num + + dummy_agent_2 = ConversableAgent( + name="user_proxy", llm_config=False, human_input_mode="TERMINATE", function_map={"add_num": add_num} + ) + + # Baseline check before hook is executed + response = await dummy_agent_2.a_generate_reply(messages=messages, sender=None) + assert response["content"] == "15", "generate_reply not working when sender is None" + + dummy_agent_2.register_hook("a_process_all_messages_before_reply", a_transform_messages) + + # Hook is applied, updating the message content for the function call + response = await dummy_agent_2.a_generate_reply(messages=messages, sender=None) + assert response["content"] == "16", "generate_reply not working when sender is None" + + def test_messages_with_carryover(): agent1 = autogen.ConversableAgent( "alice", diff --git a/test/agentchat/test_groupchat.py b/test/agentchat/test_groupchat.py index 39e8fb063026..0cb652ce54b7 100755 --- a/test/agentchat/test_groupchat.py +++ b/test/agentchat/test_groupchat.py @@ -4,11 +4,11 @@ import io import json import logging +from types import SimpleNamespace from typing import Any, Dict, List, Optional -from unittest import TestCase, mock +from unittest import mock import pytest -from test_assistant_agent import KEY_LOC, OAI_CONFIG_LIST import autogen from autogen import Agent, AssistantAgent, GroupChat, GroupChatManager @@ -2062,6 +2062,60 @@ def test_manager_resume_messages(): return_agent, return_message = manager.resume(messages="Let's get this conversation started.") +def test_custom_model_client(): + class CustomModelClient: + def __init__(self, config, **kwargs): + print(f"CustomModelClient config: {config}") + + def create(self, params): + num_of_responses = params.get("n", 1) + + response = SimpleNamespace() + response.choices = [] + response.model = "test_model_name" + + for _ in range(num_of_responses): + text = "this is a dummy text response" + choice = SimpleNamespace() + choice.message = SimpleNamespace() + choice.message.content = text + choice.message.function_call = None + response.choices.append(choice) + return response + + def message_retrieval(self, response): + choices = response.choices + return [choice.message.content for choice in choices] + + def cost(self, response) -> float: + response.cost = 0 + return 0 + + @staticmethod + def get_usage(response): + return {} + + llm_config = {"config_list": [{"model": "test_model_name", "model_client_cls": "CustomModelClient"}]} + + group_chat = autogen.GroupChat( + agents=[], + messages=[], + max_round=3, + select_speaker_auto_llm_config=llm_config, + select_speaker_auto_model_client_cls=CustomModelClient, + ) + + checking_agent, speaker_selection_agent = group_chat._create_internal_agents( + agents=[], messages=[], max_attempts=3, validate_speaker_name=(True, "test") + ) + + # Check that the custom model client is assigned to the speaker selection agent + assert isinstance(speaker_selection_agent.client._clients[0], CustomModelClient) + + # Check that the LLM Config is assigned + assert speaker_selection_agent.client._config_list == llm_config["config_list"] + + def test_select_speaker_transform_messages(): """Tests adding transform messages to a GroupChat for speaker selection when in 'auto' mode""" @@ -2127,8 +2181,9 @@ def test_select_speaker_transform_messages(): # test_select_speaker_auto_messages() # test_manager_messages_to_string() # test_manager_messages_from_string() - test_manager_resume_functions() + # test_manager_resume_functions() # test_manager_resume_returns() # test_manager_resume_messages() + # test_custom_model_client() # test_select_speaker_transform_messages() pass diff --git a/test/agentchat/test_tool_calls.py b/test/agentchat/test_tool_calls.py index e7d45c5918d9..fa7602104fa0 100755 --- a/test/agentchat/test_tool_calls.py +++ b/test/agentchat/test_tool_calls.py @@ -144,7 +144,7 @@ def test_update_tool(): config_list_gpt4 = autogen.config_list_from_json( OAI_CONFIG_LIST, filter_dict={ - "tags": ["gpt-4"], + "tags": ["gpt-4o-mini"], }, file_location=KEY_LOC, ) diff --git a/test/coding/test_kubernetes.commandline_code_executor.md b/test/coding/test_kubernetes.commandline_code_executor.md new file mode 100644 index 000000000000..ad9348dc6007 --- /dev/null +++ b/test/coding/test_kubernetes.commandline_code_executor.md @@ -0,0 +1,44 @@ +# Test Environment for autogen.coding.kubernetes.PodCommandLineCodeExecutor + +To test PodCommandLineCodeExecutor, the following environment is required. +- kubernetes cluster config file +- autogen package + +## kubernetes cluster config file + +kubernetes cluster config file, kubeconfig file's location should be set on environment variable `KUBECONFIG` or +It must be located in the .kube/config path of your home directory. + +For Windows, `C:\Users\<>\.kube\config`, +For Linux or MacOS, place the kubeconfig file in the `/home/<>/.kube/config` directory. + +## package install + +Clone autogen github repository for package install and testing + +Clone the repository with the command below. + +before contribution +```sh +git clone -b k8s-code-executor https://github.com/questcollector/autogen.git +``` + +after contribution +```sh +git clone https://github.com/microsoft/autogen.git +``` + +install autogen with kubernetes >= 27.0.2 + +```sh +cd autogen +pip install .[kubernetes] -U +``` + +## test execution + +Perform the test with the following command + +```sh +pytest test/coding/test_kubernetes_commandline_code_executor.py +``` diff --git a/test/coding/test_kubernetes_commandline_code_executor.py b/test/coding/test_kubernetes_commandline_code_executor.py new file mode 100644 index 000000000000..09e6b36aafaf --- /dev/null +++ b/test/coding/test_kubernetes_commandline_code_executor.py @@ -0,0 +1,203 @@ +import importlib +import os +import sys +from pathlib import Path + +import pytest + +from autogen.code_utils import TIMEOUT_MSG +from autogen.coding.base import CodeBlock, CodeExecutor + +try: + from autogen.coding.kubernetes import PodCommandLineCodeExecutor + + client = importlib.import_module("kubernetes.client") + config = importlib.import_module("kubernetes.config") + + kubeconfig = Path(".kube/config") + if os.environ.get("KUBECONFIG", None): + kubeconfig = Path(os.environ["KUBECONFIG"]) + elif sys.platform == "win32": + kubeconfig = os.environ["userprofile"] / kubeconfig + else: + kubeconfig = os.environ["HOME"] / kubeconfig + + if kubeconfig.is_file(): + config.load_config(config_file=str(kubeconfig)) + api_client = client.CoreV1Api() + api_client.list_namespace() + skip_kubernetes_tests = False + else: + skip_kubernetes_tests = True + + pod_spec = client.V1Pod( + metadata=client.V1ObjectMeta( + name="abcd", namespace="default", annotations={"sidecar.istio.io/inject": "false"} + ), + spec=client.V1PodSpec( + restart_policy="Never", + containers=[ + client.V1Container( + args=["-c", "while true;do sleep 5; done"], + command=["/bin/sh"], + name="abcd", + image="python:3.11-slim", + env=[ + client.V1EnvVar(name="TEST", value="TEST"), + client.V1EnvVar( + name="POD_NAME", + value_from=client.V1EnvVarSource( + field_ref=client.V1ObjectFieldSelector(field_path="metadata.name") + ), + ), + ], + ) + ], + ), + ) +except Exception: + skip_kubernetes_tests = True + + +@pytest.mark.skipif(skip_kubernetes_tests, reason="kubernetes not accessible") +def test_create_default_pod_executor(): + with PodCommandLineCodeExecutor(namespace="default", kube_config_file=str(kubeconfig)) as executor: + assert executor.timeout == 60 + assert executor.work_dir == Path("/workspace") + assert executor._container_name == "autogen-code-exec" + assert executor._pod.metadata.name.startswith("autogen-code-exec-") + _test_execute_code(executor) + + +@pytest.mark.skipif(skip_kubernetes_tests, reason="kubernetes not accessible") +def test_create_node_pod_executor(): + with PodCommandLineCodeExecutor( + image="node:22-alpine", + namespace="default", + work_dir="./app", + timeout=30, + kube_config_file=str(kubeconfig), + execution_policies={"javascript": True}, + ) as executor: + assert executor.timeout == 30 + assert executor.work_dir == Path("./app") + assert executor._container_name == "autogen-code-exec" + assert executor._pod.metadata.name.startswith("autogen-code-exec-") + assert executor.execution_policies["javascript"] + + # Test single code block. + code_blocks = [CodeBlock(code="console.log('hello world!')", language="javascript")] + code_result = executor.execute_code_blocks(code_blocks) + assert code_result.exit_code == 0 and "hello world!" in code_result.output and code_result.code_file is not None + + # Test multiple code blocks. + code_blocks = [ + CodeBlock(code="console.log('hello world!')", language="javascript"), + CodeBlock(code="let a = 100 + 100; console.log(a)", language="javascript"), + ] + code_result = executor.execute_code_blocks(code_blocks) + assert ( + code_result.exit_code == 0 + and "hello world!" in code_result.output + and "200" in code_result.output + and code_result.code_file is not None + ) + + # Test running code. + file_lines = ["console.log('hello world!')", "let a = 100 + 100", "console.log(a)"] + code_blocks = [CodeBlock(code="\n".join(file_lines), language="javascript")] + code_result = executor.execute_code_blocks(code_blocks) + assert ( + code_result.exit_code == 0 + and "hello world!" in code_result.output + and "200" in code_result.output + and code_result.code_file is not None + ) + + +@pytest.mark.skipif(skip_kubernetes_tests, reason="kubernetes not accessible") +def test_create_pod_spec_pod_executor(): + with PodCommandLineCodeExecutor( + pod_spec=pod_spec, container_name="abcd", kube_config_file=str(kubeconfig) + ) as executor: + assert executor.timeout == 60 + assert executor._container_name == "abcd" + assert executor._pod.metadata.name == pod_spec.metadata.name + assert executor._pod.metadata.namespace == pod_spec.metadata.namespace + _test_execute_code(executor) + + # Test bash script. + if sys.platform not in ["win32"]: + code_blocks = [CodeBlock(code="echo $TEST $POD_NAME", language="bash")] + code_result = executor.execute_code_blocks(code_blocks) + assert ( + code_result.exit_code == 0 and "TEST abcd" in code_result.output and code_result.code_file is not None + ) + + +@pytest.mark.skipif(skip_kubernetes_tests, reason="kubernetes not accessible") +def test_pod_executor_timeout(): + with PodCommandLineCodeExecutor(namespace="default", timeout=5, kube_config_file=str(kubeconfig)) as executor: + assert executor.timeout == 5 + assert executor.work_dir == Path("/workspace") + assert executor._container_name == "autogen-code-exec" + assert executor._pod.metadata.name.startswith("autogen-code-exec-") + # Test running code. + file_lines = ["import time", "time.sleep(10)", "a = 100 + 100", "print(a)"] + code_blocks = [CodeBlock(code="\n".join(file_lines), language="python")] + code_result = executor.execute_code_blocks(code_blocks) + assert code_result.exit_code == 124 and TIMEOUT_MSG in code_result.output and code_result.code_file is not None + + +def _test_execute_code(executor: CodeExecutor) -> None: + # Test single code block. + code_blocks = [CodeBlock(code="import sys; print('hello world!')", language="python")] + code_result = executor.execute_code_blocks(code_blocks) + assert code_result.exit_code == 0 and "hello world!" in code_result.output and code_result.code_file is not None + + # Test multiple code blocks. + code_blocks = [ + CodeBlock(code="import sys; print('hello world!')", language="python"), + CodeBlock(code="a = 100 + 100; print(a)", language="python"), + ] + code_result = executor.execute_code_blocks(code_blocks) + assert ( + code_result.exit_code == 0 + and "hello world!" in code_result.output + and "200" in code_result.output + and code_result.code_file is not None + ) + + # Test bash script. + if sys.platform not in ["win32"]: + code_blocks = [CodeBlock(code="echo 'hello world!'", language="bash")] + code_result = executor.execute_code_blocks(code_blocks) + assert code_result.exit_code == 0 and "hello world!" in code_result.output and code_result.code_file is not None + + # Test running code. + file_lines = ["import sys", "print('hello world!')", "a = 100 + 100", "print(a)"] + code_blocks = [CodeBlock(code="\n".join(file_lines), language="python")] + code_result = executor.execute_code_blocks(code_blocks) + assert ( + code_result.exit_code == 0 + and "hello world!" in code_result.output + and "200" in code_result.output + and code_result.code_file is not None + ) + + # Test running code has filename. + file_lines = ["# filename: test.py", "import sys", "print('hello world!')", "a = 100 + 100", "print(a)"] + code_blocks = [CodeBlock(code="\n".join(file_lines), language="python")] + code_result = executor.execute_code_blocks(code_blocks) + print(code_result.code_file) + assert ( + code_result.exit_code == 0 + and "hello world!" in code_result.output + and "200" in code_result.output + and code_result.code_file.find("test.py") > 0 + ) + + # Test error code. + code_blocks = [CodeBlock(code="print(sys.platform)", language="python")] + code_result = executor.execute_code_blocks(code_blocks) + assert code_result.exit_code == 1 and "Traceback" in code_result.output and code_result.code_file is not None diff --git a/test/io/test_websockets.py b/test/io/test_websockets.py index 1458afcd88cb..ee2085169e05 100644 --- a/test/io/test_websockets.py +++ b/test/io/test_websockets.py @@ -97,14 +97,8 @@ def on_connect(iostream: IOWebsockets, success_dict: Dict[str, bool] = success_d OAI_CONFIG_LIST, filter_dict={ "model": [ + "gpt-4o-mini", "gpt-3.5-turbo", - "gpt-3.5-turbo-16k", - "gpt-4", - "gpt-4-0314", - "gpt4", - "gpt-4-32k", - "gpt-4-32k-0314", - "gpt-4-32k-v0314", ], }, file_location=KEY_LOC, diff --git a/test/oai/_test_completion.py b/test/oai/_test_completion.py index fe410255d2fb..af479ca56511 100755 --- a/test/oai/_test_completion.py +++ b/test/oai/_test_completion.py @@ -143,13 +143,8 @@ def test_nocontext(): file_location=KEY_LOC, filter_dict={ "model": { + "gpt-4o-mini", "gpt-3.5-turbo", - "gpt-3.5-turbo-16k", - "gpt-3.5-turbo-16k-0613", - "gpt-3.5-turbo-0301", - "chatgpt-35-turbo-0301", - "gpt-35-turbo-v0301", - "gpt", }, }, ), @@ -179,13 +174,8 @@ def test_humaneval(num_samples=1): env_or_file=OAI_CONFIG_LIST, filter_dict={ "model": { + "gpt-4o-mini", "gpt-3.5-turbo", - "gpt-3.5-turbo-16k", - "gpt-3.5-turbo-16k-0613", - "gpt-3.5-turbo-0301", - "chatgpt-35-turbo-0301", - "gpt-35-turbo-v0301", - "gpt", }, }, file_location=KEY_LOC, diff --git a/test/oai/test_cerebras.py b/test/oai/test_cerebras.py new file mode 100644 index 000000000000..7f84ae3f9d56 --- /dev/null +++ b/test/oai/test_cerebras.py @@ -0,0 +1,248 @@ +from unittest.mock import MagicMock, patch + +import pytest + +try: + from autogen.oai.cerebras import CerebrasClient, calculate_cerebras_cost + + skip = False +except ImportError: + CerebrasClient = object + InternalServerError = object + skip = True + + +# Fixtures for mock data +@pytest.fixture +def mock_response(): + class MockResponse: + def __init__(self, text, choices, usage, cost, model): + self.text = text + self.choices = choices + self.usage = usage + self.cost = cost + self.model = model + + return MockResponse + + +@pytest.fixture +def cerebras_client(): + return CerebrasClient(api_key="fake_api_key") + + +skip_reason = "Cerebras dependency is not installed" + + +# Test initialization and configuration +@pytest.mark.skipif(skip, reason=skip_reason) +def test_initialization(): + + # Missing any api_key + with pytest.raises(AssertionError) as assertinfo: + CerebrasClient() # Should raise an AssertionError due to missing api_key + + assert ( + "Please include the api_key in your config list entry for Cerebras or set the CEREBRAS_API_KEY env variable." + in str(assertinfo.value) + ) + + # Creation works + CerebrasClient(api_key="fake_api_key") # Should create okay now. + + +# Test standard initialization +@pytest.mark.skipif(skip, reason=skip_reason) +def test_valid_initialization(cerebras_client): + assert cerebras_client.api_key == "fake_api_key", "Config api_key should be correctly set" + + +# Test parameters +@pytest.mark.skipif(skip, reason=skip_reason) +def test_parsing_params(cerebras_client): + # All parameters + params = { + "model": "llama3.1-8b", + "max_tokens": 1000, + "seed": 42, + "stream": False, + "temperature": 1, + "top_p": 0.8, + } + expected_params = { + "model": "llama3.1-8b", + "max_tokens": 1000, + "seed": 42, + "stream": False, + "temperature": 1, + "top_p": 0.8, + } + result = cerebras_client.parse_params(params) + assert result == expected_params + + # Only model, others set as defaults + params = { + "model": "llama3.1-8b", + } + expected_params = { + "model": "llama3.1-8b", + "max_tokens": None, + "seed": None, + "stream": False, + "temperature": 1, + "top_p": None, + } + result = cerebras_client.parse_params(params) + assert result == expected_params + + # Incorrect types, defaults should be set, will show warnings but not trigger assertions + params = { + "model": "llama3.1-8b", + "max_tokens": "1000", + "seed": "42", + "stream": "False", + "temperature": "1", + "top_p": "0.8", + } + result = cerebras_client.parse_params(params) + assert result == expected_params + + # Values outside bounds, should warn and set to defaults + params = { + "model": "llama3.1-8b", + "temperature": 33123, + } + result = cerebras_client.parse_params(params) + assert result == expected_params + + # No model + params = { + "temperature": 1, + } + + with pytest.raises(AssertionError) as assertinfo: + result = cerebras_client.parse_params(params) + + assert "Please specify the 'model' in your config list entry to nominate the Cerebras model to use." in str( + assertinfo.value + ) + + +# Test cost calculation +@pytest.mark.skipif(skip, reason=skip_reason) +def test_cost_calculation(mock_response): + response = mock_response( + text="Example response", + choices=[{"message": "Test message 1"}], + usage={"prompt_tokens": 500, "completion_tokens": 300, "total_tokens": 800}, + cost=None, + model="llama3.1-70b", + ) + calculated_cost = calculate_cerebras_cost( + response.usage["prompt_tokens"], response.usage["completion_tokens"], response.model + ) + + # Convert cost per milliion to cost per token. + expected_cost = ( + response.usage["prompt_tokens"] * 0.6 / 1000000 + response.usage["completion_tokens"] * 0.6 / 1000000 + ) + + assert calculated_cost == expected_cost, f"Cost for this should be ${expected_cost} but got ${calculated_cost}" + + +# Test text generation +@pytest.mark.skipif(skip, reason=skip_reason) +@patch("autogen.oai.cerebras.CerebrasClient.create") +def test_create_response(mock_chat, cerebras_client): + # Mock CerebrasClient.chat response + mock_cerebras_response = MagicMock() + mock_cerebras_response.choices = [ + MagicMock(finish_reason="stop", message=MagicMock(content="Example Cerebras response", tool_calls=None)) + ] + mock_cerebras_response.id = "mock_cerebras_response_id" + mock_cerebras_response.model = "llama3.1-70b" + mock_cerebras_response.usage = MagicMock(prompt_tokens=10, completion_tokens=20) # Example token usage + + mock_chat.return_value = mock_cerebras_response + + # Test parameters + params = { + "messages": [{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "World"}], + "model": "llama3.1-70b", + } + + # Call the create method + response = cerebras_client.create(params) + + # Assertions to check if response is structured as expected + assert ( + response.choices[0].message.content == "Example Cerebras response" + ), "Response content should match expected output" + assert response.id == "mock_cerebras_response_id", "Response ID should match the mocked response ID" + assert response.model == "llama3.1-70b", "Response model should match the mocked response model" + assert response.usage.prompt_tokens == 10, "Response prompt tokens should match the mocked response usage" + assert response.usage.completion_tokens == 20, "Response completion tokens should match the mocked response usage" + + +# Test functions/tools +@pytest.mark.skipif(skip, reason=skip_reason) +@patch("autogen.oai.cerebras.CerebrasClient.create") +def test_create_response_with_tool_call(mock_chat, cerebras_client): + # Mock `cerebras_response = client.chat(**cerebras_params)` + mock_function = MagicMock(name="currency_calculator") + mock_function.name = "currency_calculator" + mock_function.arguments = '{"base_currency": "EUR", "quote_currency": "USD", "base_amount": 123.45}' + + mock_function_2 = MagicMock(name="get_weather") + mock_function_2.name = "get_weather" + mock_function_2.arguments = '{"location": "Chicago"}' + + mock_chat.return_value = MagicMock( + choices=[ + MagicMock( + finish_reason="tool_calls", + message=MagicMock( + content="Sample text about the functions", + tool_calls=[ + MagicMock(id="gdRdrvnHh", function=mock_function), + MagicMock(id="abRdrvnHh", function=mock_function_2), + ], + ), + ) + ], + id="mock_cerebras_response_id", + model="llama3.1-70b", + usage=MagicMock(prompt_tokens=10, completion_tokens=20), + ) + + # Construct parameters + converted_functions = [ + { + "type": "function", + "function": { + "description": "Currency exchange calculator.", + "name": "currency_calculator", + "parameters": { + "type": "object", + "properties": { + "base_amount": {"type": "number", "description": "Amount of currency in base_currency"}, + }, + "required": ["base_amount"], + }, + }, + } + ] + cerebras_messages = [ + {"role": "user", "content": "How much is 123.45 EUR in USD?"}, + {"role": "assistant", "content": "World"}, + ] + + # Call the create method + response = cerebras_client.create( + {"messages": cerebras_messages, "tools": converted_functions, "model": "llama3.1-70b"} + ) + + # Assertions to check if the functions and content are included in the response + assert response.choices[0].message.content == "Sample text about the functions" + assert response.choices[0].message.tool_calls[0].function.name == "currency_calculator" + assert response.choices[0].message.tool_calls[1].function.name == "get_weather" diff --git a/test/oai/test_client.py b/test/oai/test_client.py index bd8b072e6127..3387001cf4ad 100755 --- a/test/oai/test_client.py +++ b/test/oai/test_client.py @@ -66,7 +66,8 @@ def get_usage(response): return {} -@pytest.mark.skipif(skip, reason="openai>=1 not installed") +# @pytest.mark.skipif(skip, reason="openai>=1 not installed") +@pytest.mark.skip(reason="This test is not working until Azure settings are updated") def test_aoai_chat_completion(): config_list = config_list_from_json( env_or_file=OAI_CONFIG_LIST, @@ -88,7 +89,8 @@ def test_aoai_chat_completion(): print(client.extract_text_or_completion_object(response)) -@pytest.mark.skipif(skip or not TOOL_ENABLED, reason="openai>=1.1.0 not installed") +# @pytest.mark.skipif(skip or not TOOL_ENABLED, reason="openai>=1.1.0 not installed") +@pytest.mark.skip(reason="This test is not working until Azure settings are updated") def test_oai_tool_calling_extraction(): config_list = config_list_from_json( env_or_file=OAI_CONFIG_LIST, diff --git a/test/oai/test_client_stream.py b/test/oai/test_client_stream.py index 59abd97151ad..405bc32be27d 100755 --- a/test/oai/test_client_stream.py +++ b/test/oai/test_client_stream.py @@ -33,6 +33,7 @@ @pytest.mark.skipif(skip, reason="openai>=1 not installed") +@pytest.mark.skip(reason="This test is not working until Azure settings are updated.") def test_aoai_chat_completion_stream() -> None: config_list = config_list_from_json( env_or_file=OAI_CONFIG_LIST, @@ -236,7 +237,7 @@ def test_chat_tools_stream() -> None: config_list = config_list_from_json( env_or_file=OAI_CONFIG_LIST, file_location=KEY_LOC, - filter_dict={"tags": ["multitool"]}, + filter_dict={"tags": ["tool"]}, ) tools = [ { diff --git a/test/oai/test_gemini.py b/test/oai/test_gemini.py index 61fdbe6d735a..e6ff3e02672d 100644 --- a/test/oai/test_gemini.py +++ b/test/oai/test_gemini.py @@ -1,3 +1,4 @@ +import json import os from unittest.mock import MagicMock, patch @@ -10,7 +11,9 @@ from google.cloud.aiplatform.initializer import global_config as vertexai_global_config from vertexai.generative_models import HarmBlockThreshold as VertexAIHarmBlockThreshold from vertexai.generative_models import HarmCategory as VertexAIHarmCategory + from vertexai.generative_models import Part as VertexAIPart from vertexai.generative_models import SafetySetting as VertexAISafetySetting + from vertexai.generative_models import ToolConfig as VertexAIToolConfig from autogen.oai.gemini import GeminiClient @@ -20,6 +23,8 @@ VertexAIHarmBlockThreshold = object VertexAIHarmCategory = object VertexAISafetySetting = object + VertexAIPart = object + VertexAIToolConfig = object vertexai_global_config = object InternalServerError = object skip = True @@ -234,8 +239,6 @@ def test_vertexai_safety_setting_list(gemini_client): for category in harm_categories ] - print(safety_settings) - converted_safety_settings = GeminiClient._to_vertexai_safety_settings(safety_settings) def compare_safety_settings(converted_safety_settings, expected_safety_settings): @@ -250,6 +253,59 @@ def compare_safety_settings(converted_safety_settings, expected_safety_settings) assert all(settings_comparison), "Converted safety settings are incorrect" +@pytest.mark.skipif(skip, reason="Google GenAI dependency is not installed") +def test_vertexai_tool_config(gemini_client): + + tools = [{"function_name": "calculator"}] + + tool_config = {"function_calling_config": {"mode": "ANY"}} + + expected_tool_config = VertexAIToolConfig( + function_calling_config=VertexAIToolConfig.FunctionCallingConfig( + mode=VertexAIToolConfig.FunctionCallingConfig.Mode.ANY, + allowed_function_names=["calculator"], + ) + ) + + converted_tool_config = GeminiClient._to_vertexai_tool_config(tool_config, tools) + + converted_mode = converted_tool_config._gapic_tool_config.function_calling_config.mode + expected_mode = expected_tool_config._gapic_tool_config.function_calling_config.mode + converted_allowed_func = converted_tool_config._gapic_tool_config.function_calling_config.allowed_function_names + expected_allowed_func = expected_tool_config._gapic_tool_config.function_calling_config.allowed_function_names + + assert converted_mode == expected_mode, "Function calling mode is not converted correctly" + assert ( + converted_allowed_func == expected_allowed_func + ), "Function calling allowed function names is not converted correctly" + + +@pytest.mark.skipif(skip, reason="Google GenAI dependency is not installed") +def test_vertexai_tool_config_no_functions(gemini_client): + + tools = [] + + tool_config = {"function_calling_config": {"mode": "ANY"}} + + expected_tool_config = VertexAIToolConfig( + function_calling_config=VertexAIToolConfig.FunctionCallingConfig( + mode=VertexAIToolConfig.FunctionCallingConfig.Mode.ANY, + ) + ) + + converted_tool_config = GeminiClient._to_vertexai_tool_config(tool_config, tools) + + converted_mode = converted_tool_config._gapic_tool_config.function_calling_config.mode + expected_mode = expected_tool_config._gapic_tool_config.function_calling_config.mode + converted_allowed_func = converted_tool_config._gapic_tool_config.function_calling_config.allowed_function_names + expected_allowed_func = expected_tool_config._gapic_tool_config.function_calling_config.allowed_function_names + + assert converted_mode == expected_mode, "Function calling mode is not converted correctly" + assert ( + converted_allowed_func == expected_allowed_func + ), "Function calling allowed function names is not converted correctly" + + # Test error handling @patch("autogen.oai.gemini.genai") @pytest.mark.skipif(skip, reason="Google GenAI dependency is not installed") @@ -279,9 +335,10 @@ def test_cost_calculation(gemini_client, mock_response): @pytest.mark.skipif(skip, reason="Google GenAI dependency is not installed") +@patch("autogen.oai.gemini.Content") @patch("autogen.oai.gemini.genai.GenerativeModel") @patch("autogen.oai.gemini.genai.configure") -def test_create_response(mock_configure, mock_generative_model, gemini_client): +def test_create_response(mock_configure, mock_generative_model, mock_content, gemini_client): # Mock the genai model configuration and creation process mock_chat = MagicMock() mock_model = MagicMock() @@ -292,6 +349,8 @@ def test_create_response(mock_configure, mock_generative_model, gemini_client): # Set up a mock for the chat history item access and the text attribute return mock_history_part = MagicMock() mock_history_part.text = "Example response" + mock_history_part.function_call = None + mock_chat.history.__getitem__.return_value.parts.__iter__.return_value = iter([mock_history_part]) mock_chat.history.__getitem__.return_value.parts.__getitem__.return_value = mock_history_part # Setup the mock to return a mocked chat response @@ -306,6 +365,55 @@ def test_create_response(mock_configure, mock_generative_model, gemini_client): assert response.choices[0].message.content == "Example response", "Response content should match expected output" +@pytest.mark.skipif(skip, reason="Google GenAI dependency is not installed") +@patch("autogen.oai.gemini.Part") +@patch("autogen.oai.gemini.Content") +@patch("autogen.oai.gemini.genai.GenerativeModel") +@patch("autogen.oai.gemini.genai.configure") +def test_create_function_call_response(mock_configure, mock_generative_model, mock_content, mock_part, gemini_client): + # Mock the genai model configuration and creation process + mock_chat = MagicMock() + mock_model = MagicMock() + mock_configure.return_value = None + mock_generative_model.return_value = mock_model + mock_model.start_chat.return_value = mock_chat + + mock_part.to_dict.return_value = { + "function_call": {"name": "function_name", "args": {"arg1": "value1", "arg2": "value2"}} + } + + # Set up a mock for the chat history item access and the text attribute return + mock_history_part = MagicMock() + mock_history_part.text = None + mock_history_part.function_call.name = "function_name" + mock_history_part.function_call.args = {"arg1": "value1", "arg2": "value2"} + mock_chat.history.__getitem__.return_value.parts.__iter__.return_value = iter([mock_history_part]) + + # Setup the mock to return a mocked chat response + mock_chat.send_message.return_value = MagicMock( + history=[ + MagicMock( + parts=[ + MagicMock( + function_call=MagicMock(name="function_name", arguments='{"arg1": "value1", "arg2": "value2"}') + ) + ] + ) + ] + ) + + # Call the create method + response = gemini_client.create( + {"model": "gemini-pro", "messages": [{"content": "Hello", "role": "user"}], "stream": False} + ) + + # Assertions to check if response is structured as expected + assert ( + response.choices[0].message.tool_calls[0].function.name == "function_name" + and json.loads(response.choices[0].message.tool_calls[0].function.arguments)["arg1"] == "value1" + ), "Response content should match expected output" + + @pytest.mark.skipif(skip, reason="Google GenAI dependency is not installed") @patch("autogen.oai.gemini.GenerativeModel") @patch("autogen.oai.gemini.vertexai.init") @@ -320,7 +428,9 @@ def test_vertexai_create_response(mock_init, mock_generative_model, gemini_clien # Set up a mock for the chat history item access and the text attribute return mock_history_part = MagicMock() mock_history_part.text = "Example response" - mock_chat.history.__getitem__.return_value.parts.__getitem__.return_value = mock_history_part + mock_history_part.function_call = None + mock_history_part.role = "model" + mock_chat.history.__getitem__.return_value.parts.__iter__.return_value = iter([mock_history_part]) # Setup the mock to return a mocked chat response mock_chat.send_message.return_value = MagicMock(history=[MagicMock(parts=[MagicMock(text="Example response")])]) @@ -330,10 +440,60 @@ def test_vertexai_create_response(mock_init, mock_generative_model, gemini_clien {"model": "gemini-pro", "messages": [{"content": "Hello", "role": "user"}], "stream": False} ) - # Assertions to check if response is structured as expected assert response.choices[0].message.content == "Example response", "Response content should match expected output" +@pytest.mark.skipif(skip, reason="Google GenAI dependency is not installed") +@patch("autogen.oai.gemini.VertexAIPart") +@patch("autogen.oai.gemini.VertexAIContent") +@patch("autogen.oai.gemini.GenerativeModel") +@patch("autogen.oai.gemini.vertexai.init") +def test_vertexai_create_function_call_response( + mock_init, mock_generative_model, mock_content, mock_part, gemini_client_with_credentials +): + # Mock the genai model configuration and creation process + mock_chat = MagicMock() + mock_model = MagicMock() + mock_init.return_value = None + mock_generative_model.return_value = mock_model + mock_model.start_chat.return_value = mock_chat + + mock_part.to_dict.return_value = { + "function_call": {"name": "function_name", "args": {"arg1": "value1", "arg2": "value2"}} + } + + # Set up a mock for the chat history item access and the text attribute return + mock_history_part = MagicMock() + mock_history_part.text = None + mock_history_part.function_call.name = "function_name" + mock_history_part.function_call.args = {"arg1": "value1", "arg2": "value2"} + mock_chat.history.__getitem__.return_value.parts.__iter__.return_value = iter([mock_history_part]) + + # Setup the mock to return a mocked chat response + mock_chat.send_message.return_value = MagicMock( + history=[ + MagicMock( + parts=[ + MagicMock( + function_call=MagicMock(name="function_name", arguments='{"arg1": "value1", "arg2": "value2"}') + ) + ] + ) + ] + ) + + # Call the create method + response = gemini_client_with_credentials.create( + {"model": "gemini-pro", "messages": [{"content": "Hello", "role": "user"}], "stream": False} + ) + + # Assertions to check if response is structured as expected + assert ( + response.choices[0].message.tool_calls[0].function.name == "function_name" + and json.loads(response.choices[0].message.tool_calls[0].function.arguments)["arg1"] == "value1" + ), "Response content should match expected output" + + @pytest.mark.skipif(skip, reason="Google GenAI dependency is not installed") @patch("autogen.oai.gemini.GenerativeModel") @patch("autogen.oai.gemini.vertexai.init") @@ -348,6 +508,8 @@ def test_vertexai_default_auth_create_response(mock_init, mock_generative_model, # Set up a mock for the chat history item access and the text attribute return mock_history_part = MagicMock() mock_history_part.text = "Example response" + mock_history_part.function_call = None + mock_chat.history.__getitem__.return_value.parts.__iter__.return_value = iter([mock_history_part]) mock_chat.history.__getitem__.return_value.parts.__getitem__.return_value = mock_history_part # Setup the mock to return a mocked chat response @@ -373,11 +535,11 @@ def test_create_vision_model_response(mock_configure, mock_generative_model, gem # Set up a mock to simulate the vision model behavior mock_vision_response = MagicMock() - mock_vision_part = MagicMock(text="Vision model output") + mock_vision_part = MagicMock(text="Vision model output", function_call=None) # Setting up the chain of return values for vision model response - mock_vision_response._result.candidates.__getitem__.return_value.content.parts.__getitem__.return_value = ( - mock_vision_part + mock_vision_response._result.candidates.__getitem__.return_value.content.parts.__iter__.return_value = iter( + [mock_vision_part] ) mock_model.generate_content.return_value = mock_vision_response @@ -420,10 +582,12 @@ def test_vertexai_create_vision_model_response(mock_init, mock_generative_model, # Set up a mock to simulate the vision model behavior mock_vision_response = MagicMock() - mock_vision_part = MagicMock(text="Vision model output") + mock_vision_part = MagicMock(text="Vision model output", function_call=None) # Setting up the chain of return values for vision model response - mock_vision_response.candidates.__getitem__.return_value.content.parts.__getitem__.return_value = mock_vision_part + mock_vision_response.candidates.__getitem__.return_value.content.parts.__iter__.return_value = iter( + [mock_vision_part] + ) mock_model.generate_content.return_value = mock_vision_response diff --git a/test/oai/test_ollama.py b/test/oai/test_ollama.py new file mode 100644 index 000000000000..5491a04d7bec --- /dev/null +++ b/test/oai/test_ollama.py @@ -0,0 +1,302 @@ +from unittest.mock import MagicMock, patch + +import pytest + +try: + from autogen.oai.ollama import OllamaClient, response_to_tool_call + + skip = False +except ImportError: + OllamaClient = object + InternalServerError = object + skip = True + + +# Fixtures for mock data +@pytest.fixture +def mock_response(): + class MockResponse: + def __init__(self, text, choices, usage, cost, model): + self.text = text + self.choices = choices + self.usage = usage + self.cost = cost + self.model = model + + return MockResponse + + +@pytest.fixture +def ollama_client(): + + # Set Ollama client with some default values + client = OllamaClient() + + client._native_tool_calls = True + client._tools_in_conversation = False + + return client + + +skip_reason = "Ollama dependency is not installed" + + +# Test initialization and configuration +@pytest.mark.skipif(skip, reason=skip_reason) +def test_initialization(): + + # Creation works without an api_key + OllamaClient() + + +# Test parameters +@pytest.mark.skipif(skip, reason=skip_reason) +def test_parsing_params(ollama_client): + # All parameters (with default values) + params = { + "model": "llama3.1:8b", + "temperature": 0.8, + "num_predict": 128, + "repeat_penalty": 1.1, + "seed": 42, + "top_k": 40, + "top_p": 0.9, + "stream": False, + } + expected_params = { + "model": "llama3.1:8b", + "options": { + "repeat_penalty": 1.1, + "seed": 42, + "temperature": 0.8, + "num_predict": 128, + "top_k": 40, + "top_p": 0.9, + }, + "stream": False, + } + result = ollama_client.parse_params(params) + assert result == expected_params + + # Incorrect types, defaults should be set, will show warnings but not trigger assertions + params = { + "model": "llama3.1:8b", + "temperature": "0.5", + "num_predict": "128", + "repeat_penalty": "1.1", + "seed": "42", + "top_k": "40", + "top_p": "0.9", + "stream": "True", + } + result = ollama_client.parse_params(params) + assert result == expected_params + + # Only model, others set as defaults if they are mandatory + params = { + "model": "llama3.1:8b", + } + expected_params = {"model": "llama3.1:8b", "stream": False} + result = ollama_client.parse_params(params) + assert result == expected_params + + # No model + params = { + "temperature": 0.8, + } + + with pytest.raises(AssertionError) as assertinfo: + result = ollama_client.parse_params(params) + + assert "Please specify the 'model' in your config list entry to nominate the Ollama model to use." in str( + assertinfo.value + ) + + +# Test text generation +@pytest.mark.skipif(skip, reason=skip_reason) +@patch("autogen.oai.ollama.OllamaClient.create") +def test_create_response(mock_chat, ollama_client): + # Mock OllamaClient.chat response + mock_ollama_response = MagicMock() + mock_ollama_response.choices = [ + MagicMock(finish_reason="stop", message=MagicMock(content="Example Ollama response", tool_calls=None)) + ] + mock_ollama_response.id = "mock_ollama_response_id" + mock_ollama_response.model = "llama3.1:8b" + mock_ollama_response.usage = MagicMock(prompt_tokens=10, completion_tokens=20) # Example token usage + + mock_chat.return_value = mock_ollama_response + + # Test parameters + params = { + "messages": [{"role": "user", "content": "Hello"}, {"role": "assistant", "content": "World"}], + "model": "llama3.1:8b", + } + + # Call the create method + response = ollama_client.create(params) + + # Assertions to check if response is structured as expected + assert ( + response.choices[0].message.content == "Example Ollama response" + ), "Response content should match expected output" + assert response.id == "mock_ollama_response_id", "Response ID should match the mocked response ID" + assert response.model == "llama3.1:8b", "Response model should match the mocked response model" + assert response.usage.prompt_tokens == 10, "Response prompt tokens should match the mocked response usage" + assert response.usage.completion_tokens == 20, "Response completion tokens should match the mocked response usage" + + +# Test functions/tools +@pytest.mark.skipif(skip, reason=skip_reason) +@patch("autogen.oai.ollama.OllamaClient.create") +def test_create_response_with_tool_call(mock_chat, ollama_client): + # Mock OllamaClient.chat response + mock_function = MagicMock(name="currency_calculator") + mock_function.name = "currency_calculator" + mock_function.arguments = '{"base_currency": "EUR", "quote_currency": "USD", "base_amount": 123.45}' + + mock_function_2 = MagicMock(name="get_weather") + mock_function_2.name = "get_weather" + mock_function_2.arguments = '{"location": "New York"}' + + mock_chat.return_value = MagicMock( + choices=[ + MagicMock( + finish_reason="tool_calls", + message=MagicMock( + content="Sample text about the functions", + tool_calls=[ + MagicMock(id="gdRdrvnHh", function=mock_function), + MagicMock(id="abRdrvnHh", function=mock_function_2), + ], + ), + ) + ], + id="mock_ollama_response_id", + model="llama3.1:8b", + usage=MagicMock(prompt_tokens=10, completion_tokens=20), + ) + + # Construct parameters + converted_functions = [ + { + "type": "function", + "function": { + "description": "Currency exchange calculator.", + "name": "currency_calculator", + "parameters": { + "type": "object", + "properties": { + "base_amount": {"type": "number", "description": "Amount of currency in base_currency"}, + }, + "required": ["base_amount"], + }, + }, + } + ] + ollama_messages = [ + {"role": "user", "content": "How much is 123.45 EUR in USD?"}, + {"role": "assistant", "content": "World"}, + ] + + # Call the create method + response = ollama_client.create({"messages": ollama_messages, "tools": converted_functions, "model": "llama3.1:8b"}) + + # Assertions to check if the functions and content are included in the response + assert response.choices[0].message.content == "Sample text about the functions" + assert response.choices[0].message.tool_calls[0].function.name == "currency_calculator" + assert response.choices[0].message.tool_calls[1].function.name == "get_weather" + + +# Test function parsing with manual tool calling +@pytest.mark.skipif(skip, reason=skip_reason) +def test_manual_tool_calling_parsing(ollama_client): + # Test the parsing of a tool call within the response content (fully correct) + response_content = """[{"name": "weather_forecast", "arguments":{"location": "New York"}},{"name": "currency_calculator", "arguments":{"base_amount": 123.45, "quote_currency": "EUR", "base_currency": "USD"}}]""" + + response_tool_calls = response_to_tool_call(response_content) + + expected_tool_calls = [ + {"name": "weather_forecast", "arguments": {"location": "New York"}}, + { + "name": "currency_calculator", + "arguments": {"base_amount": 123.45, "quote_currency": "EUR", "base_currency": "USD"}, + }, + ] + + assert ( + response_tool_calls == expected_tool_calls + ), "Manual Tool Calling Parsing of response did not yield correct tool_calls (full string match)" + + # Test the parsing with a substring containing the response content (should still pass) + response_content = """I will call two functions, weather_forecast and currency_calculator:\n[{"name": "weather_forecast", "arguments":{"location": "New York"}},{"name": "currency_calculator", "arguments":{"base_amount": 123.45, "quote_currency": "EUR", "base_currency": "USD"}}]""" + + response_tool_calls = response_to_tool_call(response_content) + + assert ( + response_tool_calls == expected_tool_calls + ), "Manual Tool Calling Parsing of response did not yield correct tool_calls (partial string match)" + + # Test the parsing with an invalid function call + response_content = """[{"function": "weather_forecast", "args":{"location": "New York"}},{"function": "currency_calculator", "args":{"base_amount": 123.45, "quote_currency": "EUR", "base_currency": "USD"}}]""" + + response_tool_calls = response_to_tool_call(response_content) + + assert ( + response_tool_calls is None + ), "Manual Tool Calling Parsing of response did not yield correct tool_calls (invalid function call)" + + # Test the parsing with plain text + response_content = """Call the weather_forecast function and pass in 'New York' as the 'location' argument.""" + + response_tool_calls = response_to_tool_call(response_content) + + assert ( + response_tool_calls is None + ), "Manual Tool Calling Parsing of response did not yield correct tool_calls (no function in text)" + + +# Test message conversion from OpenAI to Ollama format +@pytest.mark.skipif(skip, reason=skip_reason) +def test_oai_messages_to_ollama_messages(ollama_client): + # Test that the "name" key is removed + test_messages = [ + {"role": "system", "content": "You are a helpful AI bot."}, + {"role": "user", "name": "anne", "content": "Why is the sky blue?"}, + ] + messages = ollama_client.oai_messages_to_ollama_messages(test_messages, None) + + expected_messages = [ + {"role": "system", "content": "You are a helpful AI bot."}, + {"role": "user", "content": "Why is the sky blue?"}, + ] + + assert messages == expected_messages, "'name' was not removed from messages" + + # Test that there isn't a final system message and it's changed to user + test_messages.append({"role": "system", "content": "Summarise the conversation."}) + + messages = ollama_client.oai_messages_to_ollama_messages(test_messages, None) + + expected_messages = [ + {"role": "system", "content": "You are a helpful AI bot."}, + {"role": "user", "content": "Why is the sky blue?"}, + {"role": "user", "content": "Summarise the conversation."}, + ] + + assert messages == expected_messages, "Final 'system' message was not changed to 'user'" + + # Test that the last message is a user or system message and if not, add a continue message + test_messages[2] = {"role": "assistant", "content": "The sky is blue because that's a great colour."} + + messages = ollama_client.oai_messages_to_ollama_messages(test_messages, None) + + expected_messages = [ + {"role": "system", "content": "You are a helpful AI bot."}, + {"role": "user", "content": "Why is the sky blue?"}, + {"role": "assistant", "content": "The sky is blue because that's a great colour."}, + {"role": "user", "content": "Please continue."}, + ] + + assert messages == expected_messages, "'Please continue' message was not appended." diff --git a/website/blog/2023-06-28-MathChat/index.mdx b/website/blog/2023-06-28-MathChat/index.mdx index be2423de9eed..2521065a6252 100644 --- a/website/blog/2023-06-28-MathChat/index.mdx +++ b/website/blog/2023-06-28-MathChat/index.mdx @@ -76,7 +76,7 @@ We found that compared to basic prompting, which demonstrates the innate capabil For categories like Algebra and Prealgebra, PoT and PS showed little improvement, and in some instances, even led to a decrease in accuracy. However, MathChat was able to enhance total accuracy by around 6% compared to PoT and PS, showing competitive performance across all categories. Remarkably, MathChat improved accuracy in the Algebra category by about 15% over other methods. Note that categories like Intermediate Algebra and Precalculus remained challenging for all methods, with only about 20% of problems solved accurately. The code for experiments can be found at this [repository](https://github.com/yiranwu0/FLAML/tree/gpt_math_solver/flaml/autogen/math). -We now provide an implementation of MathChat using the interactive agents in AutoGen. See this [notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_MathChat.ipynb) for example usage. +We now provide an implementation of MathChat using the interactive agents in AutoGen. See this [notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_MathChat.ipynb) for example usage. ## Future Directions diff --git a/website/blog/2023-10-18-RetrieveChat/index.mdx b/website/blog/2023-10-18-RetrieveChat/index.mdx index d5c78148e446..84bcca5b3064 100644 --- a/website/blog/2023-10-18-RetrieveChat/index.mdx +++ b/website/blog/2023-10-18-RetrieveChat/index.mdx @@ -52,9 +52,9 @@ The conversation terminates if no more documents are available for the context. ## Basic Usage of RAG Agents 0. Install dependencies -Please install pyautogen with the [retrievechat] option before using RAG agents. +Please install autogen-agentchat with the [retrievechat] option before using RAG agents. ```bash -pip install "pyautogen[retrievechat]" +pip install "autogen-agentchat[retrievechat]~=0.2" ``` *You'll need to install `chromadb<=0.5.0` if you see issue like [#3551](https://github.com/microsoft/autogen/issues/3551).* @@ -206,7 +206,7 @@ We are using chromadb as the default vector database, you can also use mongodb, by simply set `vector_db` to `mongodb`, `pgvector`, `qdrant` and `couchbase` in `retrieve_config`, respectively. To plugin any other dbs, you can also extend class `agentchat.contrib.vectordb.base`, -check out the code [here](https://github.com/microsoft/autogen/blob/main/autogen/agentchat/contrib/vectordb/base.py). +check out the code [here](https://github.com/microsoft/autogen/blob/0.2/autogen/agentchat/contrib/vectordb/base.py). ## Advanced Usage of RAG Agents @@ -401,8 +401,9 @@ The online app and the source code are hosted in [HuggingFace](https://huggingfa ## Read More You can check out more example notebooks for RAG use cases: -- [Automated Code Generation and Question Answering with Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat.ipynb) -- [Group Chat with Retrieval Augmented Generation (with 5 group member agents and 1 manager agent)](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_groupchat_RAG.ipynb) -- [Using RetrieveChat with Qdrant for Retrieve Augmented Code Generation and Question Answering](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_qdrant.ipynb) -- [Using RetrieveChat Powered by PGVector for Retrieve Augmented Code Generation and Question Answering](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_pgvector.ipynb) -- [Using RetrieveChat Powered by MongoDB Atlas for Retrieve Augmented Code Generation and Question Answering](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_mongodb.ipynb) +- [Automated Code Generation and Question Answering with Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat.ipynb) +- [Group Chat with Retrieval Augmented Generation (with 5 group member agents and 1 manager agent)](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_groupchat_RAG.ipynb) +- [Using RetrieveChat with Qdrant for Retrieve Augmented Code Generation and Question Answering](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat_qdrant.ipynb) +- [Using RetrieveChat Powered by PGVector for Retrieve Augmented Code Generation and Question Answering](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat_pgvector.ipynb) +- [Using RetrieveChat Powered by MongoDB Atlas for Retrieve Augmented Code Generation and Question Answering](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat_mongodb.ipynb) +- [Using RetrieveChat Powered by Couchbase for Retrieve Augmented Code Generation and Question Answering](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat_couchbase.ipynb) diff --git a/website/blog/2023-10-26-TeachableAgent/index.mdx b/website/blog/2023-10-26-TeachableAgent/index.mdx index ca3992489548..9ac57fa9995f 100644 --- a/website/blog/2023-10-26-TeachableAgent/index.mdx +++ b/website/blog/2023-10-26-TeachableAgent/index.mdx @@ -24,22 +24,22 @@ In order to make effective decisions about memo storage and retrieval, the `Teac AutoGen contains four code examples that use `Teachability`. -1. Run [chat_with_teachable_agent.py](https://github.com/microsoft/autogen/blob/main/test/agentchat/contrib/capabilities/chat_with_teachable_agent.py) to converse with a teachable agent. +1. Run [chat_with_teachable_agent.py](https://github.com/microsoft/autogen/blob/0.2/test/agentchat/contrib/capabilities/chat_with_teachable_agent.py) to converse with a teachable agent. -2. Run [test_teachable_agent.py](https://github.com/microsoft/autogen/blob/main/test/agentchat/contrib/capabilities/test_teachable_agent.py) for quick unit testing of a teachable agent. +2. Run [test_teachable_agent.py](https://github.com/microsoft/autogen/blob/0.2/test/agentchat/contrib/capabilities/test_teachable_agent.py) for quick unit testing of a teachable agent. -3. Use the Jupyter notebook [agentchat_teachability.ipynb](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_teachability.ipynb) to step through examples discussed below. +3. Use the Jupyter notebook [agentchat_teachability.ipynb](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_teachability.ipynb) to step through examples discussed below. -4. Use the Jupyter notebook [agentchat_teachable_oai_assistants.ipynb](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_teachable_oai_assistants.ipynb) to make arbitrary OpenAI Assistants teachable through `GPTAssistantAgent`. +4. Use the Jupyter notebook [agentchat_teachable_oai_assistants.ipynb](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_teachable_oai_assistants.ipynb) to make arbitrary OpenAI Assistants teachable through `GPTAssistantAgent`. ## Basic Usage of Teachability 1. Install dependencies -Please install pyautogen with the [teachable] option before using `Teachability`. +Please install autogen-agentchat~=0.2 with the [teachable] option before using `Teachability`. ```bash -pip install "pyautogen[teachable]" +pip install "autogen-agentchat[teachable]~=0.2" ``` 2. Import agents diff --git a/website/blog/2023-11-06-LMM-Agent/index.mdx b/website/blog/2023-11-06-LMM-Agent/index.mdx index 452079f1c451..6613f794ee3d 100644 --- a/website/blog/2023-11-06-LMM-Agent/index.mdx +++ b/website/blog/2023-11-06-LMM-Agent/index.mdx @@ -9,8 +9,8 @@ tags: [LMM, multimodal] **In Brief:** * Introducing the **Multimodal Conversable Agent** and the **LLaVA Agent** to enhance LMM functionalities. * Users can input text and images simultaneously using the `` tag to specify image loading. -* Demonstrated through the [GPT-4V notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_gpt-4v.ipynb). -* Demonstrated through the [LLaVA notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_llava.ipynb). +* Demonstrated through the [GPT-4V notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_lmm_gpt-4v.ipynb). +* Demonstrated through the [LLaVA notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_lmm_llava.ipynb). ## Introduction Large multimodal models (LMMs) augment large language models (LLMs) with the ability to process multi-sensory data. @@ -25,7 +25,7 @@ GPT-4V represents the forefront in image comprehension, while LLaVA is an effici Incorporate the `lmm` feature during AutoGen installation: ```bash -pip install "pyautogen[lmm]" +pip install "autogen-agentchat[lmm]~=0.2" ``` Subsequently, import the **Multimodal Conversable Agent** or **LLaVA Agent** from AutoGen: @@ -62,7 +62,7 @@ The `MultimodalConversableAgent` interprets the input prompt, extracting images ## Advanced Usage Similar to other AutoGen agents, multimodal agents support multi-round dialogues with other agents, code generation, factual queries, and management via a GroupChat interface. -For example, the `FigureCreator` in our [GPT-4V notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_gpt-4v.ipynb) and [LLaVA notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_llava.ipynb) integrates two agents: a coder (an AssistantAgent) and critics (a multimodal agent). +For example, the `FigureCreator` in our [GPT-4V notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_lmm_gpt-4v.ipynb) and [LLaVA notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_lmm_llava.ipynb) integrates two agents: a coder (an AssistantAgent) and critics (a multimodal agent). The coder drafts Python code for visualizations, while the critics provide insights for enhancement. Collaboratively, these agents aim to refine visual outputs. With `human_input_mode=ALWAYS`, you can also contribute suggestions for better visualizations. diff --git a/website/blog/2023-11-13-OAI-assistants/index.mdx b/website/blog/2023-11-13-OAI-assistants/index.mdx index 07216a25969c..ae8073285a43 100644 --- a/website/blog/2023-11-13-OAI-assistants/index.mdx +++ b/website/blog/2023-11-13-OAI-assistants/index.mdx @@ -9,12 +9,12 @@ tags: [openai-assistant] ## TL;DR -OpenAI assistants are now integrated into AutoGen via [`GPTAssistantAgent`](https://github.com/microsoft/autogen/blob/main/autogen/agentchat/contrib/gpt_assistant_agent.py). +OpenAI assistants are now integrated into AutoGen via [`GPTAssistantAgent`](https://github.com/microsoft/autogen/blob/0.2/autogen/agentchat/contrib/gpt_assistant_agent.py). This enables multiple OpenAI assistants, which form the backend of the now popular GPTs, to collaborate and tackle complex tasks. Checkout example notebooks for reference: -* [Basic example](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_twoagents_basic.ipynb) -* [Code interpreter](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_code_interpreter.ipynb) -* [Function calls](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_function_call.ipynb) +* [Basic example](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_oai_assistant_twoagents_basic.ipynb) +* [Code interpreter](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_oai_code_interpreter.ipynb) +* [Function calls](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_oai_assistant_function_call.ipynb) ## Introduction @@ -30,7 +30,7 @@ This integration shows great potential and synergy, and we plan to continue enha ## Installation ```bash -pip install pyautogen==0.2.0b5 +pip install autogen-agentchat~=0.2 ``` ## Basic Example diff --git a/website/blog/2023-11-20-AgentEval/index.mdx b/website/blog/2023-11-20-AgentEval/index.mdx index 1abb9e6c9f8d..ac7fab7dc8f7 100644 --- a/website/blog/2023-11-20-AgentEval/index.mdx +++ b/website/blog/2023-11-20-AgentEval/index.mdx @@ -14,7 +14,7 @@ tags: [LLM, GPT, evaluation, task utility] **TL;DR:** * As a developer of an LLM-powered application, how can you assess the utility it brings to end users while helping them with their tasks? * To shed light on the question above, we introduce `AgentEval` — the first version of the framework to assess the utility of any LLM-powered application crafted to assist users in specific tasks. AgentEval aims to simplify the evaluation process by automatically proposing a set of criteria tailored to the unique purpose of your application. This allows for a comprehensive assessment, quantifying the utility of your application against the suggested criteria. -* We demonstrate how `AgentEval` work using [math problems dataset](https://microsoft.github.io/autogen/blog/2023/06/28/MathChat) as an example in the [following notebook](https://github.com/microsoft/autogen/blob/main/notebook/agenteval_cq_math.ipynb). Any feedback would be useful for future development. Please contact us on our [Discord](http://aka.ms/autogen-dc). +* We demonstrate how `AgentEval` work using [math problems dataset](https://microsoft.github.io/autogen/blog/2023/06/28/MathChat) as an example in the [following notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agenteval_cq_math.ipynb). Any feedback would be useful for future development. Please contact us on our [Discord](http://aka.ms/autogen-dc). ## Introduction @@ -54,7 +54,7 @@ critic = autogen.AssistantAgent( ) ``` -Next, the critic is given successful and failed examples of the task execution; then, it is able to return a list of criteria (Fig. 1). For reference, use the [following notebook](https://github.com/microsoft/autogen/blob/main/notebook/agenteval_cq_math.ipynb). +Next, the critic is given successful and failed examples of the task execution; then, it is able to return a list of criteria (Fig. 1). For reference, use the [following notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agenteval_cq_math.ipynb). * The goal of `QuantifierAgent` is to quantify each of the suggested criteria (Fig. 1), providing us with an idea of the utility of this system for the given task. Here is an example of how it can be defined: diff --git a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx index be71662ab6ed..c7329a4b1920 100644 --- a/website/blog/2023-11-26-Agent-AutoBuild/index.mdx +++ b/website/blog/2023-11-26-Agent-AutoBuild/index.mdx @@ -14,8 +14,8 @@ user prompt required, powered by a new designed class **AgentBuilder**. AgentBui leveraging [vLLM](https://docs.vllm.ai/en/latest/index.html) and [FastChat](https://github.com/lm-sys/FastChat). Checkout example notebooks and source code for reference: -- [AutoBuild Examples](https://github.com/microsoft/autogen/blob/main/notebook/autobuild_basic.ipynb) -- [AgentBuilder](https://github.com/microsoft/autogen/blob/main/autogen/agentchat/contrib/agent_builder.py) +- [AutoBuild Examples](https://github.com/microsoft/autogen/blob/0.2/notebook/autobuild_basic.ipynb) +- [AgentBuilder](https://github.com/microsoft/autogen/blob/0.2/autogen/agentchat/contrib/agent_builder.py) ## Introduction In this blog, we introduce **AutoBuild**, a pipeline that can automatically build multi-agent systems for complex tasks. @@ -29,7 +29,7 @@ up an endpoint server automatically without any user participation. ## Installation - AutoGen: ```bash -pip install pyautogen[autobuild] +pip install autogen-agentchat[autobuild]~=0.2 ``` - (Optional: if you want to use open-source LLMs) vLLM and FastChat ```bash diff --git a/website/blog/2023-12-01-AutoGenStudio/index.mdx b/website/blog/2023-12-01-AutoGenStudio/index.mdx index a2558acb01d6..b379e4520458 100644 --- a/website/blog/2023-12-01-AutoGenStudio/index.mdx +++ b/website/blog/2023-12-01-AutoGenStudio/index.mdx @@ -139,7 +139,7 @@ This section focuses on defining the properties of agents and agent workflows. I

-**Agents**: This provides an interface to declaratively specify properties for an AutoGen agent (mirrors most of the members of a base [AutoGen conversable agent](https://github.com/microsoft/autogen/blob/main/autogen/agentchat/conversable_agent.py) class). +**Agents**: This provides an interface to declaratively specify properties for an AutoGen agent (mirrors most of the members of a base [AutoGen conversable agent](https://github.com/microsoft/autogen/blob/0.2/autogen/agentchat/conversable_agent.py) class). **Agent Workflows**: An agent workflow is a specification of a set of agents that can work together to accomplish a task. The simplest version of this is a setup with two agents – a user proxy agent (that represents a user i.e. it compiles code and prints result) and an assistant that can address task requests (e.g., generating plans, writing code, evaluating responses, proposing error recovery steps, etc.). A more complex flow could be a group chat where even more agents work towards a solution. diff --git a/website/blog/2023-12-23-AgentOptimizer/index.mdx b/website/blog/2023-12-23-AgentOptimizer/index.mdx index ce92952cfb6f..8c709e931f3d 100644 --- a/website/blog/2023-12-23-AgentOptimizer/index.mdx +++ b/website/blog/2023-12-23-AgentOptimizer/index.mdx @@ -17,7 +17,7 @@ More information could be found in: **Paper**: https://arxiv.org/abs/2402.11359. -**Notebook**: https://github.com/microsoft/autogen/blob/main/notebook/agentchat_agentoptimizer.ipynb. +**Notebook**: https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_agentoptimizer.ipynb. ## Introduction In the traditional ML pipeline, we train a model by updating its weights according to the loss on the training set, while in the era of LLM agents, how should we train an agent? diff --git a/website/blog/2024-01-25-AutoGenBench/index.mdx b/website/blog/2024-01-25-AutoGenBench/index.mdx index 3f4b2d4f216b..cd58ecee6436 100644 --- a/website/blog/2024-01-25-AutoGenBench/index.mdx +++ b/website/blog/2024-01-25-AutoGenBench/index.mdx @@ -21,8 +21,8 @@ Today we are releasing AutoGenBench - a tool for evaluating AutoGen agents and w AutoGenBench is a standalone command line tool, installable from PyPI, which handles downloading, configuring, running, and reporting supported benchmarks. AutoGenBench works best when run alongside Docker, since it uses Docker to isolate tests from one another. -- See the [AutoGenBench README](https://github.com/microsoft/autogen/blob/main/samples/tools/autogenbench/README.md) for information on installation and running benchmarks. -- See the [AutoGenBench CONTRIBUTING guide](https://github.com/microsoft/autogen/blob/main/samples/tools/autogenbench/CONTRIBUTING.md) for information on developing or contributing benchmark datasets. +- See the [AutoGenBench README](https://github.com/microsoft/autogen/blob/0.2/samples/tools/autogenbench/README.md) for information on installation and running benchmarks. +- See the [AutoGenBench CONTRIBUTING guide](https://github.com/microsoft/autogen/blob/0.2/samples/tools/autogenbench/CONTRIBUTING.md) for information on developing or contributing benchmark datasets. ### Quick Start @@ -125,7 +125,7 @@ Please do not cite these values in academic work without first inspecting and ve From this output we can see the results of the three separate repetitions of each task, and final summary statistics of each run. In this case, the results were generated via GPT-4 (as defined in the OAI_CONFIG_LIST that was provided), and used the `TwoAgents` template. **It is important to remember that AutoGenBench evaluates _specific_ end-to-end configurations of agents (as opposed to evaluating a model or cognitive framework more generally).** -Finally, complete execution traces and logs can be found in the `Results` folder. See the [AutoGenBench README](https://github.com/microsoft/autogen/blob/main/samples/tools/autogenbench/README.md) for more details about command-line options and output formats. Each of these commands also offers extensive in-line help via: +Finally, complete execution traces and logs can be found in the `Results` folder. See the [AutoGenBench README](https://github.com/microsoft/autogen/blob/0.2/samples/tools/autogenbench/README.md) for more details about command-line options and output formats. Each of these commands also offers extensive in-line help via: - `autogenbench --help` - `autogenbench clone --help` @@ -145,4 +145,4 @@ For an up to date tracking of our work items on this project, please see [AutoGe ## Call for Participation -Finally, we want to end this blog post with an open call for contributions. AutoGenBench is still nascent, and has much opportunity for improvement. New benchmarks are constantly being published, and will need to be added. Everyone may have their own distinct set of metrics that they care most about optimizing, and these metrics should be onboarded. To this end, we welcome any and all contributions to this corner of the AutoGen project. If contributing is something that interests you, please see the [contributor’s guide](https://github.com/microsoft/autogen/blob/main/samples/tools/autogenbench/CONTRIBUTING.md) and join our [Discord](https://aka.ms/autogen-dc) discussion in the [#autogenbench](https://discord.com/channels/1153072414184452236/1199851779328847902) channel! +Finally, we want to end this blog post with an open call for contributions. AutoGenBench is still nascent, and has much opportunity for improvement. New benchmarks are constantly being published, and will need to be added. Everyone may have their own distinct set of metrics that they care most about optimizing, and these metrics should be onboarded. To this end, we welcome any and all contributions to this corner of the AutoGen project. If contributing is something that interests you, please see the [contributor’s guide](https://github.com/microsoft/autogen/blob/0.2/samples/tools/autogenbench/CONTRIBUTING.md) and join our [Discord](https://aka.ms/autogen-dc) discussion in the [#autogenbench](https://discord.com/channels/1153072414184452236/1199851779328847902) channel! diff --git a/website/blog/2024-01-26-Custom-Models/index.mdx b/website/blog/2024-01-26-Custom-Models/index.mdx index 81a9ad383d3e..e0d43c2ac0ac 100644 --- a/website/blog/2024-01-26-Custom-Models/index.mdx +++ b/website/blog/2024-01-26-Custom-Models/index.mdx @@ -13,7 +13,7 @@ AutoGen now supports custom models! This feature empowers users to define and lo ## Quickstart -An interactive and easy way to get started is by following the notebook [here](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_custom_model.ipynb) which loads a local model from HuggingFace into AutoGen and uses it for inference, and making changes to the class provided. +An interactive and easy way to get started is by following the notebook [here](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_custom_model.ipynb) which loads a local model from HuggingFace into AutoGen and uses it for inference, and making changes to the class provided. ### Step 1: Create the custom model client class diff --git a/website/blog/2024-03-03-AutoGen-Update/index.mdx b/website/blog/2024-03-03-AutoGen-Update/index.mdx index da036094c6e7..f1128b523680 100644 --- a/website/blog/2024-03-03-AutoGen-Update/index.mdx +++ b/website/blog/2024-03-03-AutoGen-Update/index.mdx @@ -138,7 +138,7 @@ The community has been working hard to address them in several dimensions: We are working on agent-based evaluation tools and benchmarking tools. For example: -- [AgentEval](/blog/2023/11/20/AgentEval). Our [research](https://arxiv.org/abs/2402.09015) finds that LLM agents built with AutoGen can be used to automatically identify evaluation criteria and assess the performance from task descriptions and execution logs. It is demonstrated as a [notebook example](https://github.com/microsoft/autogen/blob/main/notebook/agenteval_cq_math.ipynb). Feedback and help are welcome for building it into the library. +- [AgentEval](/blog/2023/11/20/AgentEval). Our [research](https://arxiv.org/abs/2402.09015) finds that LLM agents built with AutoGen can be used to automatically identify evaluation criteria and assess the performance from task descriptions and execution logs. It is demonstrated as a [notebook example](https://github.com/microsoft/autogen/blob/0.2/notebook/agenteval_cq_math.ipynb). Feedback and help are welcome for building it into the library. - [AutoGenBench](/blog/2024/01/25/AutoGenBench). AutoGenBench is a commandline tool for downloading, configuring, running an agentic benchmark, and reporting results. It is designed to allow repetition, isolation and instrumentation, leveraging the new [runtime logging](/docs/notebooks/agentchat_logging) feature. These tools have been used for improving the AutoGen library as well as applications. For example, the new state-of-the-art performance achieved by a multi-agent solution to the [GAIA](https://huggingface.co/spaces/gaia-benchmark/leaderboard) benchmark has benefited from these evaluation tools. @@ -147,8 +147,8 @@ These tools have been used for improving the AutoGen library as well as applicat We are making rapid progress in further improving the interface to make it even easier to build agent applications. For example: -- [AutoBuild](/blog/2023/11/26/Agent-AutoBuild). AutoBuild is an ongoing area of research to automatically create or select a group of agents for a given task and objective. If successful, it will greatly reduce the effort from users or developers when using the multi-agent technology. It also paves the way for agentic decomposition to handle complex tasks. It is available as an experimental feature and demonstrated in two modes: free-form [creation](https://github.com/microsoft/autogen/blob/main/notebook/autobuild_basic.ipynb) and [selection](https://github.com/microsoft/autogen/blob/main/notebook/autobuild_agent_library.ipynb) from a library. -- [AutoGen Studio](/blog/2023/12/01/AutoGenStudio). AutoGen Studio is a no-code UI for fast experimentation with the multi-agent conversations. It lowers the barrier of entrance to the AutoGen technology. Models, agents, and workflows can all be configured without writing code. And chatting with multiple agents in a playground is immediately available after the configuration. Although only a subset of `pyautogen` features are available in this sample app, it demonstrates a promising experience. It has generated tremendous excitement in the community. +- [AutoBuild](/blog/2023/11/26/Agent-AutoBuild). AutoBuild is an ongoing area of research to automatically create or select a group of agents for a given task and objective. If successful, it will greatly reduce the effort from users or developers when using the multi-agent technology. It also paves the way for agentic decomposition to handle complex tasks. It is available as an experimental feature and demonstrated in two modes: free-form [creation](https://github.com/microsoft/autogen/blob/0.2/notebook/autobuild_basic.ipynb) and [selection](https://github.com/microsoft/autogen/blob/0.2/notebook/autobuild_agent_library.ipynb) from a library. +- [AutoGen Studio](/blog/2023/12/01/AutoGenStudio). AutoGen Studio is a no-code UI for fast experimentation with the multi-agent conversations. It lowers the barrier of entrance to the AutoGen technology. Models, agents, and workflows can all be configured without writing code. And chatting with multiple agents in a playground is immediately available after the configuration. Although only a subset of `autogen-agentchat` features are available in this sample app, it demonstrates a promising experience. It has generated tremendous excitement in the community. - Conversation Programming+. The [AutoGen paper](https://arxiv.org/abs/2308.08155) introduced a key concept of _Conversation Programming_, which can be used to program diverse conversation patterns such as 1-1 chat, group chat, hierarchical chat, nested chat etc. While we offered dynamic group chat as an example of high-level orchestration, it made other patterns relatively less discoverable. Therefore, we have added more convenient conversation programming features which enables easier definition of other types of complex workflow, such as [finite state machine based group chat](/blog/2024/02/11/FSM-GroupChat), [sequential chats](/docs/notebooks/agentchats_sequential_chats), and [nested chats](/docs/notebooks/agentchat_nestedchat). Many users have found them useful in implementing specific patterns, which have been always possible but more obvious with the added features. I will write another blog post for a deep dive. ### Learning/Optimization/Teaching @@ -173,8 +173,8 @@ The extensible design of AutoGen makes it easy to integrate with new technologie The above only covers a subset of new features and roadmap. There are many other interesting new features, integration examples or sample apps: -- new features like stateful code execution, [tool decorators](/docs/Use-Cases/agent_chat#tool-calling), [long context handling](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_capability_long_context_handling.ipynb), [web agents](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_surfer.ipynb). -- integration examples like using [guidance](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_guidance.ipynb) to generate structured response. +- new features like stateful code execution, [tool decorators](/docs/Use-Cases/agent_chat#tool-calling), [long context handling](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_capability_long_context_handling.ipynb), [web agents](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_surfer.ipynb). +- integration examples like using [guidance](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_guidance.ipynb) to generate structured response. - sample apps like [AutoAnny](/blog/2024/02/02/AutoAnny). ## Call for Help diff --git a/website/blog/2024-06-24-AltModels-Classes/index.mdx b/website/blog/2024-06-24-AltModels-Classes/index.mdx index 1f01fb9402a4..7001e74e83f0 100644 --- a/website/blog/2024-06-24-AltModels-Classes/index.mdx +++ b/website/blog/2024-06-24-AltModels-Classes/index.mdx @@ -72,10 +72,10 @@ Now it's time to try them out. Install the appropriate client based on the model you wish to use. ```sh -pip install pyautogen["mistral"] # for Mistral AI client -pip install pyautogen["anthropic"] # for Anthropic client -pip install pyautogen["together"] # for Together.AI client -pip install pyautogen["groq"] # for Groq client +pip install autogen-agentchat["mistral"]~=0.2 # for Mistral AI client +pip install autogen-agentchat["anthropic"]~=0.2 # for Anthropic client +pip install autogen-agentchat["together"]~=0.2 # for Together.AI client +pip install autogen-agentchat["groq"]~=0.2 # for Groq client ``` ### Configuration Setup diff --git a/website/blog/2024-10-02-new-autogen-architecture-preview/img/robots.jpeg b/website/blog/2024-10-02-new-autogen-architecture-preview/img/robots.jpeg new file mode 100644 index 000000000000..5ec1aba78444 --- /dev/null +++ b/website/blog/2024-10-02-new-autogen-architecture-preview/img/robots.jpeg @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:daf14746d10ed67ab9de1f50c241827342e523fdb5bd08af8dc5a0fee5a8d37e +size 52503 diff --git a/website/blog/2024-10-02-new-autogen-architecture-preview/index.mdx b/website/blog/2024-10-02-new-autogen-architecture-preview/index.mdx new file mode 100644 index 000000000000..42a873d38bb2 --- /dev/null +++ b/website/blog/2024-10-02-new-autogen-architecture-preview/index.mdx @@ -0,0 +1,103 @@ +--- +title: New AutoGen Architecture Preview +authors: + - autogen-team +tags: [AutoGen] +--- + +# New AutoGen Architecture Preview + +
+ +![What are they doing?](img/robots.jpeg) + +
+ +One year ago, we launched AutoGen, a programming framework designed to build +agentic AI systems. The release of AutoGen sparked massive interest within the +developer community. As an early release, it provided us with a unique +opportunity to engage deeply with users, gather invaluable feedback, and learn +from a diverse range of use cases and contributions. By listening and engaging +with the community, we gained insights into what people were building or +attempting to build, how they were approaching the creation of agentic systems, +and where they were struggling. This experience was both humbling and +enlightening, revealing significant opportunities for improvement in our initial +design, especially for power users developing production-level applications with +AutoGen. + +Through engagements with the community, we learned many lessons: + +- Developers value modular and reusable agents. For example, our built-in agents + that could be directly plugged in or easily customized for specific use cases + were particularly popular. At the same time, there was a desire for more + customizability, such as integrating custom agents built using other + programming languages or frameworks. +- Chat-based agent-to-agent communication was an intuitive collaboration + pattern, making it easy for developers to get started and involve humans in + the loop. As developers began to employ agents in a wider range of scenarios, + they sought more flexibility in collaboration patterns. For instance, + developers wanted to build predictable, ordered workflows with agents, and to + integrate them with new user interfaces that are not chat-based. +- Although it was easy for developers to get started with AutoGen, debugging and + scaling agent teams applications proved more challenging. +- There were many opportunities for improving code quality. + +These learnings, along with many others from other agentic efforts across +Microsoft, prompted us to take a step back and lay the groundwork for a new +direction. A few months ago, we started dedicating time to distilling these +learnings into a roadmap for the future of AutoGen. This led to the development +of AutoGen 0.4, a complete redesign of the framework from the foundation up. +AutoGen 0.4 embraces the actor model of computing to support distributed, highly +scalable, event-driven agentic systems. This approach offers many advantages, +such as: + +- **Composability**. Systems designed in this way are more composable, allowing + developers to bring their own agents implemented in different frameworks or + programming languages and to build more powerful systems using complex agentic + patterns. +- **Flexibility**. It allows for the creation of both deterministic, ordered + workflows and event-driven or decentralized workflows, enabling customers to + bring their own orchestration or integrate with other systems more easily. It + also opens more opportunities for human-in-the-loop scenarios, both active and + reactive. +- **Debugging and Observability**. Event-driven communication moves message delivery + away from agents to a centralized component, making it easier to observe and + debug their activities regardless of agent implementation. +- **Scalability**. An event-based architecture enables distributed and + cloud-deployed agents, which is essential for building scalable AI services + and applications. + +Today, we are delighted to share our progress and invite everyone to collaborate +with us and provide feedback to evolve AutoGen and help shape the future of +multi-agent systems. + +As the first step, we are opening a [pull request](https://github.com/microsoft/autogen/pull/3600) into the main branch with the +current state of development of 0.4. After approximately a week, we plan to +merge this into main and continue development. There's still a lot left to do +before 0.4 is ready for release though, so keep in mind this is a work in +progress. + +Starting in AutoGen 0.4, the project will have three main libraries: + +- **Core** - the building blocks for an event-driven agentic system. +- **AgentChat** - a task-driven, high-level API built with core, including group + chat, code execution, pre-built agents, and more. This is the most similar API + to AutoGen [0.2](https://github.com/microsoft/autogen/tree/0.2) and will be the easiest API to migrate to. +- **Extensions** - implementations of core interfaces and third-party integrations + (e.g., Azure code executor and OpenAI model client). + +AutoGen [0.2](https://github.com/microsoft/autogen/tree/0.2) is still available, developed and maintained out of the [0.2 branch](https://github.com/microsoft/autogen/tree/0.2). +For everyone looking for a stable version, we recommend continuing to use [0.2](https://github.com/microsoft/autogen/tree/0.2) +for the time being. It can be installed using: + +```sh +pip install autogen-agentchat~=0.2 +``` + +This new package name was used to align with the new packages that will come with 0.4: +`autogen-core`, `autogen-agentchat`, and `autogen-ext`. + +Lastly, we will be using [GitHub +Discussion](https://github.com/microsoft/autogen/discussions) as the official +community forum for the new version and, going forward, all discussions related +to the AutoGen project. We look forward to meeting you there. diff --git a/website/blog/authors.yml b/website/blog/authors.yml index f9e7495c5f30..107d7d5a9de3 100644 --- a/website/blog/authors.yml +++ b/website/blog/authors.yml @@ -152,3 +152,7 @@ bboynton97: title: AI Engineer at AgentOps url: https://github.com/bboynton97 image_url: https://github.com/bboynton97.png + +autogen-team: + name: AutoGen Team + title: The humans behind the agents diff --git a/website/docs/Examples.md b/website/docs/Examples.md index 5efd71748f9d..df06f97fb124 100644 --- a/website/docs/Examples.md +++ b/website/docs/Examples.md @@ -18,7 +18,7 @@ Links to notebook examples: - Automated Task Solving by Group Chat (with 3 group member agents and 1 manager agent) - [View Notebook](/docs/notebooks/agentchat_groupchat) - Automated Data Visualization by Group Chat (with 3 group member agents and 1 manager agent) - [View Notebook](/docs/notebooks/agentchat_groupchat_vis) - Automated Complex Task Solving by Group Chat (with 6 group member agents and 1 manager agent) - [View Notebook](/docs/notebooks/agentchat_groupchat_research) -- Automated Task Solving with Coding & Planning Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_planning.ipynb) +- Automated Task Solving with Coding & Planning Agents - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_planning.ipynb) - Automated Task Solving with transition paths specified in a graph - [View Notebook](https://microsoft.github.io/autogen/docs/notebooks/agentchat_groupchat_finite_state_machine) - Running a group chat as an inner-monolgue via the SocietyOfMindAgent - [View Notebook](/docs/notebooks/agentchat_society_of_mind) - Running a group chat with custom speaker selection function - [View Notebook](/docs/notebooks/agentchat_groupchat_customized) @@ -38,73 +38,74 @@ Links to notebook examples: ### Applications -- Automated Continual Learning from New Data - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_stream.ipynb) +- Automated Continual Learning from New Data - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_stream.ipynb) - [OptiGuide](https://github.com/microsoft/optiguide) - Coding, Tool Using, Safeguarding & Question Answering for Supply Chain Optimization - [AutoAnny](https://github.com/microsoft/autogen/tree/main/samples/apps/auto-anny) - A Discord bot built using AutoGen ### Tool Use -- **Web Search**: Solve Tasks Requiring Web Info - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_web_info.ipynb) +- **Web Search**: Solve Tasks Requiring Web Info - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_web_info.ipynb) - Use Provided Tools as Functions - [View Notebook](/docs/notebooks/agentchat_function_call_currency_calculator) - Use Tools via Sync and Async Function Calling - [View Notebook](/docs/notebooks/agentchat_function_call_async) -- Task Solving with Langchain Provided Tools as Functions - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_langchain.ipynb) +- Task Solving with Langchain Provided Tools as Functions - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_langchain.ipynb) - **RAG**: Group Chat with Retrieval Augmented Generation (with 5 group member agents and 1 manager agent) - [View Notebook](/docs/notebooks/agentchat_groupchat_RAG) -- Function Inception: Enable AutoGen agents to update/remove functions during conversations. - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_inception_function.ipynb) +- Function Inception: Enable AutoGen agents to update/remove functions during conversations. - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_inception_function.ipynb) - Agent Chat with Whisper - [View Notebook](/docs/notebooks/agentchat_video_transcript_translate_with_whisper) -- Constrained Responses via Guidance - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_guidance.ipynb) -- Browse the Web with Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_surfer.ipynb) -- **SQL**: Natural Language Text to SQL Query using the [Spider](https://yale-lily.github.io/spider) Text-to-SQL Benchmark - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_sql_spider.ipynb) +- Constrained Responses via Guidance - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_guidance.ipynb) +- Browse the Web with Agents - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_surfer.ipynb) +- **SQL**: Natural Language Text to SQL Query using the [Spider](https://yale-lily.github.io/spider) Text-to-SQL Benchmark - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_sql_spider.ipynb) - **Web Scraping**: Web Scraping with Apify - [View Notebook](/docs/notebooks/agentchat_webscraping_with_apify) -- **Write a software app, task by task, with specially designed functions.** - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_function_call_code_writing.ipynb). +- **Web Crawling**: Crawl entire domain with Spider API - [View Notebook](/docs/notebooks/agentchat_webcrawling_with_spider) +- **Write a software app, task by task, with specially designed functions.** - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_function_call_code_writing.ipynb). ### Human Involvement -- Simple example in ChatGPT style [View example](https://github.com/microsoft/autogen/blob/main/samples/simple_chat.py) -- Auto Code Generation, Execution, Debugging and **Human Feedback** - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_human_feedback.ipynb) -- Automated Task Solving with GPT-4 + **Multiple Human Users** - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_two_users.ipynb) -- Agent Chat with **Async Human Inputs** - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/Async_human_input.ipynb) +- Simple example in ChatGPT style [View example](https://github.com/microsoft/autogen/blob/0.2/samples/simple_chat.py) +- Auto Code Generation, Execution, Debugging and **Human Feedback** - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_human_feedback.ipynb) +- Automated Task Solving with GPT-4 + **Multiple Human Users** - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_two_users.ipynb) +- Agent Chat with **Async Human Inputs** - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/Async_human_input.ipynb) ### Agent Teaching and Learning - Teach Agents New Skills & Reuse via Automated Chat - [View Notebook](/docs/notebooks/agentchat_teaching) - Teach Agents New Facts, User Preferences and Skills Beyond Coding - [View Notebook](/docs/notebooks/agentchat_teachability) -- Teach OpenAI Assistants Through GPTAssistantAgent - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_teachable_oai_assistants.ipynb) -- Agent Optimizer: Train Agents in an Agentic Way - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_agentoptimizer.ipynb) +- Teach OpenAI Assistants Through GPTAssistantAgent - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_teachable_oai_assistants.ipynb) +- Agent Optimizer: Train Agents in an Agentic Way - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_agentoptimizer.ipynb) ### Multi-Agent Chat with OpenAI Assistants in the loop -- Hello-World Chat with OpenAi Assistant in AutoGen - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_twoagents_basic.ipynb) -- Chat with OpenAI Assistant using Function Call - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_function_call.ipynb) -- Chat with OpenAI Assistant with Code Interpreter - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_code_interpreter.ipynb) -- Chat with OpenAI Assistant with Retrieval Augmentation - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_retrieval.ipynb) -- OpenAI Assistant in a Group Chat - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_groupchat.ipynb) -- GPTAssistantAgent based Multi-Agent Tool Use - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/gpt_assistant_agent_function_call.ipynb) +- Hello-World Chat with OpenAi Assistant in AutoGen - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_oai_assistant_twoagents_basic.ipynb) +- Chat with OpenAI Assistant using Function Call - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_oai_assistant_function_call.ipynb) +- Chat with OpenAI Assistant with Code Interpreter - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_oai_code_interpreter.ipynb) +- Chat with OpenAI Assistant with Retrieval Augmentation - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_oai_assistant_retrieval.ipynb) +- OpenAI Assistant in a Group Chat - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_oai_assistant_groupchat.ipynb) +- GPTAssistantAgent based Multi-Agent Tool Use - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/gpt_assistant_agent_function_call.ipynb) ### Non-OpenAI Models - Conversational Chess using non-OpenAI Models - [View Notebook](/docs/notebooks/agentchat_nested_chats_chess_altmodels) ### Multimodal Agent -- Multimodal Agent Chat with DALLE and GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_dalle_and_gpt4v.ipynb) -- Multimodal Agent Chat with Llava - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_llava.ipynb) -- Multimodal Agent Chat with GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_gpt-4v.ipynb) +- Multimodal Agent Chat with DALLE and GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_dalle_and_gpt4v.ipynb) +- Multimodal Agent Chat with Llava - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_lmm_llava.ipynb) +- Multimodal Agent Chat with GPT-4V - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_lmm_gpt-4v.ipynb) ### Long Context Handling - + - Long Context Handling as A Capability - [View Notebook](/docs/notebooks/agentchat_transform_messages) ### Evaluation and Assessment -- AgentEval: A Multi-Agent System for Assess Utility of LLM-powered Applications - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agenteval_cq_math.ipynb) +- AgentEval: A Multi-Agent System for Assess Utility of LLM-powered Applications - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agenteval_cq_math.ipynb) ### Automatic Agent Building -- Automatically Build Multi-agent System with AgentBuilder - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/autobuild_basic.ipynb) -- Automatically Build Multi-agent System from Agent Library - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/autobuild_agent_library.ipynb) +- Automatically Build Multi-agent System with AgentBuilder - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/autobuild_basic.ipynb) +- Automatically Build Multi-agent System from Agent Library - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/autobuild_agent_library.ipynb) ### Observability -- Track LLM calls, tool usage, actions and errors using AgentOps - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_agentops.ipynb) +- Track LLM calls, tool usage, actions and errors using AgentOps - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_agentops.ipynb) ## Enhanced Inferences @@ -112,7 +113,7 @@ Links to notebook examples: - API Unification - [View Documentation with Code Example](https://microsoft.github.io/autogen/docs/Use-Cases/enhanced_inference/#api-unification) - Utility Functions to Help Managing API configurations effectively - [View Notebook](/docs/topics/llm_configuration) -- Cost Calculation - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_cost_token_tracking.ipynb) +- Cost Calculation - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_cost_token_tracking.ipynb) ### Inference Hyperparameters Tuning @@ -120,5 +121,5 @@ AutoGen offers a cost-effective hyperparameter optimization technique [EcoOptiGe Please find documentation about this feature [here](/docs/Use-Cases/enhanced_inference). Links to notebook examples: -* [Optimize for Code Generation](https://github.com/microsoft/autogen/blob/main/notebook/oai_completion.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/autogen/blob/main/notebook/oai_completion.ipynb) -* [Optimize for Math](https://github.com/microsoft/autogen/blob/main/notebook/oai_chatgpt_gpt4.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/autogen/blob/main/notebook/oai_chatgpt_gpt4.ipynb) +* [Optimize for Code Generation](https://github.com/microsoft/autogen/blob/0.2/notebook/oai_completion.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/autogen/blob/main/notebook/oai_completion.ipynb) +* [Optimize for Math](https://github.com/microsoft/autogen/blob/0.2/notebook/oai_chatgpt_gpt4.ipynb) | [Open in colab](https://colab.research.google.com/github/microsoft/autogen/blob/main/notebook/oai_chatgpt_gpt4.ipynb) diff --git a/website/docs/FAQ.mdx b/website/docs/FAQ.mdx index a367a9b20635..14be83c7baba 100644 --- a/website/docs/FAQ.mdx +++ b/website/docs/FAQ.mdx @@ -4,12 +4,12 @@ import TOCInline from "@theme/TOCInline"; -## Install the correct package - `pyautogen` +## Install the correct package - `autogen-agentchat` -The name of Autogen package at PyPI is `pyautogen`: +The name of Autogen package at PyPI is `autogen-agentchat`: ``` -pip install pyautogen +pip install autogen-agentchat~=0.2 ``` Typical errors that you might face when using the wrong package are `AttributeError: module 'autogen' has no attribute 'Agent'`, `AttributeError: module 'autogen' has no attribute 'config_list_from_json'` etc. diff --git a/website/docs/Getting-Started.mdx b/website/docs/Getting-Started.mdx index 3e162a098327..761ed6f3d34d 100644 --- a/website/docs/Getting-Started.mdx +++ b/website/docs/Getting-Started.mdx @@ -35,7 +35,7 @@ Microsoft, Penn State University, and University of Washington. ### Quickstart ```sh -pip install pyautogen +pip install autogen-agentchat~=0.2 ``` @@ -118,7 +118,7 @@ Learn more about configuring LLMs for agents [here](/docs/topics/llm_configurati #### Multi-Agent Conversation Framework Autogen enables the next-gen LLM applications with a generic multi-agent conversation framework. It offers customizable and conversable agents which integrate LLMs, tools, and humans. -By automating chat among multiple capable agents, one can easily make them collectively perform tasks autonomously or with human feedback, including tasks that require using tools via code. For [example](https://github.com/microsoft/autogen/blob/main/test/twoagent.py), +By automating chat among multiple capable agents, one can easily make them collectively perform tasks autonomously or with human feedback, including tasks that require using tools via code. For [example](https://github.com/microsoft/autogen/blob/0.2/test/twoagent.py), The figure below shows an example conversation flow with AutoGen. @@ -131,7 +131,6 @@ The figure below shows an example conversation flow with AutoGen. - Understand the use cases for [multi-agent conversation](/docs/Use-Cases/agent_chat) and [enhanced LLM inference](/docs/Use-Cases/enhanced_inference) - Read the [API](/docs/reference/agentchat/conversable_agent/) docs - Learn about [research](/docs/Research) around AutoGen -- Chat on [Discord](https://aka.ms/autogen-dc) - Follow on [Twitter](https://twitter.com/pyautogen) - See our [roadmaps](https://aka.ms/autogen-roadmap) diff --git a/website/docs/Migration-Guide.md b/website/docs/Migration-Guide.md index f996a157a299..97e16de0770b 100644 --- a/website/docs/Migration-Guide.md +++ b/website/docs/Migration-Guide.md @@ -26,7 +26,7 @@ autogen.runtime_logging.start() # Stop logging autogen.runtime_logging.stop() ``` -Checkout [Logging documentation](https://microsoft.github.io/autogen/docs/Use-Cases/enhanced_inference#logging) and [Logging example notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_logging.ipynb) to learn more. +Checkout [Logging documentation](https://microsoft.github.io/autogen/docs/Use-Cases/enhanced_inference#logging) and [Logging example notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_logging.ipynb) to learn more. Inference parameter tuning can be done via [`flaml.tune`](https://microsoft.github.io/FLAML/docs/Use-Cases/Tune-User-Defined-Function). - `seed` in autogen is renamed into `cache_seed` to accommodate the newly added `seed` param in openai chat completion api. `use_cache` is removed as a kwarg in `OpenAIWrapper.create()` for being automatically decided by `cache_seed`: int | None. The difference between autogen's `cache_seed` and openai's `seed` is that: diff --git a/website/docs/Use-Cases/agent_chat.md b/website/docs/Use-Cases/agent_chat.md index 76f9959841b8..fa85691810c9 100644 --- a/website/docs/Use-Cases/agent_chat.md +++ b/website/docs/Use-Cases/agent_chat.md @@ -81,13 +81,13 @@ AutoGen, by integrating conversation-driven control utilizing both programming a With the pluggable auto-reply function, one can choose to invoke conversations with other agents depending on the content of the current message and context. For example: - Hierarchical chat like in [OptiGuide](https://github.com/microsoft/optiguide). -- [Dynamic Group Chat](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_groupchat.ipynb) which is a special form of hierarchical chat. In the system, we register a reply function in the group chat manager, which broadcasts messages and decides who the next speaker will be in a group chat setting. +- [Dynamic Group Chat](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_groupchat.ipynb) which is a special form of hierarchical chat. In the system, we register a reply function in the group chat manager, which broadcasts messages and decides who the next speaker will be in a group chat setting. - [Finite State Machine graphs to set speaker transition constraints](https://microsoft.github.io/autogen/docs/notebooks/agentchat_groupchat_finite_state_machine) which is a special form of dynamic group chat. In this approach, a directed transition matrix is fed into group chat. Users can specify legal transitions or specify disallowed transitions. -- Nested chat like in [conversational chess](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_nested_chats_chess.ipynb). +- Nested chat like in [conversational chess](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_nested_chats_chess.ipynb). 2. LLM-Based Function Call -Another approach involves LLM-based function calls, where LLM decides if a specific function should be invoked based on the conversation's status during each inference. This approach enables dynamic multi-agent conversations, as seen in scenarios like [multi-user math problem solving scenario](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_two_users.ipynb), where a student assistant automatically seeks expertise via function calls. +Another approach involves LLM-based function calls, where LLM decides if a specific function should be invoked based on the conversation's status during each inference. This approach enables dynamic multi-agent conversations, as seen in scenarios like [multi-user math problem solving scenario](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_two_users.ipynb), where a student assistant automatically seeks expertise via function calls. ### Diverse Applications Implemented with AutoGen diff --git a/website/docs/Use-Cases/enhanced_inference.md b/website/docs/Use-Cases/enhanced_inference.md index 14723391e8cf..825f46ff0714 100644 --- a/website/docs/Use-Cases/enhanced_inference.md +++ b/website/docs/Use-Cases/enhanced_inference.md @@ -140,7 +140,7 @@ The `OpenAIWrapper` from `autogen` tracks token counts and costs of your API cal - `mode='actual'`: only print non-cached usage. - `mode='total'`: only print all usage (including cache). -Reset your session's usage data with `clear_usage_summary()` when needed. [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/oai_client_cost.ipynb) +Reset your session's usage data with `clear_usage_summary()` when needed. [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/oai_client_cost.ipynb) Example usage: ```python @@ -307,7 +307,7 @@ When debugging or diagnosing an LLM-based system, it is often convenient to log ### For openai >= 1 -Logging example: [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_logging.ipynb) +Logging example: [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_logging.ipynb) #### Start logging: ```python diff --git a/website/docs/contributor-guide/contributing.md b/website/docs/contributor-guide/contributing.md index cd2c62e408c1..633cd6c65ab8 100644 --- a/website/docs/contributor-guide/contributing.md +++ b/website/docs/contributor-guide/contributing.md @@ -6,7 +6,7 @@ The project welcomes contributions from developers and organizations worldwide. - Code review of pull requests. - Documentation, examples and test cases. - Readability improvement, e.g., improvement on docstr and comments. -- Community participation in [issues](https://github.com/microsoft/autogen/issues), [discussions](https://github.com/microsoft/autogen/discussions), [discord](https://aka.ms/autogen-dc), and [twitter](https://twitter.com/pyautogen). +- Community participation in [issues](https://github.com/microsoft/autogen/issues), [discussions](https://github.com/microsoft/autogen/discussions), and [twitter](https://twitter.com/pyautogen). - Tutorials, blog posts, talks that promote the project. - Sharing application scenarios and/or related research. @@ -31,4 +31,4 @@ To see what we are working on and what we plan to work on, please check our ## Becoming a Reviewer -There is currently no formal reviewer solicitation process. Current reviewers identify reviewers from active contributors. If you are willing to become a reviewer, you are welcome to let us know on discord. +There is currently no formal reviewer solicitation process. Current reviewers identify reviewers from active contributors. diff --git a/website/docs/contributor-guide/docker.md b/website/docs/contributor-guide/docker.md index 8ea54ed7c228..b09a53aa35f2 100644 --- a/website/docs/contributor-guide/docker.md +++ b/website/docs/contributor-guide/docker.md @@ -48,4 +48,4 @@ docker run -it -p 8081:3000 -v /home/AutoGenDeveloper/autogen-newcode:newstuff/ ## Develop in Remote Container If you use vscode, you can open the autogen folder in a [Container](https://code.visualstudio.com/docs/remote/containers). -We have provided the configuration in [devcontainer](https://github.com/microsoft/autogen/blob/main/.devcontainer). They can be used in GitHub codespace too. Developing AutoGen in dev containers is recommended. +We have provided the configuration in [devcontainer](https://github.com/microsoft/autogen/blob/0.2/.devcontainer). They can be used in GitHub codespace too. Developing AutoGen in dev containers is recommended. diff --git a/website/docs/contributor-guide/documentation.md b/website/docs/contributor-guide/documentation.md index 70a79b700697..3380ff529280 100644 --- a/website/docs/contributor-guide/documentation.md +++ b/website/docs/contributor-guide/documentation.md @@ -2,7 +2,7 @@ ## How to get a notebook rendered on the website -See [here](https://github.com/microsoft/autogen/blob/main/notebook/contributing.md#how-to-get-a-notebook-displayed-on-the-website) for instructions on how to get a notebook in the `notebook` directory rendered on the website. +See [here](https://github.com/microsoft/autogen/blob/0.2/notebook/contributing.md#how-to-get-a-notebook-displayed-on-the-website) for instructions on how to get a notebook in the `notebook` directory rendered on the website. ## Build documentation locally diff --git a/website/docs/contributor-guide/maintainer.md b/website/docs/contributor-guide/maintainer.md index cdbe4da53a93..dd28d1926882 100644 --- a/website/docs/contributor-guide/maintainer.md +++ b/website/docs/contributor-guide/maintainer.md @@ -10,7 +10,7 @@ ## Pull Requests -- For new PR, decide whether to close without review. If not, find the right reviewers. One source to refer to is the roles on Discord. Another consideration is to ask users who can benefit from the PR to review it. +- For new PR, decide whether to close without review. If not, find the right reviewers. Another consideration is to ask users who can benefit from the PR to review it. - For old PR, check the blocker: reviewer or PR creator. Try to unblock. Get additional help when needed. - When requesting changes, make sure you can check back in time because it blocks merging. @@ -28,9 +28,9 @@ ## Issues and Discussions -- For new issues, write a reply, apply a label if relevant. Ask on discord when necessary. For roadmap issues, apply the roadmap label and encourage community discussion. Mention relevant experts when necessary. +- For new issues, write a reply, apply a label if relevant. For roadmap issues, apply the roadmap label and encourage community discussion. Mention relevant experts when necessary. -- For old issues, provide an update or close. Ask on discord when necessary. Encourage PR creation when relevant. +- For old issues, provide an update or close. Encourage PR creation when relevant. - Use “good first issue” for easy fix suitable for first-time contributors. - Use “task list” for issues that require multiple PRs. -- For discussions, create an issue when relevant. Discuss on discord when appropriate. +- For discussions, create an issue when relevant. diff --git a/website/docs/contributor-guide/tests.md b/website/docs/contributor-guide/tests.md index c5eabb907329..a4b03b6379d3 100644 --- a/website/docs/contributor-guide/tests.md +++ b/website/docs/contributor-guide/tests.md @@ -2,12 +2,12 @@ Tests are automatically run via GitHub actions. There are two workflows: -1. [build.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/build.yml) -1. [openai.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/openai.yml) +1. [build.yml](https://github.com/microsoft/autogen/blob/0.2/.github/workflows/build.yml) +1. [openai.yml](https://github.com/microsoft/autogen/blob/0.2/.github/workflows/openai.yml) -The first workflow is required to pass for all PRs (and it doesn't do any OpenAI calls). The second workflow is required for changes that affect the OpenAI tests (and does actually call LLM). The second workflow requires approval to run. When writing tests that require OpenAI calls, please use [`pytest.mark.skipif`](https://github.com/microsoft/autogen/blob/b1adac515931bf236ac59224269eeec683a162ba/test/oai/test_client.py#L19) to make them run in only when `openai` package is installed. If additional dependency for this test is required, install the dependency in the corresponding python version in [openai.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/openai.yml). +The first workflow is required to pass for all PRs (and it doesn't do any OpenAI calls). The second workflow is required for changes that affect the OpenAI tests (and does actually call LLM). The second workflow requires approval to run. When writing tests that require OpenAI calls, please use [`pytest.mark.skipif`](https://github.com/microsoft/autogen/blob/b1adac515931bf236ac59224269eeec683a162ba/test/oai/test_client.py#L19) to make them run in only when `openai` package is installed. If additional dependency for this test is required, install the dependency in the corresponding python version in [openai.yml](https://github.com/microsoft/autogen/blob/0.2/.github/workflows/openai.yml). -Make sure all tests pass, this is required for [build.yml](https://github.com/microsoft/autogen/blob/main/.github/workflows/build.yml) checks to pass +Make sure all tests pass, this is required for [build.yml](https://github.com/microsoft/autogen/blob/0.2/.github/workflows/build.yml) checks to pass ## Running tests locally @@ -27,7 +27,7 @@ Tests for the `autogen.agentchat.contrib` module may be skipped automatically if required dependencies are not installed. Please consult the documentation for each contrib module to see what dependencies are required. -See [here](https://github.com/microsoft/autogen/blob/main/notebook/contributing.md#testing) for how to run notebook tests. +See [here](https://github.com/microsoft/autogen/blob/0.2/notebook/contributing.md#testing) for how to run notebook tests. ## Skip flags for tests diff --git a/website/docs/ecosystem/agent-memory-with-zep.md b/website/docs/ecosystem/agent-memory-with-zep.md new file mode 100644 index 000000000000..c4551296dc7e --- /dev/null +++ b/website/docs/ecosystem/agent-memory-with-zep.md @@ -0,0 +1,102 @@ +# Agent Memory with Zep + +Zep logo + +[Zep](https://www.getzep.com/?utm_source=autogen) is a long-term memory service for agentic applications used by both startups and enterprises. With Zep, you can build personalized, accurate, and production-ready agent applications. + +Zep's memory continuously learns facts from interactions with users and your changing business data. With [just two API calls](https://help.getzep.com/memory?utm_source=autogen), you can persist chat history to Zep and recall facts relevant to the state of your agent. + +Zep is powered by a temporal Knowledge Graph that allows reasoning with facts as they change. A combination of semantic and graph search enables accurate and low-latency fact retrieval. + +Sign up for [Zep Cloud](https://www.getzep.com/?utm_source=autogen) or visit the [Zep Community Edition Repo](https://github.com/getzep/zep). + +| Feature | Description | +| ---------------------------------------------- | ------------------------------------------------------------------------------------- | +| 💬 **Capture Detailed Conversational Context** | Zep's Knowledge Graph-based memory captures episodic, semantic, and temporal contexts | +| 🗄️ **Business Data is Context, too** | Zep is able to extract facts from JSON and unstructured text as well | +| ⚙️ **Tailor For Your Business** | Fact Ratings and other tools allow you to fine-tune retrieval for your use case | +| ⚡️ **Instant Memory Retrieval** | Retrieve relevant facts in under 100ms | +| 🔐 **Compliance & Security** | User Privacy Management, SOC 2 Type II certification, and other controls | +| 🖼️ **Framework Agnostic & Future-Proof** | Use with AutoGen or any other framework, current or future | + +
+ Zep Community Edition Walkthrough + + Zep Fact Ratings + +
+ +
+ User Chat Session and Facts + + Chat Session and Facts + +
+ +
+ Implementing Fact Ratings + + Zep Fact Ratings + +
+ +## How Zep works + +1. Add chat messages or data artifacts to Zep during each user interaction or agent event. +2. Zep intelligently integrates new information into the user's (or groups of users) Knowledge Graph, updating existing context as needed. +3. Retrieve relevant facts from Zep for subsequent interactions or events. + +Zep's temporal Knowledge Graph maintains contextual information about facts, enabling reasoning about state changes and providing data provenance insights. Each fact includes `valid_at` and `invalid_at` dates, allowing agents to track changes in user preferences, traits, or environment. + +## Zep is fast + +Retrieving facts is simple and very fast. Unlike other memory solutions, Zep does not use agents to ensure facts are relevant. It precomputes facts, entity summaries, and other artifacts asynchronously. For on-premise use, retrieval speed primarily depends on your embedding service's performance. + +## Zep supports many types of data + +You can add a variety of data artifacts to Zep: + +- Adding chat history messages. +- Ingestion of JSON and unstructured text. + +Zep supports chat session, user, and group-level graphs. Group graphs allow for capturing organizational knowledge. + +## Getting Started + +### Zep Cloud + +1. Sign up for [Zep Cloud](https://www.getzep.com?utm_source=autogen) and create a [Project API Key](https://help.getzep.com/projects?utm_source=autogen). + +2. Install one of the [Zep Python, TypeScript or Go SDKs](https://help.getzep.com/sdks?utm_source=autogen). Python instructions shown below. + +```shell +pip install zep-cloud +``` + +3. Initialize a client + +```python +import os +from zep_cloud.client import AsyncZep + +API_KEY = os.environ.get('ZEP_API_KEY') +client = AsyncZep( + api_key=API_KEY, +) +``` + +3. Review the Zep and Autogen [notebook example](/docs/notebooks/agent_memory_using_zep/) for agent-building best practices. + +### Zep Community Edition + +Follow the [Getting Started guide](https://help.getzep.com/ce/quickstart?utm_source=autogen) or visit the [GitHub Repo](https://github.com/getzep/zep?utm_source=autogen). + +## Autogen + Zep examples + +- [Autogen Agents with Zep Memory Notebook](/docs/notebooks/agent_memory_using_zep/) + +## Extra links + +- [📙 Documentation](https://help.getzep.com/?utm_source=autogen) +- [🐦 Twitter / X](https://x.com/zep_ai/) +- [📢 Discord](https://discord.com/invite/W8Kw6bsgXQ) diff --git a/website/docs/ecosystem/img/ecosystem-zep-ce-walkthrough.png b/website/docs/ecosystem/img/ecosystem-zep-ce-walkthrough.png new file mode 100644 index 000000000000..b6937a043985 --- /dev/null +++ b/website/docs/ecosystem/img/ecosystem-zep-ce-walkthrough.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:c0829b29a48ca05e2694aca00446ef5768c1b8edec56ce5035527f25f9ee4c81 +size 421633 diff --git a/website/docs/ecosystem/img/ecosystem-zep-fact-ratings.png b/website/docs/ecosystem/img/ecosystem-zep-fact-ratings.png new file mode 100644 index 000000000000..a4b12f8a96f3 --- /dev/null +++ b/website/docs/ecosystem/img/ecosystem-zep-fact-ratings.png @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:179241bd4fa3ed89d721deeb1810a31b9838e7f54582d521bd91f29cbae044f2 +size 233905 diff --git a/website/docs/ecosystem/img/ecosystem-zep-session.gif b/website/docs/ecosystem/img/ecosystem-zep-session.gif new file mode 100644 index 000000000000..5ff9eb5dbfd3 Binary files /dev/null and b/website/docs/ecosystem/img/ecosystem-zep-session.gif differ diff --git a/website/docs/ecosystem/llamaindex.md b/website/docs/ecosystem/llamaindex.md index 4e9cffeedc7d..6561880b4d22 100644 --- a/website/docs/ecosystem/llamaindex.md +++ b/website/docs/ecosystem/llamaindex.md @@ -4,4 +4,4 @@ [Llamaindex](https://www.llamaindex.ai/) allows the users to create Llamaindex agents and integrate them in autogen conversation patterns. -- [Llamaindex + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb) +- [Llamaindex + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_group_chat_with_llamaindex_agents.ipynb) diff --git a/website/docs/ecosystem/microsoft-fabric.md b/website/docs/ecosystem/microsoft-fabric.md index d0593a4a7d04..5ef625c71763 100644 --- a/website/docs/ecosystem/microsoft-fabric.md +++ b/website/docs/ecosystem/microsoft-fabric.md @@ -4,4 +4,4 @@ [Microsoft Fabric](https://learn.microsoft.com/en-us/fabric/get-started/microsoft-fabric-overview) is an all-in-one analytics solution for enterprises that covers everything from data movement to data science, Real-Time Analytics, and business intelligence. It offers a comprehensive suite of services, including data lake, data engineering, and data integration, all in one place. In this notenook, we give a simple example for using AutoGen in Microsoft Fabric. -- [Microsoft Fabric + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_microsoft_fabric.ipynb) +- [Microsoft Fabric + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_microsoft_fabric.ipynb) diff --git a/website/docs/ecosystem/pgvector.md b/website/docs/ecosystem/pgvector.md index 99afa676e7e4..637941e6135e 100644 --- a/website/docs/ecosystem/pgvector.md +++ b/website/docs/ecosystem/pgvector.md @@ -2,4 +2,4 @@ [PGVector](https://github.com/pgvector/pgvector) is an open-source vector similarity search for Postgres. -- [PGVector + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_pgvector.ipynb) +- [PGVector + AutoGen Code Examples](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat_pgvector.ipynb) diff --git a/website/docs/ecosystem/portkey.md b/website/docs/ecosystem/portkey.md index 4825cf78d9a7..a9f67d3871fc 100644 --- a/website/docs/ecosystem/portkey.md +++ b/website/docs/ecosystem/portkey.md @@ -13,7 +13,7 @@ Portkey adds 4 core production capabilities to any AutoGen agent: 1. **Install Required Packages:** 2. ```bash - pip install -qU pyautogen portkey-ai + pip install -qU autogen-agentchat~=0.2 portkey-ai ``` **Configure AutoGen with Portkey:** diff --git a/website/docs/installation/Docker.md b/website/docs/installation/Docker.md index a7fa6bd829b5..a4506f9c10ae 100644 --- a/website/docs/installation/Docker.md +++ b/website/docs/installation/Docker.md @@ -41,7 +41,7 @@ Here's how you can run an application built with AutoGen, using the Docker image Here, `$(pwd)/myapp` is your local directory, and `/home/autogen/autogen/myapp` is the path in the Docker container where your code will be located. -2. **Mount your code:** Now suppose you have your application built with AutoGen in a main script named `twoagent.py` ([example](https://github.com/microsoft/autogen/blob/main/test/twoagent.py)) in a folder named `myapp`. With the command line below, you can mount your folder and run the application in Docker. +2. **Mount your code:** Now suppose you have your application built with AutoGen in a main script named `twoagent.py` ([example](https://github.com/microsoft/autogen/blob/0.2/test/twoagent.py)) in a folder named `myapp`. With the command line below, you can mount your folder and run the application in Docker. ```python # Mount the local folder `myapp` into docker image and run the script named "twoagent.py" in the docker. @@ -82,6 +82,6 @@ docker run -it -p {WorkstationPortNum}:{DockerPortNum} -v {WorkStation_Dir}:{Doc ## Additional Resources -- Details on all the Dockerfile options can be found in the [Dockerfile](https://github.com/microsoft/autogen/blob/main/.devcontainer/README.md) README. +- Details on all the Dockerfile options can be found in the [Dockerfile](https://github.com/microsoft/autogen/blob/0.2/.devcontainer/README.md) README. - For more information on Docker usage and best practices, refer to the [official Docker documentation](https://docs.docker.com). - Details on how to use the Dockerfile dev version can be found on the [Contributor Guide](/docs/contributor-guide/docker). diff --git a/website/docs/installation/Installation.mdx b/website/docs/installation/Installation.mdx index af3ed6620135..8c41f09cfde1 100644 --- a/website/docs/installation/Installation.mdx +++ b/website/docs/installation/Installation.mdx @@ -13,8 +13,8 @@ When installing AutoGen locally, we recommend using a virtual environment for th Create and activate: ```bash - python3 -m venv pyautogen - source pyautogen/bin/activate + python3 -m venv .venv + source .venv/bin/activate ``` To deactivate later, run: @@ -32,8 +32,8 @@ When installing AutoGen locally, we recommend using a virtual environment for th Create and activate: ```bash - conda create -n pyautogen python=3.10 - conda activate pyautogen + conda create -n autogen python=3.10 + conda activate autogen ``` To deactivate later, run: @@ -52,7 +52,7 @@ When installing AutoGen locally, we recommend using a virtual environment for th poetry init poetry shell - poetry add pyautogen + poetry add autogen-agentchat~=0.2 ``` To deactivate later, run: @@ -69,15 +69,9 @@ When installing AutoGen locally, we recommend using a virtual environment for th AutoGen requires **Python version >= 3.8, < 3.13**. It can be installed from pip: ```bash -pip install pyautogen +pip install autogen-agentchat~=0.2 ``` -:::info - -`pyautogen<0.2` required `openai<1`. Starting from pyautogen v0.2, `openai>=1` is required. - -::: - ## Install Docker for Code Execution We recommend using Docker for code execution. diff --git a/website/docs/installation/Optional-Dependencies.md b/website/docs/installation/Optional-Dependencies.md index 7d17ce50e37d..89013d023ea6 100644 --- a/website/docs/installation/Optional-Dependencies.md +++ b/website/docs/installation/Optional-Dependencies.md @@ -6,7 +6,7 @@ To use LLM caching with Redis, you need to install the Python package with the option `redis`: ```bash -pip install "pyautogen[redis]" +pip install "autogen-agentchat[redis]~=0.2" ``` See [LLM Caching](/docs/topics/llm-caching) for details. @@ -17,7 +17,7 @@ To use the IPython code executor, you need to install the `jupyter-client` and `ipykernel` packages: ```bash -pip install "pyautogen[ipython]" +pip install "autogen-agentchat[ipython]~=0.2" ``` To use the IPython code executor: @@ -38,27 +38,27 @@ pip install "pyautogen[blendsearch]<0.2" Example notebooks: -[Optimize for Code Generation](https://github.com/microsoft/autogen/blob/main/notebook/oai_completion.ipynb) +[Optimize for Code Generation](https://github.com/microsoft/autogen/blob/0.2/notebook/oai_completion.ipynb) -[Optimize for Math](https://github.com/microsoft/autogen/blob/main/notebook/oai_chatgpt_gpt4.ipynb) +[Optimize for Math](https://github.com/microsoft/autogen/blob/0.2/notebook/oai_chatgpt_gpt4.ipynb) ## retrievechat -`pyautogen` supports retrieval-augmented generation tasks such as question answering and code generation with RAG agents. Please install with the [retrievechat] option to use it with ChromaDB. +AutoGen 0.2 supports retrieval-augmented generation tasks such as question answering and code generation with RAG agents. Please install with the [retrievechat] option to use it with ChromaDB. ```bash -pip install "pyautogen[retrievechat]" +pip install "autogen-agentchat[retrievechat]" ``` *You'll need to install `chromadb<=0.5.0` if you see issue like [#3551](https://github.com/microsoft/autogen/issues/3551).* -Alternatively `pyautogen` also supports PGVector and Qdrant which can be installed in place of ChromaDB, or alongside it. +Alternatively AutoGen 0.2 also supports PGVector and Qdrant which can be installed in place of ChromaDB, or alongside it. ```bash -pip install "pyautogen[retrievechat-pgvector]" +pip install "autogen-agentchat[retrievechat-pgvector]~=0.2" ``` ```bash -pip install "pyautogen[retrievechat-qdrant]" +pip install "autogen-agentchat[retrievechat-qdrant]~=0.2" ``` RetrieveChat can handle various types of documents. By default, it can process @@ -72,33 +72,33 @@ You can find a list of all supported document types by using `autogen.retrieve_u Example notebooks: -[Automated Code Generation and Question Answering with Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat.ipynb) +[Automated Code Generation and Question Answering with Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat.ipynb) -[Group Chat with Retrieval Augmented Generation (with 5 group member agents and 1 manager agent)](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_groupchat_RAG.ipynb) +[Group Chat with Retrieval Augmented Generation (with 5 group member agents and 1 manager agent)](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_groupchat_RAG.ipynb) -[Automated Code Generation and Question Answering with Qdrant based Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_qdrant.ipynb) +[Automated Code Generation and Question Answering with Qdrant based Retrieval Augmented Agents](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat_qdrant.ipynb) ## Teachability To use Teachability, please install AutoGen with the [teachable] option. ```bash -pip install "pyautogen[teachable]" +pip install "autogen-agentchat[teachable]~=0.2" ``` -Example notebook: [Chatting with a teachable agent](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_teachability.ipynb) +Example notebook: [Chatting with a teachable agent](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_teachability.ipynb) ## Large Multimodal Model (LMM) Agents We offered Multimodal Conversable Agent and LLaVA Agent. Please install with the [lmm] option to use it. ```bash -pip install "pyautogen[lmm]" +pip install "autogen-agentchat[lmm]~=0.2" ``` Example notebooks: -[LLaVA Agent](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_lmm_llava.ipynb) +[LLaVA Agent](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_lmm_llava.ipynb) ## mathchat @@ -110,14 +110,14 @@ pip install "pyautogen[mathchat]<0.2" Example notebooks: -[Using MathChat to Solve Math Problems](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_MathChat.ipynb) +[Using MathChat to Solve Math Problems](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_MathChat.ipynb) ## Graph To use a graph in `GroupChat`, particularly for graph visualization, please install AutoGen with the [graph] option. ```bash -pip install "pyautogen[graph]" +pip install "autogen-agentchat[graph]~=0.2" ``` Example notebook: [Finite State Machine graphs to set speaker transition constraints](https://microsoft.github.io/autogen/docs/notebooks/agentchat_groupchat_finite_state_machine) @@ -127,5 +127,5 @@ Example notebook: [Finite State Machine graphs to set speaker transition constra AutoGen includes support for handling long textual contexts by leveraging the LLMLingua library for text compression. To enable this functionality, please install AutoGen with the `[long-context]` option: ```bash -pip install "pyautogen[long-context]" +pip install "autogen-agentchat[long-context]~=0.2" ``` diff --git a/website/docs/topics/code-execution/custom-executor.ipynb b/website/docs/topics/code-execution/custom-executor.ipynb index c6ee4c160189..41d3b59b8fdc 100644 --- a/website/docs/topics/code-execution/custom-executor.ipynb +++ b/website/docs/topics/code-execution/custom-executor.ipynb @@ -18,7 +18,7 @@ "metadata": {}, "outputs": [], "source": [ - "! pip -qqq install pyautogen matplotlib yfinance" + "! pip -qqq install autogen-agentchat~=0.2 matplotlib yfinance" ] }, { diff --git a/website/docs/topics/code-execution/jupyter-code-executor.ipynb b/website/docs/topics/code-execution/jupyter-code-executor.ipynb index 09f35f1fdc98..9ee72bccc0a1 100644 --- a/website/docs/topics/code-execution/jupyter-code-executor.ipynb +++ b/website/docs/topics/code-execution/jupyter-code-executor.ipynb @@ -15,7 +15,7 @@ "In order to use Jupyter based code execution some extra dependencies are required. These can be installed with the extra `jupyter-executor`:\n", "\n", "```bash\n", - "pip install 'pyautogen[jupyter-executor]'\n", + "pip install 'autogen-agentchat[jupyter-executor]~=0.2'\n", "```\n", "\n", "## Jupyter Server\n", diff --git a/website/docs/topics/code-execution/kubernetes-pod-commandline-code-executor.ipynb b/website/docs/topics/code-execution/kubernetes-pod-commandline-code-executor.ipynb new file mode 100644 index 000000000000..2cad17e0deb5 --- /dev/null +++ b/website/docs/topics/code-execution/kubernetes-pod-commandline-code-executor.ipynb @@ -0,0 +1,773 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Kubernetes Pod Commandline Code Executor" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The `PodCommandLineCodeExecutor` in the `autogen.coding.kubernetes` module is designed to execute code blocks using a pod in Kubernetes.\n", + "It functions similarly to the `DockerCommandLineCodeExecutor`, but specifically creates container within Kubernetes environments.\n", + "\n", + "There are two condition to use PodCommandLineCodeExecutor.\n", + "- Access to a Kubernetes cluster\n", + "- installation `autogen` with the extra requirements `'pyautogen[kubernetes]'`\n", + "\n", + "For local development and testing, this document uses a Minikube cluster.\n", + "\n", + "Minikube is a tool that allows you to run a single-node Kubernetes cluster on you local machine. \n", + "You can refer to the link below for installation and setup of Minikube.\n", + "\n", + "🔗 https://minikube.sigs.k8s.io/docs/start/" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Access kubernetes cluster" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "There are four options PodCommandLineCodeExecutor to access kubernetes API server.\n", + "- default kubeconfig file path: `~/.kube/config`\n", + "- Provide a custom kubeconfig file path using the `kube_config_file` argument of `PodCommandLineCodeExecutor`.\n", + "- Set the kubeconfig file path using the `KUBECONFIG` environment variable.\n", + "- Provide token from Kubernetes ServiceAccount with sufficient permissions" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Generally, if kubeconfig file is located in `~/.kube/config`, there's no need to provide kubeconfig file path on parameter or environment variables.\n", + "\n", + "The tutorial of providing ServiceAccount Token is in the last section" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Example\n", + "\n", + "In order to use kubernetes Pod based code executor, you need to install Kubernetes Python SDK.\n", + "\n", + "You can do this by running the following command:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip install 'kubernetes>=27'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Alternatively, you can install it with the extra features for Kubernetes:" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "pip install 'autogen-agentchat[kubernetes]~=0.2'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To provide kubeconfig file path with environment variable, It can be added with `os.environ[\"KUBECONFIG\"]`" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "# Set the KUBECONFIG environment variable\n", + "# if the kubeconfig file is not in the default location(~/.kube/config).\n", + "os.environ[\"KUBECONFIG\"] = \"path/to/your/kubeconfig\"" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "from autogen.coding import CodeBlock\n", + "from autogen.coding.kubernetes import PodCommandLineCodeExecutor" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "exit_code=0 output='Hello, World!\\n' code_file='/workspace/tmp_code_07da107bb575cc4e02b0e1d6d99cc204.py'\n" + ] + } + ], + "source": [ + "with PodCommandLineCodeExecutor(\n", + " namespace=\"default\",\n", + " # kube_config_file=\"kubeconfig/file/path\" # If you have another kubeconfig file, you can add it on kube_config_file argument\n", + ") as executor:\n", + " print(\n", + " executor.execute_code_blocks(\n", + " # Example of executing a simple Python code block within a Kubernetes pod.\n", + " code_blocks=[\n", + " CodeBlock(language=\"python\", code=\"print('Hello, World!')\"),\n", + " ]\n", + " )\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Using a context manager(the `with` statement), the pod created by `PodCommandLineCodeExecutor` is automatically deleted after the tasks are completed.\n", + "\n", + "Although the pod is automatically deleted when using a context manager, you might sometimes need to delete it manually. You can do this using `stop()` method, as shown below:" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "executor = PodCommandLineCodeExecutor(namespace=\"default\")" + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "NAME READY STATUS RESTARTS AGE\n", + "autogen-code-exec-afd217ac-f77b-4ede-8c53-1297eca5ec64 1/1 Running 0 10m\n" + ] + } + ], + "source": [ + "%%bash\n", + "# This command lists all pods in the default namespace. \n", + "# The default pod name follows the format autogen-code-exec-{uuid.uuid4()}.\n", + "kubectl get pod -n default" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "python:3-slim" + ] + } + ], + "source": [ + "%%bash\n", + "# This command shows container's image in the pod.\n", + "# The default container image is python:3-slim\n", + "kubectl get pod autogen-code-exec-afd217ac-f77b-4ede-8c53-1297eca5ec64 -o jsonpath={.spec.containers[0].image}" + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "executor.stop()" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To use a different container image for code executor pod, specify the desired image tag using `image` argument.\n", + "\n", + "`PodCommandLineCodeExecutor` has a default execution policy that allows Python and shell script code blocks. You can enable other languages with `execution_policies` argument." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "exit_code=0 output='Hello, World!\\n' code_file='app/tmp_code_8c34c8586cb47943728afe1297b7a51c.js'\n" + ] + } + ], + "source": [ + "with PodCommandLineCodeExecutor(\n", + " image=\"node:22-alpine\", # Specifies the runtime environments using a container image\n", + " namespace=\"default\",\n", + " work_dir=\"./app\", # Directory within the container where code block files are stored\n", + " timeout=10, # Timeout in seconds for pod creation and code block execution (default is 60 seconds)\n", + " execution_policies={\n", + " \"javascript\": True\n", + " }, # Enable execution of Javascript code blocks by updating execution policies\n", + ") as executor:\n", + " print(\n", + " executor.execute_code_blocks(\n", + " code_blocks=[\n", + " CodeBlock(language=\"javascript\", code=\"console.log('Hello, World!')\"),\n", + " ]\n", + " )\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If you want to apply custom settings for executor pod, such as annotations, environment variables, commands, volumes etc., \n", + "you can provide a custom pod specification using `kubernetes.client.V1Pod` format.\n", + "\n", + "The `container_name` argument should also be provided because `PodCommandLineCodeExecutor` does not automatically recognize the container where code blocks will be executed." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [], + "source": [ + "from kubernetes import client\n", + "\n", + "pod = client.V1Pod(\n", + " metadata=client.V1ObjectMeta(name=\"abcd\", namespace=\"default\", annotations={\"sidecar.istio.io/inject\": \"false\"}),\n", + " spec=client.V1PodSpec(\n", + " restart_policy=\"Never\",\n", + " containers=[\n", + " client.V1Container(\n", + " args=[\"-c\", \"while true;do sleep 5; done\"],\n", + " command=[\"/bin/sh\"],\n", + " name=\"abcd\", # container name where code blocks will be executed should be provided using `container_name` argument\n", + " image=\"python:3.11-slim\",\n", + " env=[\n", + " client.V1EnvVar(name=\"TEST\", value=\"TEST\"),\n", + " client.V1EnvVar(\n", + " name=\"POD_NAME\",\n", + " value_from=client.V1EnvVarSource(\n", + " field_ref=client.V1ObjectFieldSelector(field_path=\"metadata.name\")\n", + " ),\n", + " ),\n", + " ],\n", + " )\n", + " ],\n", + " ),\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "exit_code=0 output='Hello, World!\\n' code_file='/autogen/tmp_code_07da107bb575cc4e02b0e1d6d99cc204.py'\n", + "exit_code=0 output='TEST abcd\\n' code_file='/autogen/tmp_code_202399627ea7fb8d8e816f4910b7f87b.sh'\n" + ] + } + ], + "source": [ + "with PodCommandLineCodeExecutor(\n", + " pod_spec=pod, # custom executor pod spec\n", + " container_name=\"abcd\", # To use custom executor pod spec, container_name where code block will be executed should be specified\n", + " work_dir=\"/autogen\",\n", + " timeout=60,\n", + ") as executor:\n", + " print(\n", + " executor.execute_code_blocks(\n", + " code_blocks=[\n", + " CodeBlock(language=\"python\", code=\"print('Hello, World!')\"),\n", + " ]\n", + " )\n", + " )\n", + " print(\n", + " executor.execute_code_blocks(\n", + " code_blocks=[\n", + " CodeBlock(\n", + " code=\"echo $TEST $POD_NAME\", language=\"bash\"\n", + " ), # echo environment variables specified in pod_spec\n", + " ]\n", + " )\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Integrates with AutoGen Agents" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "`PodCommandLineCodeExecutor` can be integrated with Agents." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [], + "source": [ + "from autogen import config_list_from_json\n", + "\n", + "config_list = config_list_from_json(\n", + " env_or_file=\"OAI_CONFIG_LIST\",\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mcode_executor_agent\u001b[0m (to code_writer):\n", + "\n", + "Write Python code to calculate the moves of disk on tower of hanoi with 3 disks\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mcode_writer\u001b[0m (to code_executor_agent):\n", + "\n", + "The problem of the Tower of Hanoi with 3 disks involves moving the disks from one peg to another, following these rules:\n", + "1. Only one disk can be moved at a time.\n", + "2. Each move consists of taking the upper disk from one of the stacks and placing it on top of another stack or on an empty peg.\n", + "3. No disk may be placed on top of a smaller disk.\n", + "\n", + "In the solution, I will use a recursive function to calculate the moves and print them out. Here's the Python code to accomplish this:\n", + "\n", + "```python\n", + "def tower_of_hanoi(n, from_rod, to_rod, aux_rod):\n", + " if n == 1:\n", + " print(f\"Move disk 1 from rod {from_rod} to rod {to_rod}\")\n", + " return\n", + " tower_of_hanoi(n-1, from_rod, aux_rod, to_rod)\n", + " print(f\"Move disk {n} from rod {from_rod} to rod {to_rod}\")\n", + " tower_of_hanoi(n-1, aux_rod, to_rod, from_rod)\n", + "\n", + "n = 3 # Number of disks\n", + "tower_of_hanoi(n, 'A', 'C', 'B') # A, B and C are names of the rods\n", + "```\n", + "\n", + "This script defines a function `tower_of_hanoi` that will print out each move necessary to solve the Tower of Hanoi problem with the specified number of disks `n`. This specific setup will solve for 3 disks moving from rod 'A' to rod 'C' with the help of rod 'B'.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK (inferred language is python)...\u001b[0m\n", + "\u001b[33mcode_executor_agent\u001b[0m (to code_writer):\n", + "\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: Move disk 1 from rod A to rod C\n", + "Move disk 2 from rod A to rod B\n", + "Move disk 1 from rod C to rod B\n", + "Move disk 3 from rod A to rod C\n", + "Move disk 1 from rod B to rod A\n", + "Move disk 2 from rod B to rod C\n", + "Move disk 1 from rod A to rod C\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mcode_writer\u001b[0m (to code_executor_agent):\n", + "\n", + "The execution of the provided code successfully calculated and printed the moves for solving the Tower of Hanoi with 3 disks. Here are the steps it performed:\n", + "\n", + "1. Move disk 1 from rod A to rod C.\n", + "2. Move disk 2 from rod A to rod B.\n", + "3. Move disk 1 from rod C to rod B.\n", + "4. Move disk 3 from rod A to rod C.\n", + "5. Move disk 1 from rod B to rod A.\n", + "6. Move disk 2 from rod B to rod C.\n", + "7. Move disk 1 from rod A to rod C.\n", + "\n", + "This sequence effectively transfers all disks from rod A to rod C using rod B as an auxiliary, following the rules of the Tower of Hanoi puzzle. If you have any more tasks or need further explanation, feel free to ask!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mcode_executor_agent\u001b[0m (to code_writer):\n", + "\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + } + ], + "source": [ + "from autogen import ConversableAgent\n", + "\n", + "# The code writer agent's system message is to instruct the LLM on how to\n", + "# use the code executor with python or shell script code\n", + "code_writer_system_message = \"\"\"\n", + "You have been given coding capability to solve tasks using Python code.\n", + "In the following cases, suggest python code (in a python coding block) or shell script (in a sh coding block) for the user to execute.\n", + " 1. When you need to collect info, use the code to output the info you need, for example, browse or search the web, download/read a file, print the content of a webpage or a file, get the current date/time, check the operating system. After sufficient info is printed and the task is ready to be solved based on your language skill, you can solve the task by yourself.\n", + " 2. When you need to perform some task with code, use the code to perform the task and output the result. Finish the task smartly.\n", + "Solve the task step by step if you need to. If a plan is not provided, explain your plan first. Be clear which step uses code, and which step uses your language skill.\n", + "When using code, you must indicate the script type in the code block. The user cannot provide any other feedback or perform any other action beyond executing the code you suggest. The user can't modify your code. So do not suggest incomplete code which requires users to modify. Don't use a code block if it's not intended to be executed by the user.\n", + "If you want the user to save the code in a file before executing it, put # filename: inside the code block as the first line. Don't include multiple code blocks in one response. Do not ask users to copy and paste the result. Instead, use 'print' function for the output when relevant. Check the execution result returned by the user.\n", + "\"\"\"\n", + "with PodCommandLineCodeExecutor(namespace=\"default\") as executor:\n", + "\n", + " code_executor_agent = ConversableAgent(\n", + " name=\"code_executor_agent\",\n", + " llm_config=False,\n", + " code_execution_config={\n", + " \"executor\": executor,\n", + " },\n", + " human_input_mode=\"NEVER\",\n", + " )\n", + "\n", + " code_writer_agent = ConversableAgent(\n", + " \"code_writer\",\n", + " system_message=code_writer_system_message,\n", + " llm_config={\"config_list\": config_list},\n", + " code_execution_config=False, # Turn off code execution for this agent.\n", + " max_consecutive_auto_reply=2,\n", + " human_input_mode=\"NEVER\",\n", + " )\n", + "\n", + " chat_result = code_executor_agent.initiate_chat(\n", + " code_writer_agent, message=\"Write Python code to calculate the moves of disk on tower of hanoi with 10 disks\"\n", + " )" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "id": "93802984-3207-430b-a205-82f0a77df2b2", + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ChatResult(chat_id=None,\n", + " chat_history=[{'content': 'Write Python code to calculate the moves '\n", + " 'of disk on tower of hanoi with 3 disks',\n", + " 'name': 'code_executor_agent',\n", + " 'role': 'assistant'},\n", + " {'content': 'The problem of the Tower of Hanoi with 3 '\n", + " 'disks involves moving the disks from one '\n", + " 'peg to another, following these rules:\\n'\n", + " '1. Only one disk can be moved at a '\n", + " 'time.\\n'\n", + " '2. Each move consists of taking the '\n", + " 'upper disk from one of the stacks and '\n", + " 'placing it on top of another stack or on '\n", + " 'an empty peg.\\n'\n", + " '3. No disk may be placed on top of a '\n", + " 'smaller disk.\\n'\n", + " '\\n'\n", + " 'In the solution, I will use a recursive '\n", + " 'function to calculate the moves and '\n", + " \"print them out. Here's the Python code \"\n", + " 'to accomplish this:\\n'\n", + " '\\n'\n", + " '```python\\n'\n", + " 'def tower_of_hanoi(n, from_rod, to_rod, '\n", + " 'aux_rod):\\n'\n", + " ' if n == 1:\\n'\n", + " ' print(f\"Move disk 1 from rod '\n", + " '{from_rod} to rod {to_rod}\")\\n'\n", + " ' return\\n'\n", + " ' tower_of_hanoi(n-1, from_rod, '\n", + " 'aux_rod, to_rod)\\n'\n", + " ' print(f\"Move disk {n} from rod '\n", + " '{from_rod} to rod {to_rod}\")\\n'\n", + " ' tower_of_hanoi(n-1, aux_rod, to_rod, '\n", + " 'from_rod)\\n'\n", + " '\\n'\n", + " 'n = 3 # Number of disks\\n'\n", + " \"tower_of_hanoi(n, 'A', 'C', 'B') # A, B \"\n", + " 'and C are names of the rods\\n'\n", + " '```\\n'\n", + " '\\n'\n", + " 'This script defines a function '\n", + " '`tower_of_hanoi` that will print out '\n", + " 'each move necessary to solve the Tower '\n", + " 'of Hanoi problem with the specified '\n", + " 'number of disks `n`. This specific setup '\n", + " 'will solve for 3 disks moving from rod '\n", + " \"'A' to rod 'C' with the help of rod 'B'.\",\n", + " 'name': 'code_writer',\n", + " 'role': 'user'},\n", + " {'content': 'exitcode: 0 (execution succeeded)\\n'\n", + " 'Code output: Move disk 1 from rod A to '\n", + " 'rod C\\n'\n", + " 'Move disk 2 from rod A to rod B\\n'\n", + " 'Move disk 1 from rod C to rod B\\n'\n", + " 'Move disk 3 from rod A to rod C\\n'\n", + " 'Move disk 1 from rod B to rod A\\n'\n", + " 'Move disk 2 from rod B to rod C\\n'\n", + " 'Move disk 1 from rod A to rod C\\n',\n", + " 'name': 'code_executor_agent',\n", + " 'role': 'assistant'},\n", + " {'content': 'The execution of the provided code '\n", + " 'successfully calculated and printed the '\n", + " 'moves for solving the Tower of Hanoi '\n", + " 'with 3 disks. Here are the steps it '\n", + " 'performed:\\n'\n", + " '\\n'\n", + " '1. Move disk 1 from rod A to rod C.\\n'\n", + " '2. Move disk 2 from rod A to rod B.\\n'\n", + " '3. Move disk 1 from rod C to rod B.\\n'\n", + " '4. Move disk 3 from rod A to rod C.\\n'\n", + " '5. Move disk 1 from rod B to rod A.\\n'\n", + " '6. Move disk 2 from rod B to rod C.\\n'\n", + " '7. Move disk 1 from rod A to rod C.\\n'\n", + " '\\n'\n", + " 'This sequence effectively transfers all '\n", + " 'disks from rod A to rod C using rod B as '\n", + " 'an auxiliary, following the rules of the '\n", + " 'Tower of Hanoi puzzle. If you have any '\n", + " 'more tasks or need further explanation, '\n", + " 'feel free to ask!',\n", + " 'name': 'code_writer',\n", + " 'role': 'user'},\n", + " {'content': '',\n", + " 'name': 'code_executor_agent',\n", + " 'role': 'assistant'}],\n", + " summary='',\n", + " cost={'usage_excluding_cached_inference': {'total_cost': 0},\n", + " 'usage_including_cached_inference': {'gpt-4-turbo-2024-04-09': {'completion_tokens': 499,\n", + " 'cost': 0.0269,\n", + " 'prompt_tokens': 1193,\n", + " 'total_tokens': 1692},\n", + " 'total_cost': 0.0269}},\n", + " human_input=[])\n" + ] + } + ], + "source": [ + "import pprint\n", + "\n", + "pprint.pprint(chat_result)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Use ServiceAccount token" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "If a `PodCommandLineCodeExecutor` instance runs inside of Kubernetes Pod, it can use a token generated from a ServiceAccount to access Kubernetes API server.\n", + "\n", + "The `PodCommandLineCodeExecutor` requires the following permissions:\n", + "the verbs `create`, `get`, `delete` for `pods` resource, and the verb `get` for resources `pods/status`, `pods/exec`.\n", + "\n", + "You can create a ServiceAccount, ClusterRole and RoleBinding with `kubectl` as shown below:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "serviceaccount/autogen-executor-sa created\n" + ] + } + ], + "source": [ + "%%bash\n", + "# Create ServiceAccount on default namespace\n", + "kubectl create sa autogen-executor-sa" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "clusterrole.rbac.authorization.k8s.io/autogen-executor-role created\n" + ] + } + ], + "source": [ + "%%bash\n", + "# Create ClusterRole that has sufficient permissions\n", + "kubectl create clusterrole autogen-executor-role \\\n", + " --verb=get,create,delete --resource=pods,pods/status,pods/exec" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "rolebinding.rbac.authorization.k8s.io/autogen-executor-rolebinding created\n" + ] + } + ], + "source": [ + "%%bash\n", + "# Create RoleBinding that binds ClusterRole and ServiceAccount\n", + "kubectl create rolebinding autogen-executor-rolebinding \\\n", + " --clusterrole autogen-executor-role --serviceaccount default:autogen-executor-sa" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "A pod with a previously created ServiceAccount can be launched using the following command." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "pod/autogen-executor created\n" + ] + } + ], + "source": [ + "%%bash\n", + "# create pod with serviceaccount\n", + "kubectl run autogen-executor --image python:3 \\\n", + " --overrides='{\"spec\":{\"serviceAccount\": \"autogen-executor-sa\"}}' \\\n", + " -- bash -c 'pip install pyautogen[kubernetes] && sleep inifinity'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "You can execute `PodCommandLineCodeExecutor` inside the Python interpreter process from `autogen-executor` Pod.\n", + "\n", + "It creates new pod for code execution using token generated from `autogen-executor-sa` ServiceAccount." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "%%bash\n", + "kubectl exec autogen-executor -it -- python" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "kube_config_path not provided and default location (~/.kube/config) does not exist. Using inCluster Config. This might not work.\n", + "exit_code=0 output='Hello, World!\\n' code_file='/workspace/tmp_code_07da107bb575cc4e02b0e1d6d99cc204.py'" + ] + } + ], + "source": [ + "from autogen.coding import CodeBlock\n", + "from autogen.coding.kubernetes import PodCommandLineCodeExecutor\n", + "\n", + "# PodCommandLineCodeExecutor uses token generated from ServiceAccount by kubernetes incluster config\n", + "with PodCommandLineCodeExecutor() as executor:\n", + " print(\n", + " executor.execute_code_blocks(\n", + " code_blocks=[\n", + " CodeBlock(language=\"python\", code=\"print('Hello, World!')\"),\n", + " ]\n", + " )\n", + " )" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "autogen", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.10.12" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/website/docs/topics/groupchat/using_custom_models.md b/website/docs/topics/groupchat/using_custom_models.md new file mode 100644 index 000000000000..4948d8ab9f6e --- /dev/null +++ b/website/docs/topics/groupchat/using_custom_models.md @@ -0,0 +1,79 @@ +# Using Custom Models + +When using `GroupChatManager` we need to pass a `GroupChat` object in the constructor, a dataclass responsible for +gathering agents, preparing messages from prompt templates and selecting speakers +(eventually using `speaker_selection_method` as described [here](customized_speaker_selection)). + +To do so GroupChat internally initializes two instances of ConversableAgent. +In order to control the model clients used by the agents instantiated within the GroupChat, which already receives the +`llm_config` passed to GroupChatManager, the optional `model_client_cls` attribute can be set. + + +## Example +First we need to define an `llm_config` and define some agents that will partake in the group chat: +```python +from autogen import GroupChat, ConversableAgent, GroupChatManager, UserProxyAgent +from somewhere import MyModelClient + + +# Define the custom model configuration +llm_config = { + "config_list": [ + { + "model": "gpt-3.5-turbo", + "model_client_cls": "MyModelClient" + } + ] +} + +# Initialize the agents with the custom model +agent1 = ConversableAgent( + name="Agent 1", + llm_config=llm_config +) +agent1.register_model_client(model_client_cls=MyModelClient) + +agent2 = ConversableAgent( + name="Agent 2", + llm_config=llm_config +) +agent2.register_model_client(model_client_cls=MyModelClient) + +agent3 = ConversableAgent( + name="Agent 2", + llm_config=llm_config +) +agent3.register_model_client(model_client_cls=MyModelClient) + +user_proxy = UserProxyAgent(name="user", llm_config=llm_config, code_execution_config={"use_docker": False}) +user_proxy.register_model_client(MyModelClient) +``` + +Note that the agents definition illustrated here is minimal and might not suit your needs. The only aim is to show a +basic setup for a group chat scenario. + +We then create a `GroupChat` and, if we want the underlying agents used by GroupChat to use our +custom client, we will pass it in the `model_client_cls` attribute. + +Finally we create an instance of `GroupChatManager` and pass the config to it. This same config will be forwarded to +the GroupChat, that (if needed) will automatically handle registration of custom models only. + +```python +# Create a GroupChat instance and add the agents +group_chat = GroupChat(agents=[agent1, agent2, agent3], messages=[], model_client_cls=MyModelClient) + +# Create the GroupChatManager with the GroupChat, UserProxy, and model configuration +chat_manager = GroupChatManager(groupchat=group_chat, llm_config=llm_config) +chat_manager.register_model_client(model_client_cls=MyModelClient) + +# Initiate the chat using the UserProxy +user_proxy.initiate_chat(chat_manager, initial_message="Suggest me the most trending papers in microbiology that you think might interest me") + +``` + +This attribute can either be a class or a list of classes which adheres to the `ModelClient` protocol (see +[this link](../non-openai-models/about-using-nonopenai-models) for more info about defining a custom model client +class). + +Note that it is not necessary to define a `model_client_cls` when working with Azure OpenAI, OpenAI or other non-custom +models natively supported by the library. diff --git a/website/docs/topics/handling_long_contexts/compressing_text_w_llmligua.md b/website/docs/topics/handling_long_contexts/compressing_text_w_llmligua.md index e251786f5550..965bbfbd0105 100644 --- a/website/docs/topics/handling_long_contexts/compressing_text_w_llmligua.md +++ b/website/docs/topics/handling_long_contexts/compressing_text_w_llmligua.md @@ -5,10 +5,10 @@ Text compression is crucial for optimizing interactions with LLMs, especially wh This guide introduces LLMLingua's integration with AutoGen, demonstrating how to use this tool to compress text, thereby optimizing the usage of LLMs for various applications. :::info Requirements -Install `pyautogen[long-context]` and `PyMuPDF`: +Install `autogen-agentchat[long-context]~=0.2` and `PyMuPDF`: ```bash -pip install "pyautogen[long-context]" PyMuPDF +pip install "autogen-agentchat[long-context]~=0.2" PyMuPDF ``` For more information, please refer to the [installation guide](/docs/installation/). diff --git a/website/docs/topics/handling_long_contexts/intro_to_transform_messages.md b/website/docs/topics/handling_long_contexts/intro_to_transform_messages.md index 52fea15d01e5..fc854f8d8341 100644 --- a/website/docs/topics/handling_long_contexts/intro_to_transform_messages.md +++ b/website/docs/topics/handling_long_contexts/intro_to_transform_messages.md @@ -13,10 +13,10 @@ Why do we need to handle long contexts? The problem arises from several constrai The `TransformMessages` capability is designed to modify incoming messages before they are processed by the LLM agent. This can include limiting the number of messages, truncating messages to meet token limits, and more. :::info Requirements -Install `pyautogen`: +Install `autogen-agentchat`: ```bash -pip install pyautogen +pip install autogen-agentchat~=0.2 ``` For more information, please refer to the [installation guide](/docs/installation/). diff --git a/website/docs/topics/non-openai-models/cloud-anthropic.ipynb b/website/docs/topics/non-openai-models/cloud-anthropic.ipynb index a6c87b6a5ca5..ba4c831232c0 100644 --- a/website/docs/topics/non-openai-models/cloud-anthropic.ipynb +++ b/website/docs/topics/non-openai-models/cloud-anthropic.ipynb @@ -21,7 +21,7 @@ "Additionally, this client class provides support for function/tool calling and will track token usage and cost correctly as per Anthropic's API costs (as of June 2024).\n", "\n", "## Requirements\n", - "To use Anthropic Claude with AutoGen, first you need to install the `pyautogen[anthropic]` package.\n", + "To use Anthropic Claude with AutoGen, first you need to install the `autogen-agentchat[anthropic]` package.\n", "\n", "To try out the function call feature of Claude model, you need to install `anthropic>=0.23.1`.\n" ] @@ -32,7 +32,7 @@ "metadata": {}, "outputs": [], "source": [ - "!pip install pyautogen[\"anthropic\"]" + "!pip install autogen-agentchat[\"anthropic\"]~=0.2" ] }, { diff --git a/website/docs/topics/non-openai-models/cloud-bedrock.ipynb b/website/docs/topics/non-openai-models/cloud-bedrock.ipynb index 71c1e2e7ffe3..422598dd2fe1 100644 --- a/website/docs/topics/non-openai-models/cloud-bedrock.ipynb +++ b/website/docs/topics/non-openai-models/cloud-bedrock.ipynb @@ -25,7 +25,7 @@ "It does not, yet, support image generation ([contribute](https://microsoft.github.io/autogen/docs/contributor-guide/contributing/)).\n", "\n", "## Requirements\n", - "To use Amazon Bedrock with AutoGen, first you need to install the `pyautogen[bedrock]` package.\n", + "To use Amazon Bedrock with AutoGen, first you need to install the `autogen-agentchat[bedrock]` package.\n", "\n", "## Pricing\n", "\n", @@ -48,7 +48,7 @@ "outputs": [], "source": [ "# If you need to install AutoGen with Amazon Bedrock\n", - "!pip install pyautogen[\"bedrock\"]" + "!pip install autogen-agentchat[\"bedrock\"]~=0.2" ] }, { diff --git a/website/docs/topics/non-openai-models/cloud-cerebras.ipynb b/website/docs/topics/non-openai-models/cloud-cerebras.ipynb new file mode 100644 index 000000000000..71cb9da446c5 --- /dev/null +++ b/website/docs/topics/non-openai-models/cloud-cerebras.ipynb @@ -0,0 +1,505 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Cerebras\n", + "\n", + "[Cerebras](https://cerebras.ai) has developed the world's largest and fastest AI processor, the Wafer-Scale Engine-3 (WSE-3). Notably, the CS-3 system can run large language models like Llama-3.1-8B and Llama-3.1-70B at extremely fast speeds, making it an ideal platform for demanding AI workloads.\n", + "\n", + "While it's technically possible to adapt AutoGen to work with Cerebras' API by updating the `base_url`, this approach may not fully account for minor differences in parameter support. Using this library will also allow for tracking of the API costs based on actual token usage.\n", + "\n", + "For more information about Cerebras Cloud, visit [cloud.cerebras.ai](https://cloud.cerebras.ai). Their API reference is available at [inference-docs.cerebras.ai](https://inference-docs.cerebras.ai)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Requirements\n", + "To use Cerebras with AutoGen, install the `autogen-agentchat[cerebras]` package." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "!pip install autogen-agentchat[\"cerebras\"]~=0.2" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Getting Started\n", + "\n", + "Cerebras provides a number of models to use. See the list of [models here](https://inference-docs.cerebras.ai/introduction).\n", + "\n", + "See the sample `OAI_CONFIG_LIST` below showing how the Cerebras client class is used by specifying the `api_type` as `cerebras`.\n", + "```python\n", + "[\n", + " {\n", + " \"model\": \"llama3.1-8b\",\n", + " \"api_key\": \"your Cerebras API Key goes here\",\n", + " \"api_type\": \"cerebras\"\n", + " },\n", + " {\n", + " \"model\": \"llama3.1-70b\",\n", + " \"api_key\": \"your Cerebras API Key goes here\",\n", + " \"api_type\": \"cerebras\"\n", + " }\n", + "]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Credentials\n", + "\n", + "Get an API Key from [cloud.cerebras.ai](https://cloud.cerebras.ai/) and add it to your environment variables:\n", + "\n", + "```\n", + "export CEREBRAS_API_KEY=\"your-api-key-here\"\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## API parameters\n", + "\n", + "The following parameters can be added to your config for the Cerebras API. See [this link](https://inference-docs.cerebras.ai/api-reference/chat-completions) for further information on them and their default values.\n", + "\n", + "- max_tokens (null, integer >= 0)\n", + "- seed (number)\n", + "- stream (True or False)\n", + "- temperature (number 0..1.5)\n", + "- top_p (number)\n", + "\n", + "Example:\n", + "```python\n", + "[\n", + " {\n", + " \"model\": \"llama3.1-70b\",\n", + " \"api_key\": \"your Cerebras API Key goes here\",\n", + " \"api_type\": \"cerebras\"\n", + " \"max_tokens\": 10000,\n", + " \"seed\": 1234,\n", + " \"stream\" True,\n", + " \"temperature\": 0.5,\n", + " \"top_p\": 0.2, # Note: It is recommended to set temperature or top_p but not both.\n", + " }\n", + "]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Two-Agent Coding Example\n", + "\n", + "In this example, we run a two-agent chat with an AssistantAgent (primarily a coding agent) to generate code to count the number of prime numbers between 1 and 10,000 and then it will be executed.\n", + "\n", + "We'll use Meta's LLama-3.1-70B model which is suitable for coding." + ] + }, + { + "cell_type": "code", + "execution_count": 32, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "from autogen.oai.cerebras import CerebrasClient, calculate_cerebras_cost\n", + "\n", + "config_list = [{\"model\": \"llama3.1-70b\", \"api_key\": os.environ.get(\"CEREBRAS_API_KEY\"), \"api_type\": \"cerebras\"}]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Importantly, we have tweaked the system message so that the model doesn't return the termination keyword, which we've changed to FINISH, with the code block." + ] + }, + { + "cell_type": "code", + "execution_count": 33, + "metadata": {}, + "outputs": [], + "source": [ + "from pathlib import Path\n", + "\n", + "from autogen import AssistantAgent, UserProxyAgent\n", + "from autogen.coding import LocalCommandLineCodeExecutor\n", + "\n", + "# Setting up the code executor\n", + "workdir = Path(\"coding\")\n", + "workdir.mkdir(exist_ok=True)\n", + "code_executor = LocalCommandLineCodeExecutor(work_dir=workdir)\n", + "\n", + "# Setting up the agents\n", + "\n", + "# The UserProxyAgent will execute the code that the AssistantAgent provides\n", + "user_proxy_agent = UserProxyAgent(\n", + " name=\"User\",\n", + " code_execution_config={\"executor\": code_executor},\n", + " is_termination_msg=lambda msg: \"FINISH\" in msg.get(\"content\"),\n", + ")\n", + "\n", + "system_message = \"\"\"You are a helpful AI assistant who writes code and the user executes it.\n", + "Solve tasks using your coding and language skills.\n", + "In the following cases, suggest python code (in a python coding block) for the user to execute.\n", + "Solve the task step by step if you need to. If a plan is not provided, explain your plan first. Be clear which step uses code, and which step uses your language skill.\n", + "When using code, you must indicate the script type in the code block. The user cannot provide any other feedback or perform any other action beyond executing the code you suggest. The user can't modify your code. So do not suggest incomplete code which requires users to modify. Don't use a code block if it's not intended to be executed by the user.\n", + "Don't include multiple code blocks in one response. Do not ask users to copy and paste the result. Instead, use 'print' function for the output when relevant. Check the execution result returned by the user.\n", + "If the result indicates there is an error, fix the error and output the code again. Suggest the full code instead of partial code or code changes. If the error can't be fixed or if the task is not solved even after the code is executed successfully, analyze the problem, revisit your assumption, collect additional info you need, and think of a different approach to try.\n", + "When you find an answer, verify the answer carefully. Include verifiable evidence in your response if possible.\n", + "IMPORTANT: Wait for the user to execute your code and then you can reply with the word \"FINISH\". DO NOT OUTPUT \"FINISH\" after your code block.\"\"\"\n", + "\n", + "# The AssistantAgent, using Llama-3.1-70B on Cerebras Inference, will take the coding request and return code\n", + "assistant_agent = AssistantAgent(\n", + " name=\"Cerebras Assistant\",\n", + " system_message=system_message,\n", + " llm_config={\"config_list\": config_list},\n", + ")" + ] + }, + { + "cell_type": "code", + "execution_count": 35, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mUser\u001b[0m (to Cerebras Assistant):\n", + "\n", + "Provide code to count the number of prime numbers from 1 to 10000.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mCerebras Assistant\u001b[0m (to User):\n", + "\n", + "To count the number of prime numbers from 1 to 10000, we will utilize a simple algorithm that checks each number in the range to see if it is prime. A prime number is a natural number greater than 1 that has no positive divisors other than 1 and itself.\n", + "\n", + "Here's how we can do it using a Python script:\n", + "\n", + "```python\n", + "def count_primes(n):\n", + " primes = 0\n", + " for possiblePrime in range(2, n + 1):\n", + " # Assume number is prime until shown it is not. \n", + " isPrime = True\n", + " for num in range(2, int(possiblePrime ** 0.5) + 1):\n", + " if possiblePrime % num == 0:\n", + " isPrime = False\n", + " break\n", + " if isPrime:\n", + " primes += 1\n", + " return primes\n", + "\n", + "# Counting prime numbers from 1 to 10000\n", + "count = count_primes(10000)\n", + "print(count)\n", + "```\n", + "\n", + "Please execute this code. I will respond with \"FINISH\" after you provide the result.\n", + "\n", + "--------------------------------------------------------------------------------\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Replying as User. Provide feedback to Cerebras Assistant. Press enter to skip and use auto-reply, or type 'exit' to end the conversation: \n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[31m\n", + ">>>>>>>> NO HUMAN INPUT RECEIVED.\u001b[0m\n" + ] + } + ], + "source": [ + "# Start the chat, with the UserProxyAgent asking the AssistantAgent the message\n", + "chat_result = user_proxy_agent.initiate_chat(\n", + " assistant_agent,\n", + " message=\"Provide code to count the number of prime numbers from 1 to 10000.\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tool Call Example\n", + "\n", + "In this example, instead of writing code, we will show how Meta's Llama-3.1-70B model can perform parallel tool calling, where it recommends calling more than one tool at a time.\n", + "\n", + "We'll use a simple travel agent assistant program where we have a couple of tools for weather and currency conversion.\n", + "\n", + "We start by importing libraries and setting up our configuration to use Llama-3.1-70B and the `cerebras` client class." + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "import os\n", + "from typing import Literal\n", + "\n", + "from typing_extensions import Annotated\n", + "\n", + "import autogen\n", + "\n", + "config_list = [\n", + " {\n", + " \"model\": \"llama3.1-70b\",\n", + " \"api_key\": os.environ.get(\"CEREBRAS_API_KEY\"),\n", + " \"api_type\": \"cerebras\",\n", + " }\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create our two agents." + ] + }, + { + "cell_type": "code", + "execution_count": 43, + "metadata": {}, + "outputs": [], + "source": [ + "# Create the agent for tool calling\n", + "chatbot = autogen.AssistantAgent(\n", + " name=\"chatbot\",\n", + " system_message=\"\"\"\n", + " For currency exchange and weather forecasting tasks,\n", + " only use the functions you have been provided with.\n", + " When you summarize, make sure you've considered ALL previous instructions.\n", + " Output 'HAVE FUN!' when an answer has been provided.\n", + " \"\"\",\n", + " llm_config={\"config_list\": config_list},\n", + ")\n", + "\n", + "# Note that we have changed the termination string to be \"HAVE FUN!\"\n", + "user_proxy = autogen.UserProxyAgent(\n", + " name=\"user_proxy\",\n", + " is_termination_msg=lambda x: x.get(\"content\", \"\") and \"HAVE FUN!\" in x.get(\"content\", \"\"),\n", + " human_input_mode=\"NEVER\",\n", + " max_consecutive_auto_reply=1,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create the two functions, annotating them so that those descriptions can be passed through to the LLM.\n", + "\n", + "We associate them with the agents using `register_for_execution` for the user_proxy so it can execute the function and `register_for_llm` for the chatbot (powered by the LLM) so it can pass the function definitions to the LLM." + ] + }, + { + "cell_type": "code", + "execution_count": 44, + "metadata": {}, + "outputs": [], + "source": [ + "# Currency Exchange function\n", + "\n", + "CurrencySymbol = Literal[\"USD\", \"EUR\"]\n", + "\n", + "# Define our function that we expect to call\n", + "\n", + "\n", + "def exchange_rate(base_currency: CurrencySymbol, quote_currency: CurrencySymbol) -> float:\n", + " if base_currency == quote_currency:\n", + " return 1.0\n", + " elif base_currency == \"USD\" and quote_currency == \"EUR\":\n", + " return 1 / 1.1\n", + " elif base_currency == \"EUR\" and quote_currency == \"USD\":\n", + " return 1.1\n", + " else:\n", + " raise ValueError(f\"Unknown currencies {base_currency}, {quote_currency}\")\n", + "\n", + "\n", + "# Register the function with the agent\n", + "\n", + "\n", + "@user_proxy.register_for_execution()\n", + "@chatbot.register_for_llm(description=\"Currency exchange calculator.\")\n", + "def currency_calculator(\n", + " base_amount: Annotated[float, \"Amount of currency in base_currency\"],\n", + " base_currency: Annotated[CurrencySymbol, \"Base currency\"] = \"USD\",\n", + " quote_currency: Annotated[CurrencySymbol, \"Quote currency\"] = \"EUR\",\n", + ") -> str:\n", + " quote_amount = exchange_rate(base_currency, quote_currency) * base_amount\n", + " return f\"{format(quote_amount, '.2f')} {quote_currency}\"\n", + "\n", + "\n", + "# Weather function\n", + "\n", + "\n", + "# Example function to make available to model\n", + "def get_current_weather(location, unit=\"fahrenheit\"):\n", + " \"\"\"Get the weather for some location\"\"\"\n", + " if \"chicago\" in location.lower():\n", + " return json.dumps({\"location\": \"Chicago\", \"temperature\": \"13\", \"unit\": unit})\n", + " elif \"san francisco\" in location.lower():\n", + " return json.dumps({\"location\": \"San Francisco\", \"temperature\": \"55\", \"unit\": unit})\n", + " elif \"new york\" in location.lower():\n", + " return json.dumps({\"location\": \"New York\", \"temperature\": \"11\", \"unit\": unit})\n", + " else:\n", + " return json.dumps({\"location\": location, \"temperature\": \"unknown\"})\n", + "\n", + "\n", + "# Register the function with the agent\n", + "\n", + "\n", + "@user_proxy.register_for_execution()\n", + "@chatbot.register_for_llm(description=\"Weather forecast for US cities.\")\n", + "def weather_forecast(\n", + " location: Annotated[str, \"City name\"],\n", + ") -> str:\n", + " weather_details = get_current_weather(location=location)\n", + " weather = json.loads(weather_details)\n", + " return f\"{weather['location']} will be {weather['temperature']} degrees {weather['unit']}\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We pass through our customer's message and run the chat.\n", + "\n", + "Finally, we ask the LLM to summarise the chat and print that out." + ] + }, + { + "cell_type": "code", + "execution_count": 45, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "What's the weather in New York and can you tell me how much is 123.45 EUR in USD so I can spend it on my holiday? Throw a few holiday tips in as well.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mchatbot\u001b[0m (to user_proxy):\n", + "\n", + "\u001b[32m***** Suggested tool call (210f6ac6d): weather_forecast *****\u001b[0m\n", + "Arguments: \n", + "{\"location\": \"New York\"}\n", + "\u001b[32m*************************************************************\u001b[0m\n", + "\u001b[32m***** Suggested tool call (3c00ac7d5): currency_calculator *****\u001b[0m\n", + "Arguments: \n", + "{\"base_amount\": 123.45, \"base_currency\": \"EUR\", \"quote_currency\": \"USD\"}\n", + "\u001b[32m****************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION weather_forecast...\u001b[0m\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION currency_calculator...\u001b[0m\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[32m***** Response from calling tool (210f6ac6d) *****\u001b[0m\n", + "New York will be 11 degrees fahrenheit\n", + "\u001b[32m**************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[32m***** Response from calling tool (3c00ac7d5) *****\u001b[0m\n", + "135.80 USD\n", + "\u001b[32m**************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mchatbot\u001b[0m (to user_proxy):\n", + "\n", + "New York will be 11 degrees fahrenheit.\n", + "123.45 EUR is equivalent to 135.80 USD.\n", + " \n", + "For a great holiday, explore the Statue of Liberty, take a walk through Central Park, or visit one of the many world-class museums. Also, you'll find great food ranging from bagels to fine dining experiences. HAVE FUN!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "LLM SUMMARY: New York will be 11 degrees fahrenheit. 123.45 EUR is equivalent to 135.80 USD. Explore the Statue of Liberty, walk through Central Park, or visit one of the many world-class museums for a great holiday in New York.\n", + "\n", + "Duration: 73.97937774658203ms\n" + ] + } + ], + "source": [ + "import time\n", + "\n", + "start_time = time.time()\n", + "\n", + "# start the conversation\n", + "res = user_proxy.initiate_chat(\n", + " chatbot,\n", + " message=\"What's the weather in New York and can you tell me how much is 123.45 EUR in USD so I can spend it on my holiday? Throw a few holiday tips in as well.\",\n", + " summary_method=\"reflection_with_llm\",\n", + ")\n", + "\n", + "end_time = time.time()\n", + "\n", + "print(f\"LLM SUMMARY: {res.summary['content']}\\n\\nDuration: {(end_time - start_time) * 1000}ms\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see that the Cerebras Wafer-Scale Engine-3 (WSE-3) completed the query in 74ms -- faster than the blink of an eye!" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3 (ipykernel)", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.8.13" + } + }, + "nbformat": 4, + "nbformat_minor": 4 +} diff --git a/website/docs/topics/non-openai-models/cloud-cohere.ipynb b/website/docs/topics/non-openai-models/cloud-cohere.ipynb index 73dcc54a75ed..defddf983c3b 100644 --- a/website/docs/topics/non-openai-models/cloud-cohere.ipynb +++ b/website/docs/topics/non-openai-models/cloud-cohere.ipynb @@ -25,10 +25,10 @@ "\n", "## Getting started\n", "\n", - "First you need to install the `pyautogen` package to use AutoGen with the Cohere API library.\n", + "First you need to install the `autogen-agentchat~=0.2` package to use AutoGen with the Cohere API library.\n", "\n", "``` bash\n", - "pip install pyautogen[cohere]\n", + "pip install autogen-agentchat[cohere]~=0.2\n", "```" ] }, diff --git a/website/docs/topics/non-openai-models/cloud-gemini.ipynb b/website/docs/topics/non-openai-models/cloud-gemini.ipynb index a227582c592c..673d8a80f31e 100644 --- a/website/docs/topics/non-openai-models/cloud-gemini.ipynb +++ b/website/docs/topics/non-openai-models/cloud-gemini.ipynb @@ -11,7 +11,7 @@ "Install AutoGen with Gemini features:\n", "\n", "```bash\n", - "pip install pyautogen[gemini]\n", + "pip install autogen-agentchat[gemini]~=0.2\n", "```\n", "\n", "## Dependencies of This Notebook\n", @@ -19,7 +19,7 @@ "In this notebook, we will explore how to use Gemini in AutoGen alongside other tools. Install the necessary dependencies with the following command:\n", "\n", "```bash\n", - "pip install pyautogen[gemini,retrievechat,lmm]\n", + "pip install autogen-agentchat[gemini,retrievechat,lmm]~=0.2\n", "```\n", "\n", "## Features\n", @@ -495,7 +495,7 @@ "user_proxy.initiate_chat(\n", " image_agent,\n", " message=\"\"\"Describe what is in this image?\n", - ".\"\"\",\n", + ".\"\"\",\n", ")" ] }, diff --git a/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb index 637d340dc37f..2389092e8dc9 100644 --- a/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb +++ b/website/docs/topics/non-openai-models/cloud-gemini_vertexai.ipynb @@ -16,7 +16,7 @@ "\n", "Install AutoGen with Gemini features:\n", "```bash\n", - "pip install pyautogen[gemini]\n", + "pip install autogen-agentchat[gemini]~=0.2\n", "```\n", "\n", "### Install other Dependencies of this Notebook\n", @@ -392,7 +392,7 @@ "user_proxy.initiate_chat(\n", " image_agent,\n", " message=\"\"\"Describe what is in this image?\n", - ".\"\"\",\n", + ".\"\"\",\n", ")" ] }, diff --git a/website/docs/topics/non-openai-models/cloud-mistralai.ipynb b/website/docs/topics/non-openai-models/cloud-mistralai.ipynb index 1228f96db4e5..9babddf601f3 100644 --- a/website/docs/topics/non-openai-models/cloud-mistralai.ipynb +++ b/website/docs/topics/non-openai-models/cloud-mistralai.ipynb @@ -25,10 +25,10 @@ "\n", "## Getting started\n", "\n", - "First you need to install the `pyautogen` package to use AutoGen with the Mistral API library.\n", + "First you need to install the `autogen-agentchat~=0.2` package to use AutoGen with the Mistral API library.\n", "\n", "``` bash\n", - "pip install pyautogen[mistral]\n", + "pip install autogen-agentchat[mistral]~=0.2\n", "```" ] }, diff --git a/website/docs/topics/non-openai-models/cloud-togetherai.ipynb b/website/docs/topics/non-openai-models/cloud-togetherai.ipynb index eccc372ce2e9..6ec9f52bd7fd 100644 --- a/website/docs/topics/non-openai-models/cloud-togetherai.ipynb +++ b/website/docs/topics/non-openai-models/cloud-togetherai.ipynb @@ -23,10 +23,10 @@ "\n", "## Getting started\n", "\n", - "First, you need to install the `pyautogen` package to use AutoGen with the Together.AI API library.\n", + "First, you need to install the `autogen-agentchat~=0.2` package to use AutoGen with the Together.AI API library.\n", "\n", "``` bash\n", - "pip install pyautogen[together]\n", + "pip install autogen-agentchat[together]~=0.2\n", "```" ] }, diff --git a/website/docs/topics/non-openai-models/local-ollama.ipynb b/website/docs/topics/non-openai-models/local-ollama.ipynb new file mode 100644 index 000000000000..95803e50e593 --- /dev/null +++ b/website/docs/topics/non-openai-models/local-ollama.ipynb @@ -0,0 +1,631 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Ollama\n", + "\n", + "[Ollama](https://ollama.com/) is a local inference engine that enables you to run open-weight LLMs in your environment. It has native support for a large number of models such as Google's Gemma, Meta's Llama 2/3/3.1, Microsoft's Phi 3, Mistral.AI's Mistral/Mixtral, and Cohere's Command R models.\n", + "\n", + "Note: Previously, to use Ollama with AutoGen you required LiteLLM. Now it can be used directly and supports tool calling." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Features\n", + "\n", + "When using this Ollama client class, messages are tailored to accommodate the specific requirements of Ollama's API and this includes message role sequences, support for function/tool calling, and token usage.\n", + "\n", + "## Installing Ollama\n", + "\n", + "For Mac and Windows, [download Ollama](https://ollama.com/download).\n", + "\n", + "For Linux:\n", + "\n", + "```bash\n", + "curl -fsSL https://ollama.com/install.sh | sh\n", + "```\n", + "\n", + "## Downloading models for Ollama\n", + "\n", + "Ollama has a library of models to choose from, see them [here](https://ollama.com/library).\n", + "\n", + "Before you can use a model, you need to download it (using the name of the model from the library):\n", + "\n", + "```bash\n", + "ollama pull llama3.1\n", + "```\n", + "\n", + "To view the models you have downloaded and can use:\n", + "\n", + "```bash\n", + "ollama list\n", + "```\n", + "\n", + "## Getting started with AutoGen and Ollama\n", + "\n", + "When installing AutoGen, you need to install the `pyautogen` package with the Ollama library.\n", + "\n", + "``` bash\n", + "pip install pyautogen[ollama]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "See the sample `OAI_CONFIG_LIST` below showing how the Ollama client class is used by specifying the `api_type` as `ollama`.\n", + "\n", + "```python\n", + "[\n", + " {\n", + " \"model\": \"llama3.1\",\n", + " \"api_type\": \"ollama\"\n", + " },\n", + " {\n", + " \"model\": \"llama3.1:8b-instruct-q6_K\",\n", + " \"api_type\": \"ollama\"\n", + " },\n", + " {\n", + " \"model\": \"mistral-nemo\",\n", + " \"api_type\": \"ollama\"\n", + " }\n", + "]\n", + "```\n", + "\n", + "If you need to specify the URL for your Ollama install, use the `client_host` key in your config as per the below example:\n", + "\n", + "```python\n", + "[\n", + " {\n", + " \"model\": \"llama3.1\",\n", + " \"api_type\": \"ollama\",\n", + " \"client_host\": \"http://192.168.0.1:11434\"\n", + " }\n", + "]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## API parameters\n", + "\n", + "The following Ollama parameters can be added to your config. See [this link](https://github.com/ollama/ollama/blob/main/docs/api.md#parameters) for further information on them.\n", + "\n", + "- num_predict (integer): -1 is infinite, -2 is fill context, 128 is default\n", + "- repeat_penalty (float)\n", + "- seed (integer)\n", + "- stream (boolean)\n", + "- temperature (float)\n", + "- top_k (int)\n", + "- top_p (float)\n", + "\n", + "Example:\n", + "```python\n", + "[\n", + " {\n", + " \"model\": \"llama3.1:instruct\",\n", + " \"api_type\": \"ollama\",\n", + " \"num_predict\": -1,\n", + " \"repeat_penalty\": 1.1,\n", + " \"seed\": 42,\n", + " \"stream\": False,\n", + " \"temperature\": 1,\n", + " \"top_k\": 50,\n", + " \"top_p\": 0.8\n", + " }\n", + "]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Two-Agent Coding Example\n", + "\n", + "In this example, we run a two-agent chat with an AssistantAgent (primarily a coding agent) to generate code to count the number of prime numbers between 1 and 10,000 and then it will be executed.\n", + "\n", + "We'll use Meta's Llama 3.1 model which is suitable for coding.\n", + "\n", + "In this example we will specify the URL for the Ollama installation using `client_host`." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "config_list = [\n", + " {\n", + " # Let's choose the Meta's Llama 3.1 model (model names must match Ollama exactly)\n", + " \"model\": \"llama3.1:8b\",\n", + " # We specify the API Type as 'ollama' so it uses the Ollama client class\n", + " \"api_type\": \"ollama\",\n", + " \"stream\": False,\n", + " \"client_host\": \"http://192.168.0.1:11434\",\n", + " }\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Importantly, we have tweaked the system message so that the model doesn't return the termination keyword, which we've changed to FINISH, with the code block." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/usr/local/lib/python3.11/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n", + " from .autonotebook import tqdm as notebook_tqdm\n" + ] + } + ], + "source": [ + "from pathlib import Path\n", + "\n", + "from autogen import AssistantAgent, UserProxyAgent\n", + "from autogen.coding import LocalCommandLineCodeExecutor\n", + "\n", + "# Setting up the code executor\n", + "workdir = Path(\"coding\")\n", + "workdir.mkdir(exist_ok=True)\n", + "code_executor = LocalCommandLineCodeExecutor(work_dir=workdir)\n", + "\n", + "# Setting up the agents\n", + "\n", + "# The UserProxyAgent will execute the code that the AssistantAgent provides\n", + "user_proxy_agent = UserProxyAgent(\n", + " name=\"User\",\n", + " code_execution_config={\"executor\": code_executor},\n", + " is_termination_msg=lambda msg: \"FINISH\" in msg.get(\"content\"),\n", + ")\n", + "\n", + "system_message = \"\"\"You are a helpful AI assistant who writes code and the user\n", + "executes it. Solve tasks using your python coding skills.\n", + "In the following cases, suggest python code (in a python coding block) for the\n", + "user to execute. When using code, you must indicate the script type in the code block.\n", + "You only need to create one working sample.\n", + "Do not suggest incomplete code which requires users to modify it.\n", + "Don't use a code block if it's not intended to be executed by the user. Don't\n", + "include multiple code blocks in one response. Do not ask users to copy and\n", + "paste the result. Instead, use 'print' function for the output when relevant.\n", + "Check the execution result returned by the user.\n", + "\n", + "If the result indicates there is an error, fix the error.\n", + "\n", + "IMPORTANT: If it has executed successfully, ONLY output 'FINISH'.\"\"\"\n", + "\n", + "# The AssistantAgent, using the Ollama config, will take the coding request and return code\n", + "assistant_agent = AssistantAgent(\n", + " name=\"Ollama Assistant\",\n", + " system_message=system_message,\n", + " llm_config={\"config_list\": config_list},\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can now start the chat." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33mUser\u001b[0m (to Ollama Assistant):\n", + "\n", + "Provide code to count the number of prime numbers from 1 to 10000.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mOllama Assistant\u001b[0m (to User):\n", + "\n", + "```python\n", + "def is_prime(n):\n", + " if n <= 1:\n", + " return False\n", + " for i in range(2, int(n**0.5) + 1):\n", + " if n % i == 0:\n", + " return False\n", + " return True\n", + "\n", + "count = sum(is_prime(i) for i in range(1, 10001))\n", + "print(count)\n", + "```\n", + "\n", + "Please execute this code. I will wait for the result.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> NO HUMAN INPUT RECEIVED.\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> USING AUTO REPLY...\u001b[0m\n", + "\u001b[31m\n", + ">>>>>>>> EXECUTING CODE BLOCK (inferred language is python)...\u001b[0m\n", + "\u001b[33mUser\u001b[0m (to Ollama Assistant):\n", + "\n", + "exitcode: 0 (execution succeeded)\n", + "Code output: 1229\n", + "\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mOllama Assistant\u001b[0m (to User):\n", + "\n", + "FINISH\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[31m\n", + ">>>>>>>> NO HUMAN INPUT RECEIVED.\u001b[0m\n" + ] + } + ], + "source": [ + "# Start the chat, with the UserProxyAgent asking the AssistantAgent the message\n", + "chat_result = user_proxy_agent.initiate_chat(\n", + " assistant_agent,\n", + " message=\"Provide code to count the number of prime numbers from 1 to 10000.\",\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tool Calling - Native vs Manual\n", + "\n", + "Ollama supports native tool calling (Ollama v0.3.1 library onward). If you install AutoGen with `pip install pyautogen[ollama]` you will be able to use native tool calling.\n", + "\n", + "The parameter `native_tool_calls` in your configuration allows you to specify if you want to use Ollama's native tool calling (default) or manual tool calling.\n", + "\n", + "```python\n", + "[\n", + " {\n", + " \"model\": \"llama3.1\",\n", + " \"api_type\": \"ollama\",\n", + " \"client_host\": \"http://192.168.0.1:11434\",\n", + " \"native_tool_calls\": True # Use Ollama's native tool calling, False for manual\n", + " }\n", + "]\n", + "```\n", + "\n", + "Native tool calling only works with certain models and an exception will be thrown if you try to use it with an unsupported model.\n", + "\n", + "Manual tool calling allows you to use tool calling with any Ollama model. It incorporates guided tool calling messages into the prompt that guide the LLM through the process of selecting a tool and then evaluating the result of the tool. As to be expected, the ability to follow instructions and return formatted JSON is highly dependent on the model.\n", + "\n", + "You can tailor the manual tool calling messages by adding these parameters to your configuration:\n", + "\n", + "- `manual_tool_call_instruction`\n", + "- `manual_tool_call_step1`\n", + "- `manual_tool_call_step2`\n", + "\n", + "To use manual tool calling set `native_tool_calls` to `False`." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Reducing repetitive tool calls\n", + "\n", + "By incorporating tools into a conversation, LLMs can often continually recommend them to be called, even after they've been called and a result returned. This can lead to a never ending cycle of tool calls.\n", + "\n", + "To remove the chance of an LLM recommending a tool call, an additional parameter called `hide_tools` can be used to specify when tools are hidden from the LLM. The string values for the parameter are:\n", + "\n", + "- 'never': tools are never hidden\n", + "- 'if_all_run': tools are hidden if all tools have been called\n", + "- 'if_any_run': tools are hidden if any tool has been called\n", + "\n", + "This can be used with native or manual tool calling, an example of a configuration is shown below.\n", + "\n", + "```python\n", + "[\n", + " {\n", + " \"model\": \"llama3.1\",\n", + " \"api_type\": \"ollama\",\n", + " \"client_host\": \"http://192.168.0.1:11434\",\n", + " \"native_tool_calls\": True,\n", + " \"hide_tools\": \"if_any_run\" # Hide tools once any tool has been called\n", + " }\n", + "]\n", + "```" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Tool Call Example\n", + "\n", + "In this example, instead of writing code, we will have an agent assist with some trip planning using multiple tool calling.\n", + "\n", + "Again, we'll use Meta's versatile Llama 3.1.\n", + "\n", + "Native Ollama tool calling will be used and we'll utilise the `hide_tools` parameter to hide the tools once all have been called." + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "import json\n", + "from typing import Literal\n", + "\n", + "from typing_extensions import Annotated\n", + "\n", + "import autogen\n", + "\n", + "config_list = [\n", + " {\n", + " # Let's choose the Meta's Llama 3.1 model (model names must match Ollama exactly)\n", + " \"model\": \"llama3.1:8b\",\n", + " \"api_type\": \"ollama\",\n", + " \"stream\": False,\n", + " \"client_host\": \"http://192.168.0.1:11434\",\n", + " \"hide_tools\": \"if_any_run\",\n", + " }\n", + "]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We'll create our agents. Importantly, we're using native Ollama tool calling and to help guide it we add the JSON to the system_message so that the number fields aren't wrapped in quotes (becoming strings)." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "# Create the agent for tool calling\n", + "chatbot = autogen.AssistantAgent(\n", + " name=\"chatbot\",\n", + " system_message=\"\"\"For currency exchange and weather forecasting tasks,\n", + " only use the functions you have been provided with.\n", + " Example of the return JSON is:\n", + " {\n", + " \"parameter_1_name\": 100.00,\n", + " \"parameter_2_name\": \"ABC\",\n", + " \"parameter_3_name\": \"DEF\",\n", + " }.\n", + " Another example of the return JSON is:\n", + " {\n", + " \"parameter_1_name\": \"GHI\",\n", + " \"parameter_2_name\": \"ABC\",\n", + " \"parameter_3_name\": \"DEF\",\n", + " \"parameter_4_name\": 123.00,\n", + " }.\n", + " Output 'HAVE FUN!' when an answer has been provided.\"\"\",\n", + " llm_config={\"config_list\": config_list},\n", + ")\n", + "\n", + "# Note that we have changed the termination string to be \"HAVE FUN!\"\n", + "user_proxy = autogen.UserProxyAgent(\n", + " name=\"user_proxy\",\n", + " is_termination_msg=lambda x: x.get(\"content\", \"\") and \"HAVE FUN!\" in x.get(\"content\", \"\"),\n", + " human_input_mode=\"NEVER\",\n", + " max_consecutive_auto_reply=1,\n", + ")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Create and register our functions (tools). See the [tutorial chapter on tool use](/docs/tutorial/tool-use) \n", + "for more information." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# Currency Exchange function\n", + "\n", + "CurrencySymbol = Literal[\"USD\", \"EUR\"]\n", + "\n", + "# Define our function that we expect to call\n", + "\n", + "\n", + "def exchange_rate(base_currency: CurrencySymbol, quote_currency: CurrencySymbol) -> float:\n", + " if base_currency == quote_currency:\n", + " return 1.0\n", + " elif base_currency == \"USD\" and quote_currency == \"EUR\":\n", + " return 1 / 1.1\n", + " elif base_currency == \"EUR\" and quote_currency == \"USD\":\n", + " return 1.1\n", + " else:\n", + " raise ValueError(f\"Unknown currencies {base_currency}, {quote_currency}\")\n", + "\n", + "\n", + "# Register the function with the agent\n", + "\n", + "\n", + "@user_proxy.register_for_execution()\n", + "@chatbot.register_for_llm(description=\"Currency exchange calculator.\")\n", + "def currency_calculator(\n", + " base_amount: Annotated[\n", + " float,\n", + " \"Amount of currency in base_currency. Type is float, not string, return value should be a number only, e.g. 987.65.\",\n", + " ],\n", + " base_currency: Annotated[CurrencySymbol, \"Base currency\"] = \"USD\",\n", + " quote_currency: Annotated[CurrencySymbol, \"Quote currency\"] = \"EUR\",\n", + ") -> str:\n", + " quote_amount = exchange_rate(base_currency, quote_currency) * base_amount\n", + " return f\"{format(quote_amount, '.2f')} {quote_currency}\"\n", + "\n", + "\n", + "# Weather function\n", + "\n", + "\n", + "# Example function to make available to model\n", + "def get_current_weather(location, unit=\"fahrenheit\"):\n", + " \"\"\"Get the weather for some location\"\"\"\n", + " if \"chicago\" in location.lower():\n", + " return json.dumps({\"location\": \"Chicago\", \"temperature\": \"13\", \"unit\": unit})\n", + " elif \"san francisco\" in location.lower():\n", + " return json.dumps({\"location\": \"San Francisco\", \"temperature\": \"55\", \"unit\": unit})\n", + " elif \"new york\" in location.lower():\n", + " return json.dumps({\"location\": \"New York\", \"temperature\": \"11\", \"unit\": unit})\n", + " else:\n", + " return json.dumps({\"location\": location, \"temperature\": \"unknown\"})\n", + "\n", + "\n", + "# Register the function with the agent\n", + "\n", + "\n", + "@user_proxy.register_for_execution()\n", + "@chatbot.register_for_llm(description=\"Weather forecast for US cities.\")\n", + "def weather_forecast(\n", + " location: Annotated[str, \"City name\"],\n", + ") -> str:\n", + " weather_details = get_current_weather(location=location)\n", + " weather = json.loads(weather_details)\n", + " return f\"{weather['location']} will be {weather['temperature']} degrees {weather['unit']}\"" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "And run it!" + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "What's the weather in New York and can you tell me how much is 123.45 EUR in USD so I can spend it on my holiday? Throw a few holiday tips in as well.\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mchatbot\u001b[0m (to user_proxy):\n", + "\n", + "\n", + "\u001b[32m***** Suggested tool call (ollama_func_4506): weather_forecast *****\u001b[0m\n", + "Arguments: \n", + "{\"location\": \"New York\"}\n", + "\u001b[32m********************************************************************\u001b[0m\n", + "\u001b[32m***** Suggested tool call (ollama_func_4507): currency_calculator *****\u001b[0m\n", + "Arguments: \n", + "{\"base_amount\": 123.45, \"base_currency\": \"EUR\", \"quote_currency\": \"USD\"}\n", + "\u001b[32m***********************************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION weather_forecast...\u001b[0m\n", + "\u001b[35m\n", + ">>>>>>>> EXECUTING FUNCTION currency_calculator...\u001b[0m\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[32m***** Response from calling tool (ollama_func_4506) *****\u001b[0m\n", + "New York will be 11 degrees fahrenheit\n", + "\u001b[32m*********************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33muser_proxy\u001b[0m (to chatbot):\n", + "\n", + "\u001b[32m***** Response from calling tool (ollama_func_4507) *****\u001b[0m\n", + "135.80 USD\n", + "\u001b[32m*********************************************************\u001b[0m\n", + "\n", + "--------------------------------------------------------------------------------\n", + "\u001b[33mchatbot\u001b[0m (to user_proxy):\n", + "\n", + "Based on the results, it seems that:\n", + "\n", + "* The weather forecast for New York is expected to be around 11 degrees Fahrenheit.\n", + "* The exchange rate for EUR to USD is currently 1 EUR = 1.3580 USD, so 123.45 EUR is equivalent to approximately 135.80 USD.\n", + "\n", + "As a bonus, here are some holiday tips in New York:\n", + "\n", + "* Be sure to try a classic New York-style hot dog from a street cart or a diner.\n", + "* Explore the iconic Central Park and take a stroll through the High Line for some great views of the city.\n", + "* Catch a Broadway show or a concert at one of the many world-class venues in the city.\n", + "\n", + "And... HAVE FUN!\n", + "\n", + "--------------------------------------------------------------------------------\n", + "LLM SUMMARY: The weather forecast for New York is expected to be around 11 degrees Fahrenheit.\n", + "123.45 EUR is equivalent to approximately 135.80 USD.\n", + "Try a classic New York-style hot dog, explore Central Park and the High Line, and catch a Broadway show or concert during your visit.\n" + ] + } + ], + "source": [ + "# start the conversation\n", + "res = user_proxy.initiate_chat(\n", + " chatbot,\n", + " message=\"What's the weather in New York and can you tell me how much is 123.45 EUR in USD so I can spend it on my holiday? Throw a few holiday tips in as well.\",\n", + " summary_method=\"reflection_with_llm\",\n", + ")\n", + "\n", + "print(f\"LLM SUMMARY: {res.summary['content']}\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Great, we can see that Llama 3.1 has helped choose the right functions, their parameters, and then summarised them for us." + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.11.9" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/website/docs/topics/prompting-and-reasoning/react.ipynb b/website/docs/topics/prompting-and-reasoning/react.ipynb index 08f309133488..7663ebc156ff 100644 --- a/website/docs/topics/prompting-and-reasoning/react.ipynb +++ b/website/docs/topics/prompting-and-reasoning/react.ipynb @@ -26,7 +26,7 @@ "metadata": {}, "outputs": [], "source": [ - "! pip install \"pyautogen>=0.2.18\" \"tavily-python\"" + "! pip install \"autogen-agentchat~=0.2\" \"tavily-python\"" ] }, { diff --git a/website/docs/topics/prompting-and-reasoning/reflection.ipynb b/website/docs/topics/prompting-and-reasoning/reflection.ipynb index 604389044725..bb6fa50b2890 100644 --- a/website/docs/topics/prompting-and-reasoning/reflection.ipynb +++ b/website/docs/topics/prompting-and-reasoning/reflection.ipynb @@ -18,7 +18,7 @@ "id": "5cff1938", "metadata": {}, "source": [ - "First make sure the `pyautogen` package is installed." + "First make sure the `autogen-agentchat` package is installed." ] }, { @@ -28,7 +28,7 @@ "metadata": {}, "outputs": [], "source": [ - "! pip install \"pyautogen>=0.2.18\"" + "! pip install \"autogen-agentchat~=0.2\"" ] }, { diff --git a/website/docs/topics/retrieval_augmentation.md b/website/docs/topics/retrieval_augmentation.md index 7fdd39f31f92..3fa9d5754fd6 100644 --- a/website/docs/topics/retrieval_augmentation.md +++ b/website/docs/topics/retrieval_augmentation.md @@ -124,10 +124,11 @@ ragproxyagent.initiate_chat( ## More Examples and Notebooks For more detailed examples and notebooks showcasing the usage of retrieval augmented agents in AutoGen, refer to the following: - Automated Code Generation and Question Answering with Retrieval Augmented Agents - [View Notebook](/docs/notebooks/agentchat_RetrieveChat) -- Automated Code Generation and Question Answering with [PGVector](https://github.com/pgvector/pgvector) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_pgvector.ipynb) -- Automated Code Generation and Question Answering with [Qdrant](https://qdrant.tech/) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_qdrant.ipynb) -- Automated Code Generation and Question Answering with [MongoDB Atlas](https://www.mongodb.com/) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_RetrieveChat_mongodb.ipynb) -- Chat with OpenAI Assistant with Retrieval Augmentation - [View Notebook](https://github.com/microsoft/autogen/blob/main/notebook/agentchat_oai_assistant_retrieval.ipynb) +- Automated Code Generation and Question Answering with [PGVector](https://github.com/pgvector/pgvector) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat_pgvector.ipynb) +- Automated Code Generation and Question Answering with [Qdrant](https://qdrant.tech/) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat_qdrant.ipynb) +- Automated Code Generation and Question Answering with [MongoDB Atlas](https://www.mongodb.com/) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat_mongodb.ipynb) +- Automated Code Generation and Question Answering with [Couchbase](https://www.couchbase.com/) based Retrieval Augmented Agents - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_RetrieveChat_couchbase.ipynb) +- Chat with OpenAI Assistant with Retrieval Augmentation - [View Notebook](https://github.com/microsoft/autogen/blob/0.2/notebook/agentchat_oai_assistant_retrieval.ipynb) - **RAG**: Group Chat with Retrieval Augmented Generation (with 5 group member agents and 1 manager agent) - [View Notebook](/docs/notebooks/agentchat_groupchat_RAG) ## Roadmap diff --git a/website/docs/topics/task_decomposition.ipynb b/website/docs/topics/task_decomposition.ipynb index e4c24c9004ef..ffb18ebdee0e 100644 --- a/website/docs/topics/task_decomposition.ipynb +++ b/website/docs/topics/task_decomposition.ipynb @@ -16,7 +16,7 @@ "id": "a6c436c9", "metadata": {}, "source": [ - "First make sure the `pyautogen` package is installed." + "First make sure the `autogen-agentchat` package is installed." ] }, { @@ -26,7 +26,7 @@ "metadata": {}, "outputs": [], "source": [ - "! pip install \"pyautogen>=0.2.18\"" + "! pip install \"autogen-agentchat~=0.2\"" ] }, { diff --git a/website/docs/tutorial/conversation-patterns.ipynb b/website/docs/tutorial/conversation-patterns.ipynb index 56004e3b3b81..a0c8ddc217cb 100644 --- a/website/docs/tutorial/conversation-patterns.ipynb +++ b/website/docs/tutorial/conversation-patterns.ipynb @@ -728,7 +728,7 @@ "If we were to use the `round_robin` strategy, this list would specify the order\n", "of the agents to be selected.\n", "We also initialize the group chat with an empty message list and a maximum\n", - "round of 6, which means there will be at most 6 iteratiosn of selecting speaker,\n", + "round of 6, which means there will be at most 6 iterations of selecting a speaker,\n", "agent speaks and broadcasting message." ] }, diff --git a/website/docs/tutorial/introduction.ipynb b/website/docs/tutorial/introduction.ipynb index 88df66b7270f..fd5a362d0351 100644 --- a/website/docs/tutorial/introduction.ipynb +++ b/website/docs/tutorial/introduction.ipynb @@ -38,7 +38,7 @@ "source": [ "## Installation\n", "\n", - "The simplest way to install AutoGen is from pip: `pip install pyautogen`. Find more options in [Installation](/docs/installation/)." + "The simplest way to install AutoGen is from pip: `pip install autogen-agentchat~=0.2`. Find more options in [Installation](/docs/installation/)." ] }, { diff --git a/website/docs/tutorial/what-next.md b/website/docs/tutorial/what-next.md index d9a0062e8ca9..ed1542a56912 100644 --- a/website/docs/tutorial/what-next.md +++ b/website/docs/tutorial/what-next.md @@ -32,8 +32,7 @@ topics: ## Get Help If you have any questions, you can ask in our [GitHub -Discussions](https://github.com/microsoft/autogen/discussions), or join -our [Discord Server](https://aka.ms/autogen-dc). +Discussions](https://github.com/microsoft/autogen/discussions). [![](https://img.shields.io/discord/1153072414184452236?logo=discord&style=flat.png)](https://aka.ms/autogen-dc) diff --git a/website/docusaurus.config.js b/website/docusaurus.config.js index 2ae1a581ce6e..1a2de465f1f6 100644 --- a/website/docusaurus.config.js +++ b/website/docusaurus.config.js @@ -16,7 +16,7 @@ module.exports = { title: "AutoGen", tagline: "An Open-Source Programming Framework for Agentic AI", url: "https://microsoft.github.io", - baseUrl: "/autogen/", + baseUrl: "/autogen/0.2/", onBrokenLinks: "throw", onBrokenMarkdownLinks: "warn", favicon: "img/ag.ico", @@ -145,11 +145,6 @@ module.exports = { label: "GitHub", position: "right", }, - { - href: "https://aka.ms/autogen-dc", - label: "Discord", - position: "right", - }, { href: "https://twitter.com/pyautogen", label: "Twitter", @@ -177,8 +172,8 @@ module.exports = { // // href: 'https://stackoverflow.com/questions/tagged/pymarlin', // // }, { - label: "Discord", - href: "https://aka.ms/autogen-dc", + label: "GitHub Discussion", + href: "https://github.com/microsoft/autogen/discussions", }, { label: "Twitter", @@ -189,14 +184,14 @@ module.exports = { ], copyright: `Copyright © ${new Date().getFullYear()} AutoGen Authors | Privacy and Cookies | Consumer Health Privacy`, }, - // announcementBar: { - // id: "whats_new", - // content: - // 'What\'s new in AutoGen? Read this blog for an overview of updates', - // backgroundColor: "#fafbfc", - // textColor: "#091E42", - // isCloseable: true, - // }, + announcementBar: { + id: "newdocs", + content: + 'Go here to view the documentation for the work in progress version of AutoGen 0.4.', + backgroundColor: "#fafbfc", + textColor: "#091E42", + isCloseable: true, + }, /* Clarity Config */ clarity: { ID: "lnxpe6skj1", // The Tracking ID provided by Clarity diff --git a/website/process_notebooks.py b/website/process_notebooks.py index 5e25f5e69510..a3157363ab82 100755 --- a/website/process_notebooks.py +++ b/website/process_notebooks.py @@ -348,7 +348,7 @@ def post_process_mdx(rendered_mdx: Path, source_notebooks: Path, front_matter: D if "{" in title: title = title[: title.find("{")].strip() - github_link = f"https://github.com/microsoft/autogen/blob/main/{repo_relative_notebook}" + github_link = f"https://github.com/microsoft/autogen/blob/0.2/{repo_relative_notebook}" content = ( content[:title_end] + "\n[![Open on GitHub](https://img.shields.io/badge/Open%20on%20GitHub-grey?logo=github)](" diff --git a/website/src/components/GalleryPage.js b/website/src/components/GalleryPage.js index 11f4abe2e7fe..aec7907b41ef 100644 --- a/website/src/components/GalleryPage.js +++ b/website/src/components/GalleryPage.js @@ -91,7 +91,7 @@ const GalleryPage = (props) => { return null; } const colab_href = `https://colab.research.google.com/github/microsoft/autogen/blob/main/${item.source}`; - const github_href = `https://github.com/microsoft/autogen/blob/main/${item.source}`; + const github_href = `https://github.com/microsoft/autogen/blob/0.2/${item.source}`; return ( Open In Colab Static Badge diff --git a/website/src/pages/index.js b/website/src/pages/index.js index 203c72c02223..99d423e03705 100644 --- a/website/src/pages/index.js +++ b/website/src/pages/index.js @@ -1,24 +1,44 @@ -import React from 'react'; -import clsx from 'clsx'; -import Layout from '@theme/Layout'; -import Link from '@docusaurus/Link'; -import useDocusaurusContext from '@docusaurus/useDocusaurusContext'; -import styles from './index.module.css'; -import HomepageFeatures from '../components/HomepageFeatures'; +import React from "react"; +import clsx from "clsx"; +import Layout from "@theme/Layout"; +import Link from "@docusaurus/Link"; +import useDocusaurusContext from "@docusaurus/useDocusaurusContext"; +import styles from "./index.module.css"; +import HomepageFeatures from "../components/HomepageFeatures"; function HomepageHeader() { - const {siteConfig} = useDocusaurusContext(); + const { siteConfig } = useDocusaurusContext(); return ( -
+

{siteConfig.title}

{siteConfig.tagline}

- - Getting Started - 3min ⏱️ - +
+ + Get Started + +

+ Current stable version of AutoGen (autogen-agentchat~=0.2) +

+
+
+ + Preview v0.4 + +

+ A new event driven, asynchronous architecture for AutoGen +

+
@@ -26,11 +46,12 @@ function HomepageHeader() { } export default function Home() { - const {siteConfig} = useDocusaurusContext(); + const { siteConfig } = useDocusaurusContext(); return ( + description="Enabling Next-Gen LLM Applications via Multi-Agent Conversation Framework" + >
diff --git a/website/src/pages/index.module.css b/website/src/pages/index.module.css index 5c0d4f6e20c4..883ab870aaea 100644 --- a/website/src/pages/index.module.css +++ b/website/src/pages/index.module.css @@ -22,4 +22,25 @@ display: flex; align-items: center; justify-content: center; + gap: 10px; +} + +.buttonWrapper { + display: flex; + flex-direction: column; + align-items: center; + width: 200px; +} + +.buttonLink { + width: 100%; +} + +.buttonTagline { + font-size: 0.8rem; + margin-top: 5px; + text-align: center; + height: 100px; + overflow-y: auto; + padding: 5px; }