From a9171211c7533fc11e078899720a3847f45807cc Mon Sep 17 00:00:00 2001 From: Wael Karkoub Date: Mon, 29 Apr 2024 01:27:34 +0100 Subject: [PATCH] Streamline Testing with `pytest-cov` and `pytest` Defaults (#2490) * done * update docs * try fix * update workflows * undo minor fix * resolve comments * adds back pytest-asyncio * minor fix * add branch coverage * restore pip install e. * test with coverage * fix mypy * fix coverage + docker + windows combo * fix bash command * formatter * formatter * one last fix * I lied, last fix * fix * fix retrieve chat test * fix windows paths * change cache seed * down grade openai version * fix openai mypy * better error type * fix image gen cache test * fix * experimenting * fix lmm * skip cosmos test * remove cosmos db * unused imports * handle more cosmosdb skips * fix flaky test --- .github/workflows/build.yml | 7 +- .github/workflows/contrib-openai.yml | 200 +++++++++--------- .github/workflows/contrib-tests.yml | 62 +++--- .github/workflows/openai.yml | 11 +- .github/workflows/samples-tools-tests.yml | 2 +- .github/workflows/type-check.yml | 4 +- autogen/cache/cache_factory.py | 4 +- autogen/code_utils.py | 2 +- autogen/oai/openai_utils.py | 6 +- pyproject.toml | 39 ++-- setup.py | 4 +- test/cache/test_cache.py | 11 +- test/cache/test_cosmos_db_cache.py | 24 ++- test/coding/test_commandline_code_executor.py | 12 +- test/oai/test_client_stream.py | 17 +- test/test_code_utils.py | 17 +- website/docs/contributor-guide/tests.md | 10 +- 17 files changed, 204 insertions(+), 228 deletions(-) diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index b45de32ddddc..ce3654e58681 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -46,7 +46,7 @@ jobs: python -m pip install --upgrade pip wheel pip install -e .[cosmosdb] python -c "import autogen" - pip install pytest mock + pip install pytest-cov>=5 mock - name: Install optional dependencies for code executors # code executors and udfs auto skip without deps, so only run for python 3.11 if: matrix.python-version == '3.11' @@ -71,12 +71,11 @@ jobs: if: matrix.python-version == '3.10' run: | pip install -e .[test,redis,websockets] - coverage run -a -m pytest test --ignore=test/agentchat/contrib --skip-openai --durations=10 --durations-min=1.0 - coverage xml + pytest test --ignore=test/agentchat/contrib --skip-openai --durations=10 --durations-min=1.0 - name: Test with Cosmos DB run: | pip install -e .[test,cosmosdb] - coverage run -a -m pytest test/cache/test_cosmos_db_cache.py --skip-openai --durations=10 --durations-min=1.0 + pytest test/cache/test_cosmos_db_cache.py --skip-openai --durations=10 --durations-min=1.0 - name: Upload coverage to Codecov if: matrix.python-version == '3.10' uses: codecov/codecov-action@v3 diff --git a/.github/workflows/contrib-openai.yml b/.github/workflows/contrib-openai.yml index 73c2197c27ef..1bf71115d6b4 100644 --- a/.github/workflows/contrib-openai.yml +++ b/.github/workflows/contrib-openai.yml @@ -5,14 +5,15 @@ name: OpenAI4ContribTests on: pull_request: - branches: ['main'] + branches: ["main"] paths: - - 'autogen/**' - - 'test/agentchat/contrib/**' - - '.github/workflows/contrib-openai.yml' - - 'setup.py' -permissions: {} - # actions: read + - "autogen/**" + - "test/agentchat/contrib/**" + - ".github/workflows/contrib-openai.yml" + - "setup.py" +permissions: + {} + # actions: read # checks: read # contents: read # deployments: read @@ -55,7 +56,7 @@ jobs: python -m pip install --upgrade pip wheel pip install -e . python -c "import autogen" - pip install coverage pytest-asyncio + pip install pytest-cov>=5 pytest-asyncio - name: Install packages for test when needed run: | pip install docker @@ -67,8 +68,7 @@ jobs: AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} run: | - coverage run -a -m pytest -k test_retrievechat test/agentchat/contrib/retrievechat - coverage xml + pytest test/agentchat/contrib/retrievechat/ test/agentchat/contrib/retrievechat - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: @@ -98,7 +98,7 @@ jobs: python -m pip install --upgrade pip wheel pip install -e . python -c "import autogen" - pip install coverage pytest-asyncio + pip install pytest-cov>=5 pytest-asyncio - name: Install packages for test when needed run: | pip install docker @@ -109,8 +109,7 @@ jobs: AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} run: | - coverage run -a -m pytest test/agentchat/contrib/test_compressible_agent.py - coverage xml + pytest test/agentchat/contrib/test_compressible_agent.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: @@ -139,7 +138,7 @@ jobs: python -m pip install --upgrade pip wheel pip install -e . python -c "import autogen" - pip install coverage pytest-asyncio + pip install pytest-cov>=5 pytest-asyncio - name: Install packages for test when needed run: | pip install docker @@ -150,8 +149,7 @@ jobs: AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} run: | - coverage run -a -m pytest test/agentchat/contrib/test_gpt_assistant.py - coverage xml + pytest test/agentchat/contrib/test_gpt_assistant.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: @@ -180,7 +178,7 @@ jobs: python -m pip install --upgrade pip wheel pip install -e .[teachable] python -c "import autogen" - pip install coverage pytest + pip install pytest-cov>=5 - name: Coverage env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -188,8 +186,7 @@ jobs: AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} run: | - coverage run -a -m pytest test/agentchat/contrib/capabilities/test_teachable_agent.py - coverage xml + pytest test/agentchat/contrib/capabilities/test_teachable_agent.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: @@ -198,8 +195,8 @@ jobs: AgentBuilder: strategy: matrix: - os: [ ubuntu-latest ] - python-version: [ "3.11" ] + os: [ubuntu-latest] + python-version: ["3.11"] runs-on: ${{ matrix.os }} environment: openai1 steps: @@ -218,7 +215,7 @@ jobs: python -m pip install --upgrade pip wheel pip install -e . python -c "import autogen" - pip install coverage pytest-asyncio + pip install pytest-cov>=5 pytest-asyncio - name: Install packages for test when needed run: | pip install -e .[autobuild] @@ -229,8 +226,7 @@ jobs: AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} run: | - coverage run -a -m pytest test/agentchat/contrib/test_agent_builder.py - coverage xml + pytest test/agentchat/contrib/test_agent_builder.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: @@ -259,7 +255,7 @@ jobs: python -m pip install --upgrade pip wheel pip install -e .[websurfer] python -c "import autogen" - pip install coverage pytest + pip install pytest-cov>=5 - name: Coverage env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -268,93 +264,90 @@ jobs: OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} BING_API_KEY: ${{ secrets.BING_API_KEY }} run: | - coverage run -a -m pytest test/agentchat/contrib/test_web_surfer.py - coverage xml + pytest test/agentchat/contrib/test_web_surfer.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: file: ./coverage.xml flags: unittests ContextHandling: - strategy: - matrix: - os: [ubuntu-latest] - python-version: ["3.11"] - runs-on: ${{ matrix.os }} - environment: openai1 - steps: - # checkout to pr branch - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install packages and dependencies - run: | - docker --version - python -m pip install --upgrade pip wheel - pip install -e . - python -c "import autogen" - pip install coverage pytest - - name: Coverage - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} - AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} - OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} - BING_API_KEY: ${{ secrets.BING_API_KEY }} - run: | - coverage run -a -m pytest test/agentchat/contrib/capabilities/test_context_handling.py - coverage xml - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./coverage.xml - flags: unittests + strategy: + matrix: + os: [ubuntu-latest] + python-version: ["3.11"] + runs-on: ${{ matrix.os }} + environment: openai1 + steps: + # checkout to pr branch + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install packages and dependencies + run: | + docker --version + python -m pip install --upgrade pip wheel + pip install -e . + python -c "import autogen" + pip install pytest-cov>=5 + - name: Coverage + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }} + AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} + OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} + BING_API_KEY: ${{ secrets.BING_API_KEY }} + run: | + pytest test/agentchat/contrib/capabilities/test_context_handling.py + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests ImageGen: - strategy: - matrix: - os: [ubuntu-latest] - python-version: ["3.12"] - runs-on: ${{ matrix.os }} - environment: openai1 - steps: - # checkout to pr branch - - name: Checkout - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v5 - with: - python-version: ${{ matrix.python-version }} - - name: Install packages and dependencies - run: | - docker --version - python -m pip install --upgrade pip wheel - pip install -e .[lmm] - python -c "import autogen" - pip install coverage pytest - - name: Coverage - env: - OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} - run: | - coverage run -a -m pytest test/agentchat/contrib/capabilities/test_image_generation_capability.py - coverage xml - - name: Upload coverage to Codecov - uses: codecov/codecov-action@v3 - with: - file: ./coverage.xml - flags: unittests + strategy: + matrix: + os: [ubuntu-latest] + python-version: ["3.12"] + runs-on: ${{ matrix.os }} + environment: openai1 + steps: + # checkout to pr branch + - name: Checkout + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v5 + with: + python-version: ${{ matrix.python-version }} + - name: Install packages and dependencies + run: | + docker --version + python -m pip install --upgrade pip wheel + pip install -e .[lmm] + python -c "import autogen" + pip install pytest-cov>=5 + - name: Coverage + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + run: | + pytest test/agentchat/contrib/capabilities/test_image_generation_capability.py + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v3 + with: + file: ./coverage.xml + flags: unittests AgentOptimizer: strategy: matrix: - os: [ ubuntu-latest ] - python-version: [ "3.11" ] + os: [ubuntu-latest] + python-version: ["3.11"] runs-on: ${{ matrix.os }} environment: openai1 steps: @@ -373,7 +366,7 @@ jobs: python -m pip install --upgrade pip wheel pip install -e . python -c "import autogen" - pip install coverage pytest + pip install pytest-cov>=5 - name: Coverage env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} @@ -381,8 +374,7 @@ jobs: AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} run: | - coverage run -a -m pytest test/agentchat/contrib/test_agent_optimizer.py - coverage xml + pytest test/agentchat/contrib/test_agent_optimizer.py - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: diff --git a/.github/workflows/contrib-tests.yml b/.github/workflows/contrib-tests.yml index 46c8433e1f7f..d36a9d52e692 100644 --- a/.github/workflows/contrib-tests.yml +++ b/.github/workflows/contrib-tests.yml @@ -41,7 +41,7 @@ jobs: - name: Install packages and dependencies for all tests run: | python -m pip install --upgrade pip wheel - pip install pytest + pip install pytest-cov>=5 - name: Install qdrant_client when python-version is 3.10 if: matrix.python-version == '3.10' run: | @@ -57,9 +57,7 @@ jobs: fi - name: Coverage run: | - pip install coverage>=5.3 - coverage run -a -m pytest test/test_retrieve_utils.py test/agentchat/contrib/retrievechat test/agentchat/contrib/vectordb --skip-openai - coverage xml + pytest test/test_retrieve_utils.py test/agentchat/contrib/retrievechat/test_retrievechat.py test/agentchat/contrib/retrievechat/test_qdrant_retrievechat.py test/agentchat/contrib/vectordb --skip-openai - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: @@ -119,9 +117,8 @@ jobs: echo "AUTOGEN_USE_DOCKER=False" >> $GITHUB_ENV - name: Coverage run: | - pip install coverage>=5.3 - coverage run -a -m pytest test/test_retrieve_utils.py test/agentchat/contrib/retrievechat test/agentchat/contrib/vectordb --skip-openai - coverage xml + pip install pytest-cov>=5 + pytest test/test_retrieve_utils.py test/agentchat/contrib/retrievechat test/agentchat/contrib/vectordb --skip-openai - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: @@ -144,7 +141,7 @@ jobs: - name: Install packages and dependencies for all tests run: | python -m pip install --upgrade pip wheel - pip install pytest + pip install pytest-cov>=5 - name: Install packages and dependencies for Compression run: | pip install -e . @@ -156,9 +153,7 @@ jobs: fi - name: Coverage run: | - pip install coverage>=5.3 - coverage run -a -m pytest test/agentchat/contrib/test_compressible_agent.py --skip-openai - coverage xml + pytest test/agentchat/contrib/test_compressible_agent.py --skip-openai - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: @@ -181,7 +176,7 @@ jobs: - name: Install packages and dependencies for all tests run: | python -m pip install --upgrade pip wheel - pip install pytest + pip install pytest-cov>=5 - name: Install packages and dependencies for GPTAssistantAgent run: | pip install -e . @@ -193,9 +188,7 @@ jobs: fi - name: Coverage run: | - pip install coverage>=5.3 - coverage run -a -m pytest test/agentchat/contrib/test_gpt_assistant.py --skip-openai - coverage xml + pytest test/agentchat/contrib/test_gpt_assistant.py --skip-openai - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: @@ -218,7 +211,7 @@ jobs: - name: Install packages and dependencies for all tests run: | python -m pip install --upgrade pip wheel - pip install pytest + pip install pytest-cov>=5 - name: Install packages and dependencies for Teachability run: | pip install -e .[teachable] @@ -230,9 +223,7 @@ jobs: fi - name: Coverage run: | - pip install coverage>=5.3 - coverage run -a -m pytest test/agentchat/contrib/capabilities/test_teachable_agent.py --skip-openai - coverage xml + pytest test/agentchat/contrib/capabilities/test_teachable_agent.py --skip-openai - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: @@ -255,7 +246,7 @@ jobs: - name: Install packages and dependencies for all tests run: | python -m pip install --upgrade pip wheel - pip install pytest + pip install pytest-cov>=5 - name: Install packages and dependencies for WebSurfer run: | pip install -e .[websurfer] @@ -267,9 +258,7 @@ jobs: fi - name: Coverage run: | - pip install coverage>=5.3 - coverage run -a -m pytest test/test_browser_utils.py test/agentchat/contrib/test_web_surfer.py --skip-openai - coverage xml + pytest test/test_browser_utils.py test/agentchat/contrib/test_web_surfer.py --skip-openai - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: @@ -294,7 +283,7 @@ jobs: - name: Install packages and dependencies for all tests run: | python -m pip install --upgrade pip wheel - pip install pytest + pip install pytest-cov>=5 - name: Install packages and dependencies for LMM run: | pip install -e .[lmm] @@ -306,9 +295,11 @@ jobs: fi - name: Coverage run: | - pip install coverage>=5.3 - coverage run -a -m pytest test/agentchat/contrib/test_img_utils.py test/agentchat/contrib/test_lmm.py test/agentchat/contrib/test_llava.py test/agentchat/contrib/capabilities/test_image_generation_capability.py test/agentchat/contrib/capabilities/test_vision_capability.py --skip-openai - coverage xml + pytest test/agentchat/contrib/test_img_utils.py test/agentchat/contrib/test_lmm.py test/agentchat/contrib/test_llava.py test/agentchat/contrib/capabilities/test_vision_capability.py --skip-openai + - name: Image Gen Coverage + if: ${{ matrix.os != 'windows-2019' && matrix.python-version != '3.12' }} + run: | + pytest test/agentchat/contrib/capabilities/test_image_generation_capability.py --skip-openai - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: @@ -336,7 +327,7 @@ jobs: - name: Install packages and dependencies for all tests run: | python -m pip install --upgrade pip wheel - pip install pytest + pip install pytest-cov>=5 - name: Install packages and dependencies for Gemini run: | pip install -e .[gemini,test] @@ -348,8 +339,7 @@ jobs: fi - name: Coverage run: | - coverage run -a -m pytest test/oai/test_gemini.py --skip-openai - coverage xml + pytest test/oai/test_gemini.py --skip-openai - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: @@ -372,7 +362,7 @@ jobs: - name: Install packages and dependencies for all tests run: | python -m pip install --upgrade pip wheel - pip install pytest + pip install pytest-cov>=5 - name: Install packages and dependencies for Context Handling run: | pip install -e . @@ -384,9 +374,7 @@ jobs: fi - name: Coverage run: | - pip install coverage>=5.3 - coverage run -a -m pytest test/agentchat/contrib/capabilities/test_context_handling.py --skip-openai - coverage xml + pytest test/agentchat/contrib/capabilities/test_context_handling.py --skip-openai - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: @@ -409,7 +397,7 @@ jobs: - name: Install packages and dependencies for all tests run: | python -m pip install --upgrade pip wheel - pip install pytest + pip install pytest-cov>=5 - name: Install packages and dependencies for Transform Messages run: | pip install -e . @@ -421,9 +409,7 @@ jobs: fi - name: Coverage run: | - pip install coverage>=5.3 - coverage run -a -m pytest test/agentchat/contrib/capabilities/test_transform_messages.py --skip-openai - coverage xml + pytest test/agentchat/contrib/capabilities/test_transform_messages.py --skip-openai - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: diff --git a/.github/workflows/openai.yml b/.github/workflows/openai.yml index d2780eea5425..a9ab8e9e0c5f 100644 --- a/.github/workflows/openai.yml +++ b/.github/workflows/openai.yml @@ -13,7 +13,8 @@ on: - "notebook/agentchat_function_call.ipynb" - "notebook/agentchat_groupchat_finite_state_machine.ipynb" - ".github/workflows/openai.yml" -permissions: {} +permissions: + {} # actions: read # checks: read # contents: read @@ -49,7 +50,7 @@ jobs: python -m pip install --upgrade pip wheel pip install -e. python -c "import autogen" - pip install coverage pytest-asyncio + pip install pytest-cov>=5 pytest-asyncio - name: Install packages for test when needed if: matrix.python-version == '3.9' run: | @@ -63,8 +64,7 @@ jobs: AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }} OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} run: | - coverage run -a -m pytest test --ignore=test/agentchat/contrib --durations=10 --durations-min=1.0 - coverage xml + pytest test --ignore=test/agentchat/contrib --durations=10 --durations-min=1.0 - name: Coverage and check notebook outputs if: matrix.python-version != '3.9' env: @@ -75,8 +75,7 @@ jobs: OAI_CONFIG_LIST: ${{ secrets.OAI_CONFIG_LIST }} run: | pip install nbconvert nbformat ipykernel - coverage run -a -m pytest test/test_notebook.py --durations=10 --durations-min=1.0 - coverage xml + pytest test/test_notebook.py --durations=10 --durations-min=1.0 cat "$(pwd)/test/executed_openai_notebook_output.txt" - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 diff --git a/.github/workflows/samples-tools-tests.yml b/.github/workflows/samples-tools-tests.yml index af7dc6c47433..e774e5cb0b1f 100644 --- a/.github/workflows/samples-tools-tests.yml +++ b/.github/workflows/samples-tools-tests.yml @@ -37,7 +37,7 @@ jobs: run: | python -m pip install --upgrade pip wheel pip install -e . - pip install pytest + pip install pytest-cov>=5 - name: Set AUTOGEN_USE_DOCKER based on OS shell: bash run: | diff --git a/.github/workflows/type-check.yml b/.github/workflows/type-check.yml index f6896d1145d6..c66fb6ad7b10 100644 --- a/.github/workflows/type-check.yml +++ b/.github/workflows/type-check.yml @@ -1,6 +1,6 @@ name: Type check # see: https://help.github.com/en/actions/reference/events-that-trigger-workflows -on: # Trigger the workflow on pull request or merge +on: # Trigger the workflow on pull request or merge pull_request: merge_group: types: [checks_requested] @@ -19,7 +19,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: ${{ matrix.version }} + python-version: ${{ matrix.version }} # All additional modules should be defined in setup.py - run: pip install ".[types]" # Any additional configuration should be defined in pyproject.toml diff --git a/autogen/cache/cache_factory.py b/autogen/cache/cache_factory.py index 437893570b45..7c9d71884cb6 100644 --- a/autogen/cache/cache_factory.py +++ b/autogen/cache/cache_factory.py @@ -1,4 +1,5 @@ import logging +import os from typing import Any, Dict, Optional, Union from .abstract_cache_base import AbstractCache @@ -74,4 +75,5 @@ def cache_factory( logging.warning("CosmosDBCache is not available. Fallback to DiskCache.") # Default to DiskCache if neither Redis nor Cosmos DB configurations are provided - return DiskCache(f"./{cache_path_root}/{seed}") + path = os.path.join(cache_path_root, str(seed)) + return DiskCache(os.path.join(".", path)) diff --git a/autogen/code_utils.py b/autogen/code_utils.py index 5057c8615ea9..e556497388f2 100644 --- a/autogen/code_utils.py +++ b/autogen/code_utils.py @@ -281,7 +281,7 @@ def in_docker_container() -> bool: return os.path.exists("/.dockerenv") -def decide_use_docker(use_docker) -> bool: +def decide_use_docker(use_docker: Optional[bool]) -> Optional[bool]: if use_docker is None: env_var_use_docker = os.environ.get("AUTOGEN_USE_DOCKER", "True") diff --git a/autogen/oai/openai_utils.py b/autogen/oai/openai_utils.py index 25ac8dae2988..7e738b7bd61e 100644 --- a/autogen/oai/openai_utils.py +++ b/autogen/oai/openai_utils.py @@ -692,7 +692,11 @@ def detect_gpt_assistant_api_version() -> str: def create_gpt_vector_store(client: OpenAI, name: str, fild_ids: List[str]) -> Any: """Create a openai vector store for gpt assistant""" - vector_store = client.beta.vector_stores.create(name=name) + try: + vector_store = client.beta.vector_stores.create(name=name) + except Exception as e: + raise AttributeError(f"Failed to create vector store, please install the latest OpenAI python package: {e}") + # poll the status of the file batch for completion. batch = client.beta.vector_stores.file_batches.create_and_poll(vector_store_id=vector_store.id, file_ids=fild_ids) diff --git a/pyproject.toml b/pyproject.toml index d1851339743b..7981ef4b43d5 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,10 +4,8 @@ description-file = "README.md" [tool.pytest.ini_options] -addopts = '-m "not conda"' -markers = [ - "conda: test related to conda forge distribution" -] +addopts = '--cov=. --cov-append --cov-branch --cov-report=xml -m "not conda"' +markers = ["conda: test related to conda forge distribution"] [tool.black] # https://github.com/psf/black @@ -16,28 +14,20 @@ exclude = "(.eggs|.git|.hg|.mypy_cache|.venv|_build|buck-out|build|dist)" [tool.ruff] - line-length = 120 [tool.ruff.lint] - - # Enable Pyflakes `E` and `F` codes by default. select = [ - "E", "W", # see: https://pypi.org/project/pycodestyle - "F", # see: https://pypi.org/project/pyflakes -# "D", # see: https://pypi.org/project/pydocstyle -# "N", # see: https://pypi.org/project/pep8-naming -# "S", # see: https://pypi.org/project/flake8-bandit - "I", # see: https://pypi.org/project/isort/ -] - -ignore = [ - "E501", - "F401", - "F403", - "C901", + "E", + "W", # see: https://pypi.org/project/pycodestyle + "F", # see: https://pypi.org/project/pyflakes + # "D", # see: https://pypi.org/project/pydocstyle + # "N", # see: https://pypi.org/project/pep8-naming + # "S", # see: https://pypi.org/project/flake8-bandit + "I", # see: https://pypi.org/project/isort/ ] +ignore = ["E501", "F401", "F403", "C901"] # Exclude a variety of commonly ignored directories. exclude = [ @@ -50,7 +40,7 @@ exclude = [ "build", "dist", "docs", - # This file needs to be either upgraded or removed and therefore should be + # This file needs to be either upgraded or removed and therefore should be # ignore from type checking for now "math_utils\\.py$", "**/cap/py/autogencap/proto/*", @@ -63,7 +53,6 @@ unfixable = ["F401"] max-complexity = 10 [tool.mypy] - files = [ "autogen/logger", "autogen/exception_utils.py", @@ -76,12 +65,12 @@ files = [ "test/test_function_utils.py", "test/io", ] - exclude = [ "autogen/math_utils\\.py", "autogen/oai/completion\\.py", "autogen/agentchat/contrib/compressible_agent\\.py", "autogen/agentchat/contrib/math_user_proxy_agent.py", + "autogen/oai/openai_utils.py", ] strict = true @@ -89,9 +78,7 @@ python_version = "3.8" ignore_missing_imports = true install_types = true non_interactive = true -plugins = [ - "pydantic.mypy" -] +plugins = ["pydantic.mypy"] # remove after all files in the repo are fixed follow_imports = "silent" diff --git a/setup.py b/setup.py index 870a10899eca..a93f1de07aaa 100644 --- a/setup.py +++ b/setup.py @@ -14,7 +14,7 @@ __version__ = version["__version__"] install_requires = [ - "openai>=1.23.3", + "openai>=1.3", "diskcache", "termcolor", "flaml", @@ -39,11 +39,11 @@ extra_require = { "test": [ - "coverage>=5.3", "ipykernel", "nbconvert", "nbformat", "pre-commit", + "pytest-cov>=5", "pytest-asyncio", "pytest>=6.1.1,<8", "pandas", diff --git a/test/cache/test_cache.py b/test/cache/test_cache.py index d01b1cf49520..05331c165b52 100755 --- a/test/cache/test_cache.py +++ b/test/cache/test_cache.py @@ -1,16 +1,17 @@ #!/usr/bin/env python3 -m pytest import unittest -from typing import Optional, TypedDict, Union from unittest.mock import ANY, MagicMock, patch try: from azure.cosmos import CosmosClient + + skip_azure = False except ImportError: - CosmosClient = None + CosmosClient = object + skip_azure = True from autogen.cache.cache import Cache -from autogen.cache.cosmos_db_cache import CosmosDBConfig class TestCache(unittest.TestCase): @@ -37,6 +38,7 @@ def test_redis_cache_initialization(self, mock_cache_factory): mock_cache_factory.assert_called() @patch("autogen.cache.cache_factory.CacheFactory.cache_factory", return_value=MagicMock()) + @unittest.skipIf(skip_azure, "requires azure.cosmos") def test_cosmosdb_cache_initialization(self, mock_cache_factory): cache = Cache(self.cosmos_config) self.assertIsInstance(cache.cache, MagicMock) @@ -65,6 +67,7 @@ def context_manager_common(self, config): def test_redis_context_manager(self): self.context_manager_common(self.redis_config) + @unittest.skipIf(skip_azure, "requires azure.cosmos") def test_cosmos_context_manager(self): self.context_manager_common(self.cosmos_config) @@ -83,6 +86,7 @@ def get_set_common(self, config): def test_redis_get_set(self): self.get_set_common(self.redis_config) + @unittest.skipIf(skip_azure, "requires azure.cosmos") def test_cosmos_get_set(self): self.get_set_common(self.cosmos_config) @@ -96,6 +100,7 @@ def close_common(self, config): def test_redis_close(self): self.close_common(self.redis_config) + @unittest.skipIf(skip_azure, "requires azure.cosmos") def test_cosmos_close(self): self.close_common(self.cosmos_config) diff --git a/test/cache/test_cosmos_db_cache.py b/test/cache/test_cosmos_db_cache.py index f89a4c96cf42..80b97bf57ed0 100644 --- a/test/cache/test_cosmos_db_cache.py +++ b/test/cache/test_cosmos_db_cache.py @@ -4,18 +4,28 @@ import unittest from unittest.mock import MagicMock, patch -from azure.cosmos.exceptions import CosmosResourceNotFoundError +try: + from azure.cosmos.exceptions import CosmosResourceNotFoundError -from autogen.cache.cosmos_db_cache import CosmosDBCache + from autogen.cache.cosmos_db_cache import CosmosDBCache + + skip_test = False +except ImportError: + CosmosResourceNotFoundError = Exception + CosmosDBCache = object + skip_test = True class TestCosmosDBCache(unittest.TestCase): def setUp(self): - self.seed = "42" - self.connection_string = "AccountEndpoint=https://example.documents.azure.com:443/;" - self.database_id = "autogen_cache" - self.container_id = "TestContainer" - self.client = MagicMock() + if skip_test: + self.skipTest("requires azure.cosmos") + else: + self.seed = "42" + self.connection_string = "AccountEndpoint=https://example.documents.azure.com:443/;" + self.database_id = "autogen_cache" + self.container_id = "TestContainer" + self.client = MagicMock() @patch("autogen.cache.cosmos_db_cache.CosmosClient.from_connection_string", return_value=MagicMock()) def test_init(self, mock_from_connection_string): diff --git a/test/coding/test_commandline_code_executor.py b/test/coding/test_commandline_code_executor.py index 20041c54b420..09562357235c 100644 --- a/test/coding/test_commandline_code_executor.py +++ b/test/coding/test_commandline_code_executor.py @@ -7,7 +7,7 @@ import pytest from autogen.agentchat.conversable_agent import ConversableAgent -from autogen.code_utils import is_docker_running +from autogen.code_utils import decide_use_docker, is_docker_running from autogen.coding.base import CodeBlock, CodeExecutor from autogen.coding.docker_commandline_code_executor import DockerCommandLineCodeExecutor from autogen.coding.factory import CodeExecutorFactory @@ -16,9 +16,11 @@ sys.path.append(os.path.join(os.path.dirname(__file__), "..")) from conftest import MOCK_OPEN_AI_API_KEY, skip_docker # noqa: E402 -if skip_docker or not is_docker_running(): +if skip_docker or not is_docker_running() or not decide_use_docker(use_docker=None): + skip_docker_test = True classes_to_test = [LocalCommandLineCodeExecutor] else: + skip_docker_test = False classes_to_test = [LocalCommandLineCodeExecutor, DockerCommandLineCodeExecutor] UNIX_SHELLS = ["bash", "sh", "shell"] @@ -70,7 +72,7 @@ def test_create_local() -> None: @pytest.mark.skipif( - skip_docker or not is_docker_running(), + skip_docker_test, reason="docker is not running or requested to skip docker tests", ) def test_create_docker() -> None: @@ -99,7 +101,6 @@ def test_commandline_executor_execute_code(cls, py_variant) -> None: @pytest.mark.parametrize("py_variant", PYTHON_VARIANTS) def _test_execute_code(py_variant, executor: CodeExecutor) -> None: - # Test single code block. code_blocks = [CodeBlock(code="import sys; print('hello world!')", language=py_variant)] code_result = executor.execute_code_blocks(code_blocks) @@ -159,7 +160,6 @@ def test_local_commandline_code_executor_save_files_only() -> None: def _test_save_files(executor: CodeExecutor, save_file_only: bool) -> None: - def _check_output(code_result: CodeBlock, expected_output: str) -> None: if save_file_only: return expected_output not in code_result.output @@ -243,7 +243,7 @@ def test_local_commandline_code_executor_restart() -> None: # This is kind of hard to test because each exec is a new env @pytest.mark.skipif( - skip_docker or not is_docker_running(), + skip_docker_test, reason="docker is not running or requested to skip docker tests", ) def test_docker_commandline_code_executor_restart() -> None: diff --git a/test/oai/test_client_stream.py b/test/oai/test_client_stream.py index 9e85bff7039a..456a8fe761ef 100755 --- a/test/oai/test_client_stream.py +++ b/test/oai/test_client_stream.py @@ -260,29 +260,22 @@ def test_chat_tools_stream() -> None: ] client = OpenAIWrapper(config_list=config_list) response = client.create( - # the intention is to trigger two tool invocations as a response to a single message - messages=[{"role": "user", "content": "What's the weather like today in San Francisco and New York?"}], + messages=[{"role": "user", "content": "What's the weather like today in San Francisco?"}], tools=tools, stream=True, ) - print(f"{response=}") - print(f"{type(response)=}") - print(f"{client.extract_text_or_completion_object(response)=}") # check response choices = response.choices assert isinstance(choices, list) - assert len(choices) == 1 + assert len(choices) > 0 + choice = choices[0] assert choice.finish_reason == "tool_calls" + message = choice.message tool_calls = message.tool_calls assert isinstance(tool_calls, list) - assert len(tool_calls) == 2 - arguments = [tool_call.function.arguments for tool_call in tool_calls] - locations = [json.loads(argument)["location"] for argument in arguments] - print(f"{locations=}") - assert any(["San Francisco" in location for location in locations]) - assert any(["New York" in location for location in locations]) + assert len(tool_calls) > 0 @pytest.mark.skipif(skip, reason="openai>=1 not installed") diff --git a/test/test_code_utils.py b/test/test_code_utils.py index a4c6cf697cc5..d6084f9b0297 100755 --- a/test/test_code_utils.py +++ b/test/test_code_utils.py @@ -30,6 +30,11 @@ OAI_CONFIG_LIST = "OAI_CONFIG_LIST" here = os.path.abspath(os.path.dirname(__file__)) +if skip_docker or not is_docker_running() or not decide_use_docker(use_docker=None): + skip_docker_test = True +else: + skip_docker_test = False + # def test_find_code(): # try: @@ -302,10 +307,7 @@ def scrape(url): assert len(codeblocks) == 1 and codeblocks[0] == ("", "source setup.sh") -@pytest.mark.skipif( - skip_docker or not is_docker_running(), - reason="docker is not running or requested to skip docker tests", -) +@pytest.mark.skipif(skip_docker_test, reason="docker is not running or requested to skip docker tests") def test_execute_code(use_docker=True): # Test execute code and save the code to a file. with tempfile.TemporaryDirectory() as tempdir: @@ -369,10 +371,7 @@ def test_execute_code(use_docker=True): assert isinstance(image, str) -@pytest.mark.skipif( - skip_docker or not is_docker_running(), - reason="docker is not running or requested to skip docker tests", -) +@pytest.mark.skipif(skip_docker_test, reason="docker is not running or requested to skip docker tests") def test_execute_code_with_custom_filename_on_docker(): with tempfile.TemporaryDirectory() as tempdir: filename = "codetest.py" @@ -387,7 +386,7 @@ def test_execute_code_with_custom_filename_on_docker(): @pytest.mark.skipif( - skip_docker or not is_docker_running(), + skip_docker_test, reason="docker is not running or requested to skip docker tests", ) def test_execute_code_with_misformed_filename_on_docker(): diff --git a/website/docs/contributor-guide/tests.md b/website/docs/contributor-guide/tests.md index 69092cc4d68a..c5eabb907329 100644 --- a/website/docs/contributor-guide/tests.md +++ b/website/docs/contributor-guide/tests.md @@ -44,12 +44,12 @@ pytest test --skip-openai --skip-docker ## Coverage -Any code you commit should not decrease coverage. To run all unit tests, install the [test] option: +Any code you commit should not decrease coverage. To ensure your code maintains or increases coverage, use the following commands after installing the required test dependencies: ```bash -pip install -e."[test]" -coverage run -m pytest test +pip install -e ."[test]" + +pytest test --cov-report=html ``` -Then you can see the coverage report by -`coverage report -m` or `coverage html`. +Pytest generated a code coverage report and created a htmlcov directory containing an index.html file and other related files. Open index.html in any web browser to visualize and navigate through the coverage data interactively. This interactive visualization allows you to identify uncovered lines and review coverage statistics for individual files.