Run Tests #309
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| # Copyright 2024 Canonical Ltd. | |
| # See LICENSE file for licensing details. | |
| name: Run Tests | |
| on: | |
| schedule: | |
| - cron: '53 0 * * *' # Daily at 00:53 UTC | |
| workflow_call: | |
| jobs: | |
| checks: | |
| uses: ./.github/workflows/ci-checks.yaml | |
| unit-tests: | |
| name: Unit Tests | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 10 | |
| strategy: | |
| fail-fast: true | |
| matrix: | |
| python-version: | |
| # - '3.9' | |
| - '3.10' | |
| needs: | |
| - checks | |
| steps: | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - id: setup-python | |
| name: Setup Python | |
| uses: actions/[email protected] | |
| with: | |
| python-version: ${{matrix.python-version}} | |
| architecture: x64 | |
| - name: Install tox & poetry | |
| run: | | |
| pip install tox | |
| pip install "poetry<2.0.0" | |
| - id: tests-unit | |
| name: Run Unittests | |
| run: | | |
| cd python && tox -e unit | |
| integration-tests: | |
| name: Integration Tests | |
| runs-on: ubuntu-22.04 | |
| timeout-minutes: 120 | |
| strategy: | |
| max-parallel: 7 | |
| fail-fast: false | |
| matrix: | |
| version: | |
| - --spark-version "3.4.2" | |
| - --spark-version "3.5.1" | |
| backend: | |
| - --backend yaml | |
| - --backend terraform | |
| cos: | |
| - | |
| - --cos-model cos | |
| tox-environments: | |
| - integration-basic | |
| - integration-sparkjob | |
| - integration-sparkjob-azure | |
| - integration-bundle | |
| - integration-kyuubi | |
| - integration-kyuubi-azure | |
| juju: | |
| - snap_channel: "3.4/stable" | |
| agent: "3.4.2" | |
| - snap_channel: "3.6/stable" | |
| agent: "3.6.0" | |
| exclude: | |
| - tox-environments: integration-basic # Basic tests, test without COS | |
| cos: --cos-model cos | |
| - tox-environments: integration-basic # Basic tests, test without terraform | |
| backend: --backend terraform | |
| - tox-environments: integration-sparkjob # COS is mandatory for the sparkjob test | |
| cos: | |
| - tox-environments: integration-sparkjob-azure # COS is mandatory for the sparkjob test | |
| cos: | |
| - tox-environments: integration-sparkjob-azure # Azure tests only supported in YAML bundle at the moment | |
| backend: --backend terraform | |
| - tox-environments: integration-kyuubi # Kyuubi tests do not require COS | |
| cos: --cos-model cos | |
| - tox-environments: integration-kyuubi-azure # Kyuubi tests do not require COS | |
| cos: --cos-model cos | |
| - tox-environments: integration-kyuubi-azure # Azure tests only supported in YAML bundle at the moment | |
| backend: --backend terraform | |
| needs: | |
| - checks | |
| - unit-tests | |
| steps: | |
| - name: (GitHub hosted) Free up disk space | |
| timeout-minutes: 5 | |
| run: | | |
| printf '\nDisk usage before cleanup\n' | |
| df --human-readable | |
| # Based on https://github.com/actions/runner-images/issues/2840#issuecomment-790492173 | |
| rm -r /usr/share/dotnet | |
| rm -r /opt/hostedtoolcache/ | |
| printf '\nDisk usage after cleanup\n' | |
| df --human-readable | |
| - name: Checkout repo | |
| uses: actions/checkout@v4 | |
| - id: setup-python | |
| name: Setup Python | |
| uses: actions/[email protected] | |
| with: | |
| python-version: "3.10" | |
| architecture: x64 | |
| - name: Get prefsrc | |
| run: | | |
| echo "IPADDR=$(ip -4 -j route get 2.2.2.2 | jq -r '.[] | .prefsrc')" >> $GITHUB_ENV | |
| - name: Setup operator environment | |
| uses: charmed-kubernetes/actions-operator@main | |
| with: | |
| juju-channel: ${{ matrix.juju.snap_channel }} | |
| provider: microk8s | |
| channel: 1.28-strict/stable | |
| bootstrap-options: "--agent-version ${{ matrix.juju.agent }}" | |
| microk8s-group: snap_microk8s | |
| microk8s-addons: "rbac dns minio metallb:${{ env.IPADDR }}-${{ env.IPADDR }}" | |
| - name: Install tox & poetry | |
| run: | | |
| pip install tox | |
| pip install "poetry<2.0.0" | |
| - name: Select tests | |
| id: select-tests | |
| run: | | |
| if [ "${{ github.event_name }}" == "schedule" ] | |
| then | |
| echo Running unstable and stable tests | |
| echo "mark_expression=" >> $GITHUB_OUTPUT | |
| else | |
| echo Skipping unstable tests | |
| echo "mark_expression=not unstable" >> $GITHUB_OUTPUT | |
| fi | |
| - id: setup-terraform | |
| name: Install terraform if needed | |
| run: | | |
| if ! [ -x "$(command -v terraform)" ]; then | |
| echo "Installing terraform from snap" | |
| sudo snap install terraform --classic | |
| fi | |
| - id: cache-image | |
| name: Cache Image Locally | |
| run: | | |
| # Download image for avoiding time out | |
| IMAGE="ghcr.io/canonical/charmed-spark-kyuubi@sha256:9268d19a6eef91914e874734b320fab64908faf0f7adb8856be809bc60ecd1d0" | |
| docker pull $IMAGE | |
| docker save $IMAGE -o image.tar | |
| sudo microk8s ctr images import --base-name $IMAGE image.tar | |
| docker rmi $IMAGE | |
| rm image.tar | |
| - id: tests-integration | |
| name: Run Integration Tests | |
| timeout-minutes: 30 | |
| env: | |
| AZURE_STORAGE_ACCOUNT: ${{ secrets.AZURE_STORAGE_ACCOUNT }} | |
| AZURE_STORAGE_KEY: ${{ secrets.AZURE_STORAGE_KEY }} | |
| run: | | |
| cd python && tox run -e ${{ matrix.tox-environments }} -- -m '${{ steps.select-tests.outputs.mark_expression }}' ${{ matrix.backend }} ${{ matrix.cos }} ${{ matrix.version }} --keep-models | |
| - name: Setup tmate session | |
| timeout-minutes: 20 | |
| if: ${{ failure() }} | |
| uses: mxschmitt/action-tmate@v3 |