e2e-test.yaml

name: e2e-test
on:
  pull_request:
  push:
    branches:
      - main
  workflow_dispatch:
    inputs:
      kernel-image:
        type: string
        description: 'The kernel image to use for the VMs. If not specified, a kernel will be built from source'
        required: false
      cluster:
        type: choice
        description: 'The cluster to run the tests on'
        options:
          - k3d
          - kind
        default: k3d

env:
  PRESERVE_RUNNER_PODS: "true"

defaults:
  run:
    shell: bash -euo pipefail {0}

jobs:
  vm-kernel:
    uses: ./.github/workflows/vm-kernel.yaml
    with:
      return-image-for-tag: ${{ inputs.kernel-image }}
    secrets: inherit

  e2e-tests:
    needs: vm-kernel
    strategy:
      fail-fast: false
      matrix:
        cluster:
          - ${{ inputs.cluster || 'k3d' }}
    runs-on: [ self-hosted, gen3, large ]
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0 # fetch all, so that we also include tags

      - uses: actions/setup-go@v4
        with:
          go-version-file: 'go.mod'
          # Disable cache on self-hosted runners to avoid /usr/bin/tar errors, see https://github.com/actions/setup-go/issues/403
          cache: false
        # Sometimes setup-go gets stuck. Without this, it'll keep going until the job gets killed
        timeout-minutes: 10

      - uses: docker/setup-buildx-action@v2

      - name: Install dependencies
        run: |
          make e2e-tools
          echo $(pwd)/bin >> $GITHUB_PATH

      - name: Check dependencies
        run: |
          kubectl version --client --output=yaml
          k3d version
          kind version
          kuttl version
          docker version
          docker buildx version

      - name: Load VM kernel
        env:
          IMAGE: ${{ needs.vm-kernel.outputs.image }}
        run: |
          docker pull --quiet $IMAGE
          ID=$(docker create $IMAGE true)
          docker cp ${ID}:/vmlinuz neonvm/hack/kernel/vmlinuz
          docker rm -f ${ID}

      # our docker builds use the output of 'git describe' for embedding git information
      - run: git describe --long --dirty

      # Explicitly build all the images beforehand. Building images while the cluster is up can
      # sometimes affect the cluster. For more information:
      # https://github.com/neondatabase/autoscaling/issues/120#issuecomment-1493405844
      - run: make build
      - run: make docker-build
      - run: make docker-build-examples

      - run: make ${{ matrix.cluster }}-setup

      - run: make deploy
        timeout-minutes: 10
      - run: make example-vms
        timeout-minutes: 10
      - run: make e2e
        timeout-minutes: 15

      - name: Get k8s logs and events
        if: always()
        run: |
          if ! kubectl config current-context; then
            echo "skipping cluster logs because no cluster found in kubectl context"
            exit 0
          fi

          namespaces=$(kubectl get namespaces -o jsonpath='{.items[*].metadata.name}')
          for namespace in $namespaces; do
            if [[ "$namespace" == "neonvm-system" ]] || [[ "$namespace" == kuttl-test-* ]]; then
              tee_if_needed=$GITHUB_STEP_SUMMARY
            else
              tee_if_needed=/dev/null
            fi

            {
              echo "<details>"
              echo "<summary>Namespace=$namespace</summary>"
            } | tee -a $tee_if_needed

            pods=$(kubectl get pods -n $namespace -o jsonpath='{.items[*].metadata.name}')
            for pod in $pods; do
              {
                echo "<details>"
                echo "<summary>- Namespace=$namespace Pod=$pod Logs</summary>"
                echo "<pre>"
              } | tee -a $tee_if_needed

              restarts=$(
                kubectl get pod -n $namespace $pod -o jsonpath='{.status.containerStatuses[0].restartCount}' || echo '0'
              )
              {
                if [ "$restarts" -ne 0 ]; then
                  echo "CONTAINER RESTARTED $restarts TIME(S)"
                  echo "Previous logs:"
                  kubectl logs -n $namespace -p $pod || echo 'Error getting logs'
                  echo "Current logs:"
                  kubectl logs -n $namespace $pod || echo 'Error getting logs'
                else
                  echo "Logs:"
                  kubectl logs -n $namespace $pod || echo 'Error getting logs'
                fi
              } | tee -a $tee_if_needed
              {
                echo "</pre>"
                echo "</details>"
              } | tee -a $tee_if_needed

              {
                echo "<details>"
                echo "<summary>- Namespace=$namespace Pod=$pod Events</summary>"
                echo "<pre>"
              } | tee -a $tee_if_needed

              (kubectl get events --namespace $namespace --field-selector involvedObject.name=$pod || echo 'Error getting events') | tee -a $tee_if_needed

              {
                echo "</pre>"
                echo "</pre>"
                echo "</details>"
              } | tee -a $tee_if_needed
            done

            echo "</details>" | tee -a $tee_if_needed
          done

      - name: Cleanup
        if: always()
        run: make ${{ matrix.cluster }}-destroy