test.yml

name: test

on:
  push:
    branches:
      - main
      - 'release/**'
  pull_request:
    paths-ignore:
      - '**.md'

env:
  GO_VERSION: 1.24.x
  GOTOOLCHAIN: local
  SHORT_TIMEOUT: 5
  LONG_TIMEOUT: 60

jobs:
  # This job builds the dependency target of the test docker image for all supported architectures and cache it in GHA
  build-dependencies:
    timeout-minutes: 15
    name: dependencies |  ${{ matrix.containerd }} | ${{ matrix.arch }}
    runs-on: "${{ matrix.runner }}"
    strategy:
      fail-fast: false
      matrix:
        include:
          - runner: ubuntu-24.04
            containerd: v1.6.36
            arch: amd64
          - runner: ubuntu-24.04
            containerd: v2.0.3
            arch: amd64
          - runner: ubuntu-24.04-arm
            containerd: v2.0.3
            arch: arm64
    env:
      CONTAINERD_VERSION: "${{ matrix.containerd }}"
      ARCH: "${{ matrix.arch }}"
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
        with:
          fetch-depth: 1
      - name: "Expose GitHub Runtime variables for gha"
        uses: crazy-max/ghaction-github-runtime@b3a9207c0e1ef41f4cf215303c976869d0c2c1c4  # v3.0.0
      - name: "Build dependencies for the integration test environment image"
        run: |
          docker buildx create --name with-gha --use
          docker buildx build \
            --cache-to type=gha,compression=zstd,mode=max,scope=test-integration-dependencies-${ARCH} \
            --cache-from type=gha,scope=test-integration-dependencies-${ARCH} \
            --target build-dependencies --build-arg CONTAINERD_VERSION=${CONTAINERD_VERSION} .

  test-unit:
    # FIXME:
    # Supposed to work: https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/evaluate-expressions-in-workflows-and-actions#example-returning-a-json-data-type
    # Apparently does not
    # timeout-minutes: ${{ fromJSON(env.SHORT_TIMEOUT) }}
    timeout-minutes: 10
    name: unit | ${{ matrix.goos }}
    runs-on: "${{ matrix.os }}"
    defaults:
      run:
        shell: bash
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: windows-2022
            goos: windows
          - os: ubuntu-24.04
            goos: linux
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
        with:
          fetch-depth: 1
      - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34  # v5.3.0
        with:
          go-version: ${{ env.GO_VERSION }}
          check-latest: true
      - if: ${{ matrix.goos=='windows' }}
        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
        with:
          repository: containerd/containerd
          ref: v1.7.25
          path: containerd
          fetch-depth: 1
      - if: ${{ matrix.goos=='windows' }}
        name: "Set up CNI"
        working-directory: containerd
        run: GOPATH=$(go env GOPATH) script/setup/install-cni-windows
      - name: "Run unit tests"
        run: make test-unit

  test-integration:
    needs: build-dependencies
    timeout-minutes: 40
    name: rootful | ${{ matrix.containerd }} | ${{ matrix.runner }}
    runs-on: "${{ matrix.runner }}"
    strategy:
      fail-fast: false
      matrix:
        include:
          - ubuntu: 22.04
            containerd: v1.6.36
            runner: "ubuntu-22.04"
            arch: amd64
          - ubuntu: 24.04
            containerd: v2.0.3
            runner: "ubuntu-24.04"
            arch: amd64
          - ubuntu: 24.04
            containerd: v2.0.3
            runner: "ubuntu-24.04-arm"
            arch: arm64
    env:
      CONTAINERD_VERSION: "${{ matrix.containerd }}"
      ARCH: "${{ matrix.arch }}"
      UBUNTU_VERSION: "${{ matrix.ubuntu }}"
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
        with:
          fetch-depth: 1
      - name: "Expose GitHub Runtime variables for gha"
        uses: crazy-max/ghaction-github-runtime@b3a9207c0e1ef41f4cf215303c976869d0c2c1c4  # v3.0.0
      - name: "Prepare integration test environment"
        run: |
          docker buildx create --name with-gha --use
          docker buildx build \
            --output=type=docker \
            --cache-from type=gha,scope=test-integration-dependencies-${ARCH} \
            -t test-integration --target test-integration --build-arg UBUNTU_VERSION=${UBUNTU_VERSION} --build-arg CONTAINERD_VERSION=${CONTAINERD_VERSION} .
      - name: "Remove snap loopback devices (conflicts with our loopback devices in TestRunDevice)"
        run: |
          sudo systemctl disable --now snapd.service snapd.socket
          sudo apt-get purge -qq snapd
          sudo losetup -Dv
          sudo losetup -lv
      - name: "Register QEMU (tonistiigi/binfmt)"
        run: |
          # `--install all` will only install emulation for architectures that cannot be natively executed
          # Since some arm64 platforms do provide native fallback execution for 32 bits,
          # armv7 emulation may or may not be installed, causing variance in the result of `uname -m`.
          # To avoid that, we explicitly list the architectures we do want emulation for.
          docker run --privileged --rm tonistiigi/binfmt --install linux/amd64
          docker run --privileged --rm tonistiigi/binfmt --install linux/arm64
          docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7
      - name: "Run integration tests"
        run: docker run -t --rm --privileged test-integration ./hack/test-integration.sh -test.only-flaky=false
      - name: "Run integration tests (flaky)"
        run: docker run -t --rm --privileged test-integration ./hack/test-integration.sh -test.only-flaky=true

  test-integration-ipv6:
    needs: build-dependencies
    timeout-minutes: 15
    name: ipv6 | ${{ matrix.containerd }} | ${{ matrix.ubuntu }}
    runs-on: "ubuntu-${{ matrix.ubuntu }}"
    strategy:
      fail-fast: false
      matrix:
        include:
          - ubuntu: 24.04
            containerd: v2.0.3
            arch: amd64
    env:
      CONTAINERD_VERSION: "${{ matrix.containerd }}"
      ARCH: "${{ matrix.arch }}"
      UBUNTU_VERSION: "${{ matrix.ubuntu }}"
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
        with:
          fetch-depth: 1
      - name: Enable ipv4 and ipv6 forwarding
        run: |
          sudo sysctl -w net.ipv6.conf.all.forwarding=1
          sudo sysctl -w net.ipv4.ip_forward=1
      - name: "Expose GitHub Runtime variables for gha"
        uses: crazy-max/ghaction-github-runtime@b3a9207c0e1ef41f4cf215303c976869d0c2c1c4  # v3.0.0
      - name: Enable IPv6 for Docker, and configure docker to use containerd for gha
        run: |
          sudo mkdir -p /etc/docker
          echo '{"ipv6": true, "fixed-cidr-v6": "2001:db8:1::/64", "experimental": true, "ip6tables": true}' | sudo tee /etc/docker/daemon.json
          sudo systemctl restart docker
      - name: "Prepare integration test environment"
        run: |
          docker buildx create --name with-gha --use
          docker buildx build \
            --output=type=docker \
            --cache-from type=gha,scope=test-integration-dependencies-${ARCH} \
            -t test-integration --target test-integration --build-arg UBUNTU_VERSION=${UBUNTU_VERSION} --build-arg CONTAINERD_VERSION=${CONTAINERD_VERSION} .
      - name: "Remove snap loopback devices (conflicts with our loopback devices in TestRunDevice)"
        run: |
          sudo systemctl disable --now snapd.service snapd.socket
          sudo apt-get purge -qq snapd
          sudo losetup -Dv
          sudo losetup -lv
      - name: "Register QEMU (tonistiigi/binfmt)"
        run: |
          # `--install all` will only install emulation for architectures that cannot be natively executed
          # Since some arm64 platforms do provide native fallback execution for 32 bits,
          # armv7 emulation may or may not be installed, causing variance in the result of `uname -m`.
          # To avoid that, we explicitly list the architectures we do want emulation for.
          docker run --privileged --rm tonistiigi/binfmt --install linux/amd64
          docker run --privileged --rm tonistiigi/binfmt --install linux/arm64
          docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7
      - name: "Run integration tests"
        # The nested IPv6 network inside docker and qemu is complex and needs a bunch of sysctl config.
        # Therefore, it's hard to debug why the IPv6 tests fail in such an isolation layer.
        # On the other side, using the host network is easier at configuration.
        # Besides, each job is running on a different instance, which means using host network here
        # is safe and has no side effects on others.
        run: docker run --network host -t --rm --privileged test-integration ./hack/test-integration.sh -test.only-ipv6

  test-integration-rootless:
    needs: build-dependencies
    timeout-minutes: 40
    name: "${{ matrix.target }} | ${{ matrix.containerd }} | ${{ matrix.rootlesskit }} | ${{ matrix.ubuntu }}"
    runs-on: "${{ matrix.runner }}"
    strategy:
      fail-fast: false
      matrix:
        include:
          - ubuntu: 22.04
            containerd: v1.6.36
            rootlesskit: v1.1.1  # Deprecated
            target: rootless
            runner: "ubuntu-22.04"
            arch: amd64
          - ubuntu: 24.04
            containerd: v2.0.3
            rootlesskit: v2.3.4
            target: rootless
            arch: amd64
            runner: "ubuntu-24.04"
          - ubuntu: 24.04
            containerd: v2.0.3
            rootlesskit: v2.3.4
            target: rootless
            arch: arm64
            runner: "ubuntu-24.04-arm"
          - ubuntu: 24.04
            containerd: v2.0.3
            rootlesskit: v2.3.4
            target: rootless-port-slirp4netns
            arch: amd64
            runner: "ubuntu-24.04"
    env:
      CONTAINERD_VERSION: "${{ matrix.containerd }}"
      ARCH: "${{ matrix.arch }}"
      UBUNTU_VERSION: "${{ matrix.ubuntu }}"
      ROOTLESSKIT_VERSION: "${{ matrix.rootlesskit }}"
      TEST_TARGET: "test-integration-${{ matrix.target }}"
    steps:
      - name: "Set up AppArmor"
        if: matrix.ubuntu == '24.04'
        run: |
          cat <<EOT | sudo tee "/etc/apparmor.d/usr.local.bin.rootlesskit"
          abi <abi/4.0>,
          include <tunables/global>

          /usr/local/bin/rootlesskit flags=(unconfined) {
            userns,

            # Site-specific additions and overrides. See local/README for details.
            include if exists <local/usr.local.bin.rootlesskit>
          }
          EOT
          sudo systemctl restart apparmor.service
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
        with:
          fetch-depth: 1
      - name: "Register QEMU (tonistiigi/binfmt)"
        run: |
          # `--install all` will only install emulation for architectures that cannot be natively executed
          # Since some arm64 platforms do provide native fallback execution for 32 bits,
          # armv7 emulation may or may not be installed, causing variance in the result of `uname -m`.
          # To avoid that, we explicitly list the architectures we do want emulation for.
          docker run --privileged --rm tonistiigi/binfmt --install linux/amd64
          docker run --privileged --rm tonistiigi/binfmt --install linux/arm64
          docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7
      - name: "Expose GitHub Runtime variables for gha"
        uses: crazy-max/ghaction-github-runtime@b3a9207c0e1ef41f4cf215303c976869d0c2c1c4  # v3.0.0
      - name: "Prepare (network driver=slirp4netns, port driver=builtin)"
        run: |
          docker buildx create --name with-gha --use
          docker buildx build \
            --output=type=docker \
            --cache-from type=gha,scope=test-integration-dependencies-${ARCH} \
            -t ${TEST_TARGET} --target ${TEST_TARGET} --build-arg UBUNTU_VERSION=${UBUNTU_VERSION} --build-arg CONTAINERD_VERSION=${CONTAINERD_VERSION} --build-arg ROOTLESSKIT_VERSION=${ROOTLESSKIT_VERSION} .
      - name: "Disable BuildKit for RootlessKit v1 (workaround for issue #622)"
        run: |
          # https://github.com/containerd/nerdctl/issues/622
          WORKAROUND_ISSUE_622=
          if echo "${ROOTLESSKIT_VERSION}" | grep -q v1; then
            WORKAROUND_ISSUE_622=1
          fi
          echo "WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622}" >> "$GITHUB_ENV"
      - name: "Test (network driver=slirp4netns, port driver=builtin)"
        run: docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} ${TEST_TARGET} /test-integration-rootless.sh ./hack/test-integration.sh -test.only-flaky=false
      - name: "Test (network driver=slirp4netns, port driver=builtin) (flaky)"
        run: docker run -t --rm --privileged -e WORKAROUND_ISSUE_622=${WORKAROUND_ISSUE_622} ${TEST_TARGET} /test-integration-rootless.sh ./hack/test-integration.sh -test.only-flaky=true

  build:
    timeout-minutes: 5
    name: "build | ${{ matrix.go-version }}"
    runs-on: ubuntu-24.04
    strategy:
      fail-fast: false
      matrix:
        go-version: ["1.23.x", "1.24.x"]
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
        with:
          fetch-depth: 1
      - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34  # v5.3.0
        with:
          go-version: ${{ matrix.go-version }}
          check-latest: true
      - name: "build"
        run: GO_VERSION="$(echo ${{ matrix.go-version }} | sed -e s/.x//)" make binaries

  test-integration-docker-compatibility:
    timeout-minutes: 40
    name: docker
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
        with:
          fetch-depth: 1
      - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34  # v5.3.0
        with:
          go-version: ${{ env.GO_VERSION }}
          check-latest: true
      - name: "Register QEMU (tonistiigi/binfmt)"
        run: |
          # `--install all` will only install emulation for architectures that cannot be natively executed
          # Since some arm64 platforms do provide native fallback execution for 32 bits,
          # armv7 emulation may or may not be installed, causing variance in the result of `uname -m`.
          # To avoid that, we explicitly list the architectures we do want emulation for.
          docker run --privileged --rm tonistiigi/binfmt --install linux/amd64
          docker run --privileged --rm tonistiigi/binfmt --install linux/arm64
          docker run --privileged --rm tonistiigi/binfmt --install linux/arm/v7
      - name: "Prepare integration test environment"
        run: |
          # FIXME: remove expect when we are done removing unbuffer from tests
          sudo apt-get install -qq expect
          make install-dev-tools
      - name: "Ensure that the integration test suite is compatible with Docker"
        run: WITH_SUDO=true ./hack/test-integration.sh -test.target=docker
      - name: "Ensure that the IPv6 integration test suite is compatible with Docker"
        run: WITH_SUDO=true ./hack/test-integration.sh -test.target=docker -test.only-ipv6
      - name: "Ensure that the integration test suite is compatible with Docker (flaky only)"
        run: WITH_SUDO=true ./hack/test-integration.sh -test.target=docker -test.only-flaky

  test-integration-windows:
    timeout-minutes: 40
    name: windows
    runs-on: windows-2022
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
        with:
          fetch-depth: 1
      - uses: actions/setup-go@f111f3307d8850f501ac008e886eec1fd1932a34  # v5.3.0
        with:
          go-version: ${{ env.GO_VERSION }}
          check-latest: true
      - run: |
          go install ./cmd/nerdctl
          make install-dev-tools
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
        with:
          repository: containerd/containerd
          ref: v1.7.25
          path: containerd
          fetch-depth: 1
      - name: "Set up CNI"
        working-directory: containerd
        run: GOPATH=$(go env GOPATH) script/setup/install-cni-windows
      - name: "Set up containerd"
        env:
          ctrdVersion: 1.7.25
        run: powershell hack/configure-windows-ci.ps1
      - name: "Run integration tests"
        run: ./hack/test-integration.sh -test.only-flaky=false
      - name: "Run integration tests (flaky)"
        run: ./hack/test-integration.sh -test.only-flaky=true

  test-integration-freebsd:
    timeout-minutes: 40
    name: FreeBSD
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
        with:
          fetch-depth: 1
      - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf  # v4.2.2
        with:
          path: /root/.vagrant.d
          key: vagrant-${{ matrix.box }}
      - name: Set up vagrant
        run: |
          # from https://github.com/containerd/containerd/blob/v2.0.2/.github/workflows/ci.yml#L583-L596
          # which is based on https://github.com/opencontainers/runc/blob/v1.1.8/.cirrus.yml#L41-L49
          curl -fsSL https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
          echo "deb [signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
          sudo sed -i 's/^Types: deb$/Types: deb deb-src/' /etc/apt/sources.list.d/ubuntu.sources
          sudo apt-get update -qq
          sudo apt-get install -qq libvirt-daemon libvirt-daemon-system vagrant ovmf
          # https://github.com/vagrant-libvirt/vagrant-libvirt/issues/1725#issuecomment-1454058646
          sudo cp /usr/share/OVMF/OVMF_VARS_4M.fd /var/lib/libvirt/qemu/nvram/
          sudo systemctl enable --now libvirtd
          sudo apt-get build-dep -qq ruby-libvirt
          sudo apt-get install -qq --no-install-recommends libxslt-dev libxml2-dev libvirt-dev ruby-bundler ruby-dev zlib1g-dev
          sudo vagrant plugin install vagrant-libvirt
      - name: Boot VM
        run: |
          ln -sf Vagrantfile.freebsd Vagrantfile
          sudo vagrant up --no-tty
      - name: test-unit
        run: sudo vagrant up --provision-with=test-unit
      - name: test-integration
        run: sudo vagrant up --provision-with=test-integration

  # EL8 is used for testing compatibility with cgroup v1.
  # Do not upgrade this to EL9 (cgroup v2).
  test-integration-el8:
    timeout-minutes: 60
    name: "[flaky, see #3988] EL8 (cgroup v1)"
    strategy:
      fail-fast: false
      matrix:
        mode: ["rootful", "rootless"]
    runs-on: ubuntu-24.04
    env:
      MODE: ${{ matrix.mode }}
      # FIXME: this is only necessary to access the build cache. To remove with build cleanup.
      CONTAINERD_VERSION: v2.0.3
    steps:
      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683  # v4.2.2
        with:
          fetch-depth: 1
      - uses: lima-vm/lima-actions/setup@be564a1408f84557d067b099a475652288074b2e  # v1.0.0
        id: lima-actions-setup
      - uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf  # v4.2.2
        with:
          path: ~/.cache/lima
          key: lima-${{ steps.lima-actions-setup.outputs.version }}
      - name: "Start the guest VM"
        run: |
          set -eux
          # containerd=none is set because the built-in containerd support conflicts with Docker
          limactl start \
            --name=default \
            --cpus=4 \
            --memory=12 \
            --containerd=none \
            --set '.mounts=null | .portForwards=[{"guestSocket":"/var/run/docker.sock","hostSocket":"{{.Dir}}/sock/docker.sock"}]' \
            template://almalinux-8
      # FIXME: the tests should be directly executed in the VM without nesting Docker inside it
      # https://github.com/containerd/nerdctl/issues/3858
      - name: "Install dockerd in the guest VM"
        run: |
          set -eux
          lima sudo mkdir -p /etc/systemd/system/docker.socket.d
          cat <<-EOF | lima sudo tee /etc/systemd/system/docker.socket.d/override.conf
          [Socket]
          SocketUser=$(whoami)
          EOF
          lima sudo dnf config-manager --add-repo=https://download.docker.com/linux/centos/docker-ce.repo
          lima sudo dnf -q -y install docker-ce --nobest
          lima sudo systemctl enable --now docker
      - name: "Configure the host to use dockerd in the guest VM"
        run: |
          set -eux
          sudo systemctl disable --now docker.service docker.socket
          export DOCKER_HOST="unix://$(limactl ls --format '{{.Dir}}/sock/docker.sock' default)"
          echo "DOCKER_HOST=${DOCKER_HOST}" >>$GITHUB_ENV
          docker info
          docker version
      - name: "Expose GitHub Runtime variables for gha"
        uses: crazy-max/ghaction-github-runtime@b3a9207c0e1ef41f4cf215303c976869d0c2c1c4  # v3.0.0
      - name: "Prepare integration tests"
        run: |
          set -eux

          sudo losetup -Dv
          sudo losetup -lv

          TARGET=test-integration
          [ "$MODE" = "rootless" ] && TARGET=test-integration-rootless
          docker buildx create --name with-gha --use
          docker buildx build \
            --output=type=docker \
            --cache-from type=gha,scope=amd64-${CONTAINERD_VERSION} \
            -t test-integration --target "${TARGET}" \
            .

      - name: "Run integration tests"
        # Presumably, something is broken with the way docker exposes /dev to the container, as it appears to only
        # randomly work. Mounting /dev does workaround the issue.
        # This might be due to the old kernel shipped with Alma (4.18), or something else between centos/docker.
        run: |
          set -eux
          [ "$MODE" = "rootless" ] && {
            echo "rootless"
            docker run -t -v /dev:/dev --rm --privileged test-integration /test-integration-rootless.sh ./hack/test-integration.sh -test.only-flaky=false
          } || {
            echo "rootful"
            docker run -t -v /dev:/dev --rm --privileged test-integration ./hack/test-integration.sh -test.only-flaky=false
          }
      - name: "Run integration tests (flaky)"
        run: |
          set -eux
          [ "$MODE" = "rootless" ] && {
            echo "rootless"
            docker run -t -v /dev:/dev --rm --privileged test-integration /test-integration-rootless.sh ./hack/test-integration.sh -test.only-flaky=true
          } || {
            echo "rootful"
            docker run -t -v /dev:/dev --rm --privileged test-integration ./hack/test-integration.sh -test.only-flaky=true
          }