Merge pull request kata-containers#7931 from BbolroC/migrate-to-gha-s…

…390x

tests|gha: add containerd and k8s tests for s390x

.github/workflows/ci-nightly-s390x.yaml

@@ -1,6 +1,6 @@
 on:
   schedule:
-    - cron: '0 2 * * *'
+    - cron: '0 5 * * *'
 
 name: Nightly CI for s390x
 jobs:

.github/workflows/ci.yaml

@@ -36,6 +36,25 @@ jobs:
       target-branch: ${{ inputs.target-branch }}
     secrets: inherit
 
+  build-kata-static-tarball-s390x:
+    uses: ./.github/workflows/build-kata-static-tarball-s390x.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+
+  publish-kata-deploy-payload-s390x:
+    needs: build-kata-static-tarball-s390x
+    uses: ./.github/workflows/publish-kata-deploy-payload-s390x.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-s390x
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}
+    secrets: inherit
+
   build-and-publish-tee-confidential-unencrypted-image:
     runs-on: ubuntu-latest
     steps:
@@ -152,6 +171,17 @@ jobs:
       pr-number: ${{ inputs.pr-number }}
       target-branch: ${{ inputs.target-branch }}
 
+  run-k8s-tests-on-zvsi:
+    needs: [publish-kata-deploy-payload-s390x, build-and-publish-tee-confidential-unencrypted-image]
+    uses: ./.github/workflows/run-k8s-tests-on-zvsi.yaml
+    with:
+      registry: ghcr.io
+      repo: ${{ github.repository_owner }}/kata-deploy-ci
+      tag: ${{ inputs.tag }}-s390x
+      commit-hash: ${{ inputs.commit-hash }}
+      pr-number: ${{ inputs.pr-number }}
+      target-branch: ${{ inputs.target-branch }}
+
   run-metrics-tests:
     needs: build-kata-static-tarball-amd64
     uses: ./.github/workflows/run-metrics.yaml
@@ -167,3 +197,11 @@ jobs:
       tarball-suffix: -${{ inputs.tag }}
       commit-hash: ${{ inputs.commit-hash }}
       target-branch: ${{ inputs.target-branch }}
+
+  run-cri-containerd-tests-s390x:
+    needs: build-kata-static-tarball-s390x
+    uses: ./.github/workflows/run-cri-containerd-tests-s390x.yaml
+    with:
+      tarball-suffix: -${{ inputs.tag }}
+      commit-hash: ${{ inputs.commit-hash }}
+      target-branch: ${{ inputs.target-branch }}

.github/workflows/run-cri-containerd-tests-s390x.yaml

@@ -0,0 +1,66 @@
+name: CI | Run cri-containerd tests
+on:
+  workflow_call:
+    inputs:
+      tarball-suffix:
+        required: false
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-cri-containerd:
+    strategy:
+      # We can set this to true whenever we're 100% sure that
+      # the all the tests are not flaky, otherwise we'll fail
+      # all the tests due to a single flaky instance
+      fail-fast: false
+      matrix:
+        containerd_version: ['active']
+        vmm: ['qemu']
+    runs-on: s390x
+    env:
+      CONTAINERD_VERSION: ${{ matrix.containerd_version }}
+      GOPATH: ${{ github.workspace }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+    steps:
+      - name: Adjust a permission for repo
+        run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - name: Take a pre-action for self-hosted runner
+        run: ${HOME}/script/pre_action.sh ubuntu-2204
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Install dependencies
+        run: bash tests/integration/cri-containerd/gha-run.sh install-dependencies
+
+      - name: get-kata-tarball
+        uses: actions/download-artifact@v3
+        with:
+          name: kata-static-tarball-s390x${{ inputs.tarball-suffix }}
+          path: kata-artifacts
+
+      - name: Install kata
+        run: bash tests/integration/cri-containerd/gha-run.sh install-kata kata-artifacts
+
+      - name: Run cri-containerd tests
+        run: bash tests/integration/cri-containerd/gha-run.sh run
+
+      - name: Take a post-action for self-hosted runner
+        if: always()
+        run: ${HOME}/script/post_action.sh ubuntu-2204

.github/workflows/run-k8s-tests-on-zvsi.yaml

@@ -0,0 +1,83 @@
+name: CI | Run kubernetes tests on IBM Cloud Z virtual server instance (zVSI)
+on:
+  workflow_call:
+    inputs:
+      registry:
+        required: true
+        type: string
+      repo:
+        required: true
+        type: string
+      tag:
+        required: true
+        type: string
+      pr-number:
+        required: true
+        type: string
+      commit-hash:
+        required: false
+        type: string
+      target-branch:
+        required: false
+        type: string
+        default: ""
+
+jobs:
+  run-k8s-tests:
+    strategy:
+      fail-fast: false
+      matrix:
+        vmm:
+          - qemu
+        snapshotter:
+          - devmapper
+        k8s:
+          - k3s
+    runs-on: s390x
+    env:
+      DOCKER_REGISTRY: ${{ inputs.registry }}
+      DOCKER_REPO: ${{ inputs.repo }}
+      DOCKER_TAG: ${{ inputs.tag }}
+      PR_NUMBER: ${{ inputs.pr-number }}
+      KATA_HYPERVISOR: ${{ matrix.vmm }}
+      KUBERNETES: "k3s"
+      SNAPSHOTTER: ${{ matrix.snapshotter }}
+      USING_NFD: "true"
+      TARGET_ARCH: "s390x"
+    steps:
+      - name: Adjust a permission for repo
+        run: sudo chown -R $USER:$USER $GITHUB_WORKSPACE
+
+      - name: Take a pre-action for self-hosted runner
+        run: ${HOME}/script/pre_action.sh ubuntu-2204
+
+      - uses: actions/checkout@v4
+        with:
+          ref: ${{ inputs.commit-hash }}
+          fetch-depth: 0
+
+      - name: Rebase atop of the latest target branch
+        run: |
+          ./tests/git-helper.sh "rebase-atop-of-the-latest-target-branch"
+        env:
+          TARGET_BRANCH: ${{ inputs.target-branch }}
+
+      - name: Deploy ${{ matrix.k8s }}
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-k8s
+
+      - name: Configure the ${{ matrix.snapshotter }} snapshotter
+        run: bash tests/integration/kubernetes/gha-run.sh configure-snapshotter
+
+      - name: Deploy Kata
+        timeout-minutes: 10
+        run: bash tests/integration/kubernetes/gha-run.sh deploy-kata-zvsi
+
+      - name: Run tests
+        timeout-minutes: 30
+        run: bash tests/integration/kubernetes/gha-run.sh run-tests
+
+      - name: Take a post-action
+        if: always()
+        run: |
+          bash tests/integration/kubernetes/gha-run.sh cleanup-zvsi || true
+          ${HOME}/script/post_action.sh ubuntu-2204

tests/common.bash

@@ -223,6 +223,13 @@ function restart_systemd_service_with_no_burst_limit() {
 		local unit_file=$(systemctl show "$service.service" -p FragmentPath | cut -d'=' -f2)
 		[ -f "$unit_file" ] || { warn "Can't find $service's unit file: $unit_file"; return 1; }
 
+		# If the unit file is in /lib, copy it to /etc
+		if [[ $unit_file == /lib* ]]; then
+			tmp_unit_file="/etc/${unit_file#*lib/}"
+			sudo cp "$unit_file" "$tmp_unit_file"
+			unit_file="$tmp_unit_file"
+		fi
+
 		local start_burst_set=$(sudo grep StartLimitBurst $unit_file | wc -l)
 		if [ "$start_burst_set" -eq 0 ]
 		then

tests/integration/kubernetes/filter_k8s_test.sh

@@ -8,15 +8,12 @@ set -o errexit
 set -o nounset
 set -o pipefail
 
-GOPATH_LOCAL="${GOPATH%%:*}"
-KATA_DIR="${GOPATH_LOCAL}/src/github.com/kata-containers"
-TEST_DIR="${KATA_DIR}/tests"
-CI_DIR="${TEST_DIR}/.ci"
+script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+repo_root_dir="$(cd "${script_dir}/../../.." && pwd)"
 
+GOPATH_LOCAL="${GOPATH%%:*}"
 K8S_FILTER_FLAG="kubernetes"
 
-source "${CI_DIR}/lib.sh"
-
 main()
 {
 	local K8S_CONFIG_FILE="$1"
@@ -25,8 +22,9 @@ main()
 
 	mapfile -d " " -t _K8S_TEST_UNION <<< "${K8S_TEST_UNION}"
 
-	# install yq if not exist
-        ${CI_DIR}/install_yq.sh > /dev/null
+	if [ ! -f ${GOPATH_LOCAL}/bin/yq ]; then
+		${repo_root_dir}/ci/install_yq.sh > /dev/null
+	fi
 
         local K8S_SKIP_UNION=$("${GOPATH_LOCAL}/bin/yq" read "${K8S_CONFIG_FILE}" "${K8S_FILTER_FLAG}")
         [ "${K8S_SKIP_UNION}" == "null" ] && return

tests/integration/kubernetes/gha-run.sh

@@ -252,13 +252,15 @@ function main() {
         deploy-kata-snp) deploy_kata "snp" ;;
         deploy-kata-tdx) deploy_kata "tdx" ;;
         deploy-kata-garm) deploy_kata "garm" ;;
+        deploy-kata-zvsi) deploy_kata "zvsi" ;;
         run-tests) run_tests ;;
         run-tests-kcli) run_tests "kcli" ;;
         cleanup-kcli) cleanup "kcli" ;;
         cleanup-sev) cleanup "sev" ;;
         cleanup-snp) cleanup "snp" ;;
         cleanup-tdx) cleanup "tdx" ;;
         cleanup-garm) cleanup "garm" ;;
+        cleanup-zvsi) cleanup "zvsi" ;;
         delete-cluster) cleanup "aks" ;;
         delete-cluster-kcli) delete_cluster_kcli ;;
         *) >&2 echo "Invalid argument"; exit 2 ;;