vm-kernel: cache image (#715)

bayandin · web-flow · commit 2754495bc6f6 · 2024-01-09T19:04:08.000Z
This PR adds caching logic for `neondatabase:vm-kernel` image (it uses the latest git commit sha related to the kernel directory as a cache key) Changes: - All the stuff related to the kernel (Dockerfile + linux-config file) moved to a separate directory `neonvm/hack/kernel` `vm-kernel` workflow: - Gets the last commit sha of `neonvm/hack/kernel` - if an image with such a tag exists — return it to be used in e2e tests. - if the image doesn't exist — build an image and add cache tag for it, return new image for e2e tests - accepts `force-rebuild` parameter, which forces CI to rebuild the image Part of #660
diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yaml
@@ -21,13 +21,15 @@ on:
 env:
   PRESERVE_RUNNER_PODS: "true"
 
+defaults:
+  run:
+    shell: bash -euo pipefail {0}
+
 jobs:
   vm-kernel:
     uses: ./.github/workflows/vm-kernel.yaml
     with:
-      push: true
-      tag: ${{ github.run_id }}
-      kernel-image-tag-override: ${{ inputs.kernel-image }}
+      return-image-for-tag: ${{ inputs.kernel-image }}
     secrets: inherit
 
   e2e-tests:
@@ -73,7 +75,7 @@ jobs:
         run: |
           docker pull --quiet $IMAGE
           ID=$(docker create $IMAGE true)
-          docker cp ${ID}:/vmlinuz neonvm/hack/vmlinuz
+          docker cp ${ID}:/vmlinuz neonvm/hack/kernel/vmlinuz
           docker rm -f ${ID}
 
       # our docker builds use the output of 'git describe' for embedding git information
diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml
@@ -22,7 +22,6 @@ jobs:
   vm-kernel:
     uses: ./.github/workflows/vm-kernel.yaml
     with:
-      push: true
       tag: ${{ github.ref_name }}
 
   release:
@@ -82,7 +81,7 @@ jobs:
         run: |
           docker pull --quiet $IMAGE
           ID=$(docker create $IMAGE true)
-          docker cp ${ID}:/vmlinuz neonvm/hack/vmlinuz
+          docker cp ${ID}:/vmlinuz neonvm/hack/kernel/vmlinuz
           docker rm -f ${ID}
 
       - name: build and push runner image
diff --git a/.github/workflows/vm-kernel.yaml b/.github/workflows/vm-kernel.yaml
@@ -1,66 +1,144 @@
 name: vm-kernel
 
 on:
-  schedule:
-    - cron:  '42 4 * * 2'  # run once a week
   workflow_dispatch: # adds ability to run this manually
     inputs:
-      push:
-        description: 'Push to Docker Hub'
-        type: boolean
-        default: false
       tag:
         description: 'Tag to use for Docker image'
         type: string
         required: false
-  workflow_call:
-    inputs:
-      push:
-        description: 'Push to Docker Hub'
+      force-rebuild:
+        description: 'Rebuild the kernel image even if it already exists'
         type: boolean
+        required: false
         default: false
+  workflow_call:
+    inputs:
       tag:
         description: 'Tag to use for Docker image'
         type: string
         required: false
-      kernel-image-tag-override:
-        description: 'If set, the workflow return the full image name for this tag'
+      return-image-for-tag:
+        description: 'Make workflow to return image for the passed tag without building or tagging anything'
         type: string
         required: false
         default: ''
+      force-rebuild:
+        description: 'Rebuild the kernel image even if it already exists. No-op if `return-image-for-tag` is set'
+        type: boolean
+        required: false
+        default: false
     outputs:
       image:
         description: 'vm-kernel Docker image'
-        value: ${{ jobs.check-kernel-image-override.outputs.image || jobs.vm-kernel.outputs.image }}
+        value: ${{ jobs.setup-build-vm-kernel-image.outputs.image || jobs.build-vm-kernel-image.outputs.image }}
 
 env:
   VM_KERNEL_IMAGE: "neondatabase/vm-kernel"
 
+defaults:
+  run:
+    shell: bash -euo pipefail {0}
+
 jobs:
-  check-kernel-image-override:
+  setup-build-vm-kernel-image:
     outputs:
-      image: ${{ steps.check-kernel-image-override.outputs.image }}
+      image: ${{ steps.get-kernel-image.outputs.image }}
+      last-kernel-sha: ${{ steps.get-last-kernel-commit-sha.outputs.last-kernel-sha }}
 
     runs-on: ubuntu-latest
 
     steps:
-      - id: check-kernel-image-override
+      - name: get last kernel commit sha
+        id: get-last-kernel-commit-sha
+        env:
+          COMMIT_SHA: ${{ github.event.pull_request.head.sha || github.sha }}
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          CACHE_TAG=$(
+            gh api \
+              -H "Accept: application/vnd.github+json" \
+              -H "X-GitHub-Api-Version: 2022-11-28" \
+              --method GET \
+              --field path=neonvm/hack/kernel \
+              --field sha=${COMMIT_SHA} \
+              --field per_page=1 \
+              --jq ".[0].sha" \
+              "/repos/${GITHUB_REPOSITORY}/commits"
+          )
+          echo "last-kernel-sha=${CACHE_TAG}" >> $GITHUB_OUTPUT
+
+      - name: get kernel image
+        id: get-kernel-image
         env:
-          OVERRIDE_TAG: ${{ inputs.kernel-image-tag-override }}
+          FORCED_TAG: ${{ inputs.return-image-for-tag }}
+          FORCE_REBUILD: ${{ inputs.force-rebuild }}
+          CACHE_TAG: ${{ steps.get-last-kernel-commit-sha.outputs.last-kernel-sha }}
         run: |
-          if [ -n "${OVERRIDE_TAG}" ]; then
-            DIGEST=$(docker manifest inspect ${VM_KERNEL_IMAGE}:${OVERRIDE_TAG} -v | jq -r '.Descriptor.digest')
-            IMAGE="${VM_KERNEL_IMAGE}:${OVERRIDE_TAG}@${DIGEST}"
+          if [ -n "${FORCED_TAG}" ]; then
+            DIGEST=$(docker manifest inspect ${VM_KERNEL_IMAGE}:${FORCED_TAG} -v | jq -r '.Descriptor.digest')
+            IMAGE="${VM_KERNEL_IMAGE}:${FORCED_TAG}@${DIGEST}"
+          elif [ "${FORCE_REBUILD}" == "false" ]; then
+            CACHE_TAG_DIGEST=$(docker manifest inspect ${VM_KERNEL_IMAGE}:${CACHE_TAG} -v | jq -r '.Descriptor.digest' || true)
+            if [ -n "${CACHE_TAG_DIGEST}" ]; then
+              IMAGE="${VM_KERNEL_IMAGE}:${CACHE_TAG}@${CACHE_TAG_DIGEST}"
+            else
+              IMAGE=""
+            fi
           else
             IMAGE=""
           fi
+
           echo "image=${IMAGE}" >> $GITHUB_OUTPUT
 
-  vm-kernel:
-    needs: check-kernel-image-override
-    if: needs.check-kernel-image-override.outputs.image == ''
+      - name: check if we need to retag the image
+        id: check-if-retag-needed
+        env:
+          CACHED_IMAGE: ${{ steps.get-kernel-image.outputs.image }}
+          FORCE_REBUILD: "${{ inputs.force-rebuild }}"
+          FORCED_TAG: ${{ inputs.return-image-for-tag }}
+          NEW_TAG: ${{ inputs.tag }}
+        run: |
+          if [ -z "${NEW_TAG}" ]; then
+            # there's no tag provided to retag the image with
+            RETAG_NEEDED=false
+          elif [ -z "${CACHED_IMAGE}" ]; then
+            # there's no image to retag
+            RETAG_NEEDED=false
+          elif [ -n "${FORCED_TAG}"]; then
+            # we're asked to return image for a specific tag, so no need to retag
+            RETAG_NEEDED=false
+          elif [ "${FORCE_REBUILD}" == "true" ]; then
+            # the image is going to be rebuilt anyway, no need to retag it now
+            RETAG_NEEDED=false
+          else
+            RETAG_NEEDED=true
+          fi
+
+          echo "retag-needed=${RETAG_NEEDED}" >> $GITHUB_OUTPUT
+
+      - name: login to docker hub
+        if: steps.check-if-retag-needed.outputs.retag-needed == 'true'
+        uses: docker/login-action@v2
+        with:
+          username: ${{ secrets.NEON_DOCKERHUB_USERNAME }}
+          password: ${{ secrets.NEON_DOCKERHUB_PASSWORD }}
+
+      - name: tag image with new tag
+        if: steps.check-if-retag-needed.outputs.retag-needed == 'true'
+        env:
+          CACHED_IMAGE: ${{ steps.get-kernel-image.outputs.image }}
+          NEW_TAG: ${{ inputs.tag }}
+        run: |
+          docker pull ${CACHED_IMAGE}
+          docker tag ${CACHED_IMAGE} ${VM_KERNEL_IMAGE}:${NEW_TAG}
+          docker push ${VM_KERNEL_IMAGE}:${NEW_TAG}
+
+  build-vm-kernel-image:
+    needs: setup-build-vm-kernel-image
+    if: needs.setup-build-vm-kernel-image.outputs.image == ''
     outputs:
-      image: ${{ fromJSON(steps.build-linux-kernel.outputs.metadata)['image.name'] }}@${{ steps.build-linux-kernel.outputs.digest }}
+      image: ${{ steps.get-tags.outputs.canonical }}@${{ steps.build-linux-kernel.outputs.digest }}
 
     runs-on: [ self-hosted, gen3, large ]
     steps:
@@ -87,25 +165,42 @@ jobs:
       - name: get kernel version
         id: get-kernel-version
         run: |
-          linux_config=$(ls neonvm/hack/linux-config-*)  # returns something like "neonvm/hack/linux-config-6.1.63"
-          kernel_version=${linux_config##*-}             # returns something like "6.1.63"
+          linux_config=$(ls neonvm/hack/kernel/linux-config-*)  # returns something like "neonvm/hack/kernel/linux-config-6.1.63"
+          kernel_version=${linux_config##*-}                    # returns something like "6.1.63"
 
           echo VM_KERNEL_VERSION=$kernel_version >> $GITHUB_OUTPUT
 
+      - name: get docker tags
+        id: get-tags
+        env:
+          KERNEL_VERSION_TAG: ${{ inputs.tag || steps.get-kernel-version.outputs.VM_KERNEL_VERSION }}
+          CACHE_TAG: ${{ needs.setup-build-vm-kernel-image.outputs.last-kernel-sha }}
+        run: |
+          # A comma-separated list of tags
+          TAGS="${VM_KERNEL_IMAGE}:${KERNEL_VERSION_TAG}"
+          TAGS="${VM_KERNEL_IMAGE}:${CACHE_TAG},${TAGS}"
+          TAGS="${VM_KERNEL_IMAGE}:${GITHUB_RUN_ID},${TAGS}"
+
+          echo "tags=${TAGS}" >> $GITHUB_OUTPUT
+
+          # `docker/build-push-action@v3` returns all ${TAGS} in metadata ("image.name" field), so it can't be used a image name right away.
+          # Choose one of them as a "canonical" tag and use it to construct the job output (along with a digest provided by `docker/build-push-action@v3`).
+          echo "canonical=${VM_KERNEL_IMAGE}:${GITHUB_RUN_ID}" >> $GITHUB_OUTPUT
+
       - name: build linux kernel
         id: build-linux-kernel
         uses: docker/build-push-action@v3
         with:
           build-args: KERNEL_VERSION=${{ steps.get-kernel-version.outputs.VM_KERNEL_VERSION }}
-          context: neonvm/hack
+          context: neonvm/hack/kernel
           platforms: linux/amd64
           # Push kernel image only for scheduled builds or if workflow_dispatch/workflow_call input is true
-          push: ${{ github.event_name == 'schedule' && 'true' || ( inputs.push || 'false' ) }}
+          push: true
           pull: true
           no-cache: true
-          file: neonvm/hack/Dockerfile.kernel-builder
-          # Tag kernel image with workflow_dispatch/workflow_call input or with the kernel version by default
-          tags: ${{ env.VM_KERNEL_IMAGE }}:${{ inputs.tag || steps.get-kernel-version.outputs.VM_KERNEL_VERSION }}
+          file: neonvm/hack/kernel/Dockerfile.kernel-builder
+          tags: ${{ steps.get-tags.outputs.tags }}
+
       - name: remove custom docker config directory
         if: always()
         run: |
diff --git a/.gitignore b/.gitignore
@@ -25,5 +25,6 @@ testbin/*
 
 *.qcow2
 neonvm/hack/vmlinuz
+neonvm/hack/kernel/vmlinuz
 
 rendered_manifests
diff --git a/Makefile b/Makefile
@@ -223,8 +223,8 @@ endif
 
 .PHONY: kernel
 kernel: ## Build linux kernel.
-	rm -f neonvm/hack/vmlinuz; \
-	linux_config=$$(ls neonvm/hack/linux-config-*) \
+	rm -f neonvm/hack/vmlinuz neonvm/hack/kernel/vmlinuz; \
+	linux_config=$$(ls neonvm/hack/kernel/linux-config-*) \
 	kernel_version=$${linux_config##*-} \
 	iidfile=$$(mktemp /tmp/iid-XXXXXX); \
 	trap "rm $$iidfile" EXIT; \
@@ -234,10 +234,10 @@ kernel: ## Build linux kernel.
 		--pull \
 		--load \
 		--iidfile $$iidfile \
-		--file neonvm/hack/Dockerfile.kernel-builder \
-		neonvm/hack; \
+		--file neonvm/hack/kernel/Dockerfile.kernel-builder \
+		neonvm/hack/kernel; \
 	id=$$(docker create $$(cat $$iidfile)); \
-	docker cp $$id:/vmlinuz neonvm/hack/vmlinuz; \
+	docker cp $$id:/vmlinuz neonvm/hack/kernel/vmlinuz; \
 	docker rm -f $$id
 
 .PHONY: check-local-context
diff --git a/neonvm/hack/kernel/Dockerfile.kernel-builder b/neonvm/hack/kernel/Dockerfile.kernel-builder
diff --git a/neonvm/hack/kernel/linux-config-6.1.63 b/neonvm/hack/kernel/linux-config-6.1.63
diff --git a/neonvm/runner/Dockerfile b/neonvm/runner/Dockerfile
@@ -45,6 +45,6 @@ RUN apk add --no-cache \
     cgroup-tools
 
 COPY --from=builder /runner /usr/bin/runner
-COPY neonvm/hack/vmlinuz /vm/kernel/vmlinuz
+COPY neonvm/hack/kernel/vmlinuz /vm/kernel/vmlinuz
 
 ENTRYPOINT ["/sbin/tini", "--", "runner"]
