microsoft · ms-mahuber · Aug 20, 2024 · Jul 24, 2024
@@ -39,6 +39,10 @@ deploy: deploy-package deploy-uvm
 deploy-package:
 	./package_install.sh
 
+.PHONY: deploy-package-tools
+deploy-package-tools:
+	./package_tools_install.sh
+
 .PHONY: deploy-uvm
 deploy-uvm:
 	./uvm_install.sh
@@ -50,6 +54,10 @@ deploy-confpods: deploy-confpods-package deploy-confpods-uvm
 deploy-confpods-package:
 	CONF_PODS=yes ./package_install.sh
 
+.PHONY: deploy-confpods-package-tools
+deploy-confpods-package-tools:
+	CONF_PODS=yes ./package_tools_install.sh
+
 .PHONY: deploy-confpods-uvm
 deploy-confpods-uvm:
 	CONF_PODS=yes ./uvm_install.sh
@@ -89,16 +89,6 @@ sudo dnf -y install git vim golang rust cargo build-essential protobuf-compiler
 
 **Note:** The kernel-uvm-devel package in step above is only required for Confidential Containers and can be omitted for regular Kata Containers builds.
 
-When intending to build the components for Confidential Containers, install the IGVM tool that will be used by the build tooling to create IGVM files with their reference measurements for the ConfPods UVM.
-
-```
-pushd kata-containers/tools/osbuilder/igvm-builder
-sudo ./igvm_builder.sh -i
-popd
-```
-
-This command installs the latest release of the [IGVM tooling](https://github.com/microsoft/igvm-tooling/) using `pip3 install`. The tool can be uninstalled at any time by calling the script using the -u parameter instead.
-
 # Optional: Build and deploy the containerd fork from scratch
 
 ```
@@ -117,8 +107,23 @@ sudo systemctl restart containerd
 # Build and install the Kata(-CC) host and guest components
 
 Clone the Microsoft's fork of the kata-containers repository:
+
 ```git clone https://github.com/microsoft/kata-containers.git```
 
+## Install IGVM tooling for ConfPods
+
+When intending to build the components for Confidential Containers, install the IGVM tool that will be used by the build tooling to create IGVM files with their reference measurements for the ConfPods UVM.
+
+```
+pushd kata-containers/tools/osbuilder/igvm-builder
+sudo ./igvm_builder.sh -i
+popd
+```
+
+This command installs the latest release of the [IGVM tooling](https://github.com/microsoft/igvm-tooling/) using `pip3 install`. The tool can be uninstalled at any time by calling the script using the -u parameter instead.
+
+## Build and deploy
+
 To build and install Kata Containers for AKS components, run:
 ```
 pushd kata-containers/tools/osbuilder/node-builder/azure-linux
@@ -139,7 +144,7 @@ The `all[-confpods]` target runs the targets `package[-confpods]` and `uvm[-conf
 
 Notes:
   - To retrieve more detailed build output, prefix the make commands with `DEBUG=1`.
-  - To build for Mariner 3, prefix the make commands that build artifacts with `OS_VERSION=3.0`
+  - To build for Azure Linux 3, prefix the make commands that build artifacts with `OS_VERSION=3.0`
   - For build and deployment of both Kata and Kata-CC artifacts, first run the `make all` and `make deploy` commands to build and install the Kata Containers for AKS components followed by `make clean`, and then run `make all-confpods` and `make deploy-confpods` to build and install the Confidential Containers for AKS components - or vice versa (using `make clean-confpods`).
 
 # Run Kata (Confidential) Containers
@@ -164,17 +169,17 @@ The following instructions serve as a general reference:
   sudo dnf -y install cni
   ```
 
-- Set a proper CNI configuration: This step is omitted as it depends on the individual needs.
+- Set a proper CNI configuration and create a sample pod manifest: This step is omitted as it depends on the individual needs.
 
 - Run pods with `crictl`, for example:
 
   `sudo crictl runp -T 30s -r <handler-name> <sample-pod.yaml>`
 
 - Run containers with `ctr`, for example a confidential container:
 
-  `sudo ctr image pull --snapshotter=tardev docker.io/library/busybox:latest`
+  `sudo ctr -n=k8s.io image pull --snapshotter=tardev docker.io/library/busybox:latest`
 
-  `sudo ctr run --cni --runtime io.containerd.run.kata-cc.v2 --runtime-config-path /opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml --snapshotter tardev -t --rm docker.io/library/busybox:latest hello sh`
+  `sudo ctr -n=k8s.io run --cni --runtime io.containerd.run.kata-cc.v2 --runtime-config-path /opt/confidential-containers/share/defaults/kata-containers/configuration-clh-snp.toml --snapshotter tardev -t --rm docker.io/library/busybox:latest hello sh`
 
 For further usage we refer to the upstream `crictl` (or `ctr`) and CNI documentation.
 
@@ -183,7 +188,7 @@ For further usage we refer to the upstream `crictl` (or `ctr`) and CNI documenta
 If your environment was set up through `az aks create` the respective node is ready to run Kata (Confidential) Containers as AKS Kubernetes pods.
 Other types of Kubernetes clusters should work as well - but this document doesn't cover how to set-up those clusters.
 
-Next, apply the kata and kata-cc runtime classes on the machine that holds your kubeconfig file:
+Next, apply the kata and kata-cc runtime classes on the machine that holds your kubeconfig file, example AKS:
 ```
 cat << EOF > runtimeClass-kata-cc.yaml
 kind: RuntimeClass
@@ -193,8 +198,7 @@ metadata:
 handler: kata-cc
 overhead:
     podFixed:
-        memory: "160Mi"
-        cpu: "250m"
+        memory: "2Gi"
 scheduling:
   nodeSelector:
     katacontainers.io/kata-runtime: "true"
@@ -206,10 +210,6 @@ apiVersion: node.k8s.io/v1
 metadata:
     name: kata
 handler: kata
-overhead:
-    podFixed:
-        memory: "160Mi"
-        cpu: "250m"
 scheduling:
   nodeSelector:
     katacontainers.io/kata-runtime: "true"

@@ -14,6 +14,8 @@ OS_VERSION=${OS_VERSION:-2.0}
 
 if [ "${CONF_PODS}" == "yes" ]; then
 	INSTALL_PATH_PREFIX="/opt/confidential-containers"
+	UVM_TOOLS_PATH_OSB="${INSTALL_PATH_PREFIX}/uvm/tools/osbuilder"
+	UVM_TOOLS_PATH_SRC="${INSTALL_PATH_PREFIX}/uvm/src"
 	UVM_PATH="${INSTALL_PATH_PREFIX}/share/kata-containers"
 	IMG_FILE_NAME="kata-containers.img"
 	IGVM_FILE_NAME="kata-containers-igvm.img"
@@ -22,16 +24,21 @@ if [ "${CONF_PODS}" == "yes" ]; then
 	UVM_DBG_MEASUREMENT_FILE_NAME="igvm-debug-measurement.cose"
 	SHIM_CONFIG_PATH="${INSTALL_PATH_PREFIX}/share/defaults/kata-containers"
 	SHIM_CONFIG_FILE_NAME="configuration-clh-snp.toml"
+	SHIM_CONFIG_INST_FILE_NAME="${SHIM_CONFIG_FILE_NAME}"
 	SHIM_DBG_CONFIG_FILE_NAME="configuration-clh-snp-debug.toml"
+	SHIM_DBG_CONFIG_INST_FILE_NAME="${SHIM_DBG_CONFIG_FILE_NAME}"
 	DEBUGGING_BINARIES_PATH="${INSTALL_PATH_PREFIX}/bin"
 	SHIM_BINARIES_PATH="/usr/local/bin"
 	SHIM_BINARY_NAME="containerd-shim-kata-cc-v2"
 else
 	INSTALL_PATH_PREFIX="/usr"
+	UVM_TOOLS_PATH_OSB="/opt/kata-containers/uvm/tools/osbuilder"
+	UVM_TOOLS_PATH_SRC="/opt/kata-containers/uvm/src"
 	UVM_PATH="/var/cache/kata-containers/osbuilder-images/kernel-uvm"
 	INITRD_FILE_NAME="kata-containers-initrd.img"
 	SHIM_CONFIG_PATH="${INSTALL_PATH_PREFIX}/share/defaults/kata-containers"
 	SHIM_CONFIG_FILE_NAME="configuration-clh.toml"
+	SHIM_CONFIG_INST_FILE_NAME="configuration.toml"
 	DEBUGGING_BINARIES_PATH="${INSTALL_PATH_PREFIX}/local/bin"
 	SHIM_BINARIES_PATH="${INSTALL_PATH_PREFIX}/local/bin"
 	SHIM_BINARY_NAME="containerd-shim-kata-v2"

@@ -36,9 +36,8 @@ fi
 # On Mariner 3.0 we use cgroupsv2 with a single sandbox cgroup
 if [ "${OS_VERSION}" == "3.0" ]; then
 	runtime_make_flags+=" DEFSANDBOXCGROUPONLY=true"
-	echo "test1"
 fi
-echo "test"
+
 # add BUILD_TYPE=debug to build a debug agent (result in significantly increased agent binary size)
 # this will require to add same flag to the `make install` section for the agent in uvm_build.sh
 agent_make_flags="LIBC=gnu OPENSSL_NO_VENDOR=Y DESTDIR=${AGENT_INSTALL_DIR}"

@@ -38,7 +38,7 @@ if [ "${CONF_PODS}" == "yes" ]; then
 	cp -a --backup=numbered src/tardev-snapshotter/tardev-snapshotter.service ${PREFIX}/usr/lib/systemd/system/
 
 	echo "Installing SNP shim debug configuration"
-	cp -a --backup=numbered src/runtime/config/"${SHIM_DBG_CONFIG_FILE_NAME}" "${PREFIX}/${SHIM_CONFIG_PATH}"
+	cp -a --backup=numbered src/runtime/config/"${SHIM_DBG_CONFIG_FILE_NAME}" "${PREFIX}/${SHIM_CONFIG_PATH}"/"${SHIM_DBG_CONFIG_INST_FILE_NAME}"
 
 	echo "Enabling and starting snapshotter service"
 	if [ "${START_SERVICES}" == "yes" ]; then
@@ -55,6 +55,6 @@ cp -a --backup=numbered src/runtime/data/kata-collect-data.sh "${PREFIX}/${DEBUG
 echo "Installing shim binary and configuration"
 cp -a --backup=numbered src/runtime/containerd-shim-kata-v2 "${PREFIX}/${SHIM_BINARIES_PATH}"/"${SHIM_BINARY_NAME}"
 
-cp -a --backup=numbered src/runtime/config/"${SHIM_CONFIG_FILE_NAME}" "${PREFIX}/${SHIM_CONFIG_PATH}"
+cp -a --backup=numbered src/runtime/config/"${SHIM_CONFIG_FILE_NAME}" "${PREFIX}/${SHIM_CONFIG_PATH}/${SHIM_CONFIG_INST_FILE_NAME}"
 
 popd
@@ -0,0 +1,68 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2024 Microsoft Corporation
+#
+# SPDX-License-Identifier: Apache-2.0
+
+set -o errexit
+set -o pipefail
+set -o errtrace
+
+[ -n "$DEBUG" ] && set -x
+
+CONF_PODS=${CONF_PODS:-no}
+PREFIX=${PREFIX:-}
+
+script_dir="$(dirname $(readlink -f $0))"
+repo_dir="${script_dir}/../../../../"
+
+common_file="common.sh"
+source "${common_file}"
+
+pushd "${repo_dir}"
+
+echo "Creating target directories"
+mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/scripts"
+mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/rootfs-builder/cbl-mariner"
+mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/bin"
+mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/lib/systemd/system"
+
+if [ "${CONF_PODS}" == "yes" ]; then
+	mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_SRC}/kata-opa"
+	mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_SRC}/tarfs"
+	mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/image-builder"
+	mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/igvm-builder/azure-linux"
+else
+	mkdir -p "${PREFIX}/${UVM_TOOLS_PATH_OSB}/initrd-builder"
+fi
+
+echo "Installing UVM build scripting"
+cp -a --backup=numbered tools/osbuilder/Makefile "${PREFIX}/${UVM_TOOLS_PATH_OSB}/"
+cp -a --backup=numbered tools/osbuilder/scripts/lib.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/scripts/"
+cp -a --backup=numbered tools/osbuilder/rootfs-builder/rootfs.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/rootfs-builder/"
+cp -a --backup=numbered tools/osbuilder/rootfs-builder/cbl-mariner/config.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/rootfs-builder/cbl-mariner/"
+cp -a --backup=numbered tools/osbuilder/rootfs-builder/cbl-mariner/rootfs_lib.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/rootfs-builder/cbl-mariner/"
+cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/Makefile "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/"
+cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/clean.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/"
+cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/common.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/"
+cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/uvm_build.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/"
+
+echo "Installing agent binary and service files"
+cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/agent-install/usr/bin/kata-agent "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/bin/"
+cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/agent-install/usr/lib/systemd/system/kata-containers.target "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/lib/systemd/system/"
+cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/agent-install/usr/lib/systemd/system/kata-agent.service "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/lib/systemd/system/"
+
+if [ "${CONF_PODS}" == "yes" ]; then
+	cp -a --backup=numbered src/kata-opa/allow-all.rego "${PREFIX}/${UVM_TOOLS_PATH_SRC}/kata-opa/"
+	cp -a --backup=numbered src/kata-opa/allow-set-policy.rego "${PREFIX}/${UVM_TOOLS_PATH_SRC}/kata-opa/"
+	cp -a --backup=numbered src/tarfs/Makefile "${PREFIX}/${UVM_TOOLS_PATH_SRC}/tarfs/"
+	cp -a --backup=numbered src/tarfs/tarfs.c "${PREFIX}/${UVM_TOOLS_PATH_SRC}/tarfs/"
+	cp -a --backup=numbered tools/osbuilder/image-builder/image_builder.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/image-builder/"
+	cp -a --backup=numbered tools/osbuilder/igvm-builder/igvm_builder.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/igvm-builder/"
+	cp -a --backup=numbered tools/osbuilder/igvm-builder/azure-linux/config.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/igvm-builder/azure-linux/"
+	cp -a --backup=numbered tools/osbuilder/igvm-builder/azure-linux/igvm_lib.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/igvm-builder/azure-linux/"
+else
+	cp -a --backup=numbered tools/osbuilder/initrd-builder/initrd_builder.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/initrd-builder/"
+fi
+
+popd