tools: Improve igvm-builder and node-builder/azure-linux scripting

- Support for Mariner 3 builds using OS_VERSION variable - Improvements to IGVM build process and flow as described in README - Adoption of using only cloud-hypervisor-cvm on CBL-Mariner Signed-off-by: Manuel Huber <[email protected]>
microsoft · Jul 15, 2024 · 2f672a1 · 2f672a1
1 parent e7cb0d1
commit 2f672a1
Show file tree

Hide file tree

Showing 17 changed files with 203 additions and 126 deletions.
diff --git a/src/runtime/Makefile b/src/runtime/Makefile
@@ -175,9 +175,6 @@ QEMUVIRTIOFSPATH := $(QEMUBINDIR)/$(QEMUVIRTIOFSCMD)
 CLHPATH := $(CLHBINDIR)/$(CLHCMD)
 CLHVALIDHYPERVISORPATHS := [\"$(CLHPATH)\"]
 
-CLHSNPPATH := $(CLHBINDIR)/$(CLHSNPCMD)
-CLHSNPVALIDHYPERVISORPATHS := [\"$(CLHSNPPATH)\"]
-
 FCPATH = $(FCBINDIR)/$(FCCMD)
 FCVALIDHYPERVISORPATHS := [\"$(FCPATH)\"]
 FCJAILERPATH = $(FCBINDIR)/$(FCJAILERCMD)
@@ -566,9 +563,6 @@ USER_VARS += ACRNCTLPATH
 USER_VARS += ACRNVALIDCTLPATHS
 USER_VARS += CLHPATH
 USER_VARS += CLHVALIDHYPERVISORPATHS
-USER_VARS += CLHSNPCMD
-USER_VARS += CLHSNPPATH
-USER_VARS += CLHSNPVALIDHYPERVISORPATHS
 USER_VARS += FIRMWAREPATH_CLH
 USER_VARS += FCCMD
 USER_VARS += FCPATH

diff --git a/src/runtime/arch/amd64-options.mk b/src/runtime/arch/amd64-options.mk
@@ -26,7 +26,6 @@ ACRNCTLCMD := acrnctl
 
 # cloud-hypervisor binary name
 CLHCMD := cloud-hypervisor
-CLHSNPCMD := cloud-hypervisor-snp
 
 DEFSTATICRESOURCEMGMT_CLH := false
 

diff --git a/src/runtime/arch/arm64-options.mk b/src/runtime/arch/arm64-options.mk
@@ -19,7 +19,6 @@ FCJAILERCMD := jailer
 
 # cloud-hypervisor binary name
 CLHCMD := cloud-hypervisor
-CLHSNPCMD := cloud-hypervisor-snp
 
 DEFSTATICRESOURCEMGMT_CLH := true
 

diff --git a/src/runtime/config/configuration-clh-snp.toml.in b/src/runtime/config/configuration-clh-snp.toml.in
@@ -11,7 +11,7 @@
 # XXX:   Type: @PROJECT_TYPE@
 
 [hypervisor.clh]
-path = "@CLHSNPPATH@"
+path = "@CLHPATH@"
 igvm = "@IGVMPATH@"
 image = "@IMAGEPATH@"
 
@@ -80,8 +80,8 @@ enable_annotations = @DEFENABLEANNOTATIONS@
 # List of valid annotations values for the hypervisor
 # Each member of the list is a path pattern as described by glob(3).
 # The default if not set is empty (all annotations rejected.)
-# Your distribution recommends: @CLHSNPVALIDHYPERVISORPATHS@
-valid_hypervisor_paths = @CLHSNPVALIDHYPERVISORPATHS@
+# Your distribution recommends: @CLHVALIDHYPERVISORPATHS@
+valid_hypervisor_paths = @CLHVALIDHYPERVISORPATHS@
 
 # Optional space-separated list of options to pass to the guest kernel.
 # For example, use `kernel_params = "vsyscall=emulate"` if you are having

diff --git a/tools/osbuilder/.gitignore b/tools/osbuilder/.gitignore
@@ -9,3 +9,5 @@ kata-containers-initrd.img
 kata-containers.img
 rootfs-builder/centos/RPM-GPG-KEY-*
 typescript
+node-builder/azure-linux/agent-install
+igvm-builder/igvm-tooling
diff --git a/tools/osbuilder/Makefile b/tools/osbuilder/Makefile
@@ -9,7 +9,6 @@ ROOTFS_BUILDER := $(MK_DIR)/rootfs-builder/rootfs.sh
 INITRD_BUILDER := $(MK_DIR)/initrd-builder/initrd_builder.sh
 IMAGE_BUILDER  := $(MK_DIR)/image-builder/image_builder.sh
 IGVM_BUILDER   := $(MK_DIR)/igvm-builder/igvm_builder.sh
-IGVM_TOOL_SRC  := $(MK_DIR)/igvm-tooling
 
 DISTRO                ?= ubuntu
 BUILD_METHOD          := distro
@@ -222,7 +221,6 @@ install-scripts:
 clean:
 	rm -rf $(TARGET_ROOTFS_MARKER) $(TARGET_ROOTFS) $(TARGET_IMAGE) $(TARGET_INITRD) $(DRACUT_OVERLAY_DIR) $(TARGET_IGVM) $(TARGET_IGVM_DEBUG) $(TARGET_IGVM_MSMT) $(TARGET_IGVM_DEBUG_MSMT) $(TARGET_IGVM_LOG)
 	rm -rf $(IGVM_TOOL_SRC)
-	pip3 uninstall -y msigvm
 
 # Prints the name of the variable passed as suffix to the print- target,
 # E.g., if Makefile contains:

diff --git a/tools/osbuilder/igvm-builder/azure-linux/config.sh b/tools/osbuilder/igvm-builder/azure-linux/config.sh
@@ -5,16 +5,21 @@
 # SPDX-License-Identifier: Apache-2.0
 
 # this is where the kernel-uvm package installation places bzImage, see SPEC file
-bzimage_bin="/usr/share/cloud-hypervisor/bzImage"
+BZIMAGE_BIN="/usr/share/cloud-hypervisor/bzImage"
 
-igvm_extract_folder="igvm-tooling"
-clh_acpi_tables_dir="${igvm_extract_folder}/src/igvm/acpi/acpi-clh/"
-igvmgen_py_file="${igvm_extract_folder}/src/igvm/igvmgen.py"
+IGVM_EXTRACT_FOLDER="${SCRIPT_DIR}/igvm-tooling"
+CLH_ACPI_TABLES_DIR="${IGVM_EXTRACT_FOLDER}/src/igvm/acpi/acpi-clh/"
+IGVM_PY_FILE="${IGVM_EXTRACT_FOLDER}/src/igvm/igvmgen.py"
 
-igvm_vars="-kernel ${bzimage_bin} -boot_mode x64 -vtl 0 -svme 1 -encrypted_page 1 -pvalidate_opt 1 -acpi ${clh_acpi_tables_dir}"
+IGVM_BUILD_VARS="-kernel ${BZIMAGE_BIN} -boot_mode x64 -vtl 0 -svme 1 -encrypted_page 1 -pvalidate_opt 1 -acpi ${CLH_ACPI_TABLES_DIR}"
 
-igvm_kernel_params_common="dm-mod.create=\"dm-verity,,,ro,0 ${data_sectors} verity 1 /dev/vda1 /dev/vda2 ${data_block_size} ${hash_block_size} ${data_blocks} 0 sha256 ${root_hash} ${salt}\" \
+IGVM_KERNEL_PARAMS_COMMON="dm-mod.create=\"dm-verity,,,ro,0 ${IMAGE_DATA_SECTORS} verity 1 /dev/vda1 /dev/vda2 ${IMAGE_DATA_BLOCK_SIZE} ${IMAGE_HASH_BLOCK_SIZE} ${IMAGE_DATA_BLOCKS} 0 sha256 ${IMAGE_ROOT_HASH} ${IMAGE_SALT}\" \
 	root=/dev/dm-0 rootflags=data=ordered,errors=remount-ro ro rootfstype=ext4 panic=1 no_timer_check noreplace-smp systemd.unit=kata-containers.target systemd.mask=systemd-networkd.service \
 	systemd.mask=systemd-networkd.socket agent.enable_signature_verification=false"
-igvm_kernel_prod_params="${igvm_kernel_params_common} quiet"
-igvm_kernel_debug_params="${igvm_kernel_params_common} console=hvc0 systemd.log_target=console agent.log=debug agent.debug_console agent.debug_console_vport=1026"
+IGVM_KERNEL_PROD_PARAMS="${IGVM_KERNEL_PARAMS_COMMON} quiet"
+IGVM_KERNEL_DEBUG_PARAMS="${IGVM_KERNEL_PARAMS_COMMON} console=hvc0 systemd.log_target=console agent.log=debug agent.debug_console agent.debug_console_vport=1026"
+
+IGVM_FILE_NAME="kata-containers-igvm.img"
+IGVM_DBG_FILE_NAME="kata-containers-igvm-debug.img"
+IGVM_MEASUREMENT_FILE_NAME="igvm-measurement.cose"
+IGVM_DBG_MEASUREMENT_FILE_NAME="igvm-debug-measurement.cose"
diff --git a/tools/osbuilder/igvm-builder/azure-linux/igvm_lib.sh b/tools/osbuilder/igvm-builder/azure-linux/igvm_lib.sh
@@ -4,10 +4,11 @@
 #
 # SPDX-License-Identifier: Apache-2.0
 
-install_igvm()
+install_igvm_tool()
 {
-	if [ -d ${igvm_extract_folder} ]; then
-		echo "${igvm_extract_folder} folder already exists, assuming tool is already installed"
+	echo "Installing IGVM tool"
+	if [ -d ${IGVM_EXTRACT_FOLDER} ]; then
+		echo "${IGVM_EXTRACT_FOLDER} folder already exists, assuming tool is already installed"
 		return
 	fi
 
@@ -16,10 +17,54 @@ install_igvm()
 	echo "Determining and downloading latest IGVM tooling release, and extracting including ACPI tables"
 	IGVM_VER=$(curl -sL "https://api.github.com/repos/microsoft/igvm-tooling/releases/latest" | jq -r .tag_name | sed 's/^v//')
 	curl -sL "https://github.com/microsoft/igvm-tooling/archive/refs/tags/${IGVM_VER}.tar.gz" | tar --no-same-owner -xz
-	mv igvm-tooling-${IGVM_VER} ${igvm_extract_folder}
+	mv igvm-tooling-${IGVM_VER} ${IGVM_EXTRACT_FOLDER}
 
-	echo "Installing IGVM module msigvm via pip3"
-	pushd ${igvm_extract_folder}/src
+	echo "Installing IGVM module msigvm (${IGVM_VER}) via pip3"
+	pushd ${IGVM_EXTRACT_FOLDER}/src
 	pip3 install --no-deps ./
 	popd
 }
+
+uninstall_igvm_tool()
+{
+	echo "Uninstalling IGVM tool"
+
+	rm -rf ${IGVM_EXTRACT_FOLDER}
+	pip3 uninstall -y msigvm
+}
+
+build_igvm_files()
+{
+	echo "Reading Kata image dm_verity root hash information from root_hash file"
+	ROOT_HASH_FILE="${SCRIPT_DIR}/../root_hash.txt"
+
+	if [ ! -f "${ROOT_HASH_FILE}" ]; then
+		echo "Could no find image root hash file '${ROOT_HASH_FILE}', aborting"
+		exit 1
+	fi
+
+	IMAGE_ROOT_HASH=$(sed -e 's/Root hash:\s*//g;t;d' "${ROOT_HASH_FILE}")
+	IMAGE_SALT=$(sed -e 's/Salt:\s*//g;t;d' "${ROOT_HASH_FILE}")
+	IMAGE_DATA_BLOCKS=$(sed -e 's/Data blocks:\s*//g;t;d' "${ROOT_HASH_FILE}")
+	IMAGE_DATA_BLOCK_SIZE=$(sed -e 's/Data block size:\s*//g;t;d' "${ROOT_HASH_FILE}")
+	IMAGE_DATA_SECTORS_PER_BLOCK=$((IMAGE_DATA_BLOCK_SIZE / 512))
+	IMAGE_DATA_SECTORS=$((IMAGE_DATA_BLOCKS * IMAGE_DATA_SECTORS_PER_BLOCK))
+	IMAGE_HASH_BLOCK_SIZE=$(sed -e 's/Hash block size:\s*//g;t;d' "${ROOT_HASH_FILE}")
+
+	# reloading the config file as various variables depend on above values
+	load_config_distro
+
+	echo "Building (debug) IGVM files and creating their reference measurement files"
+	# we could call into the installed binary '~/.local/bin/igvmgen' when adding to PATH or, better, into 'python3 -m msigvm'
+	# however, as we still need the installation directory for the ACPI tables, we leave things as is for now
+	# at the same time we seem to need to call pip3 install for invoking the tool at all
+	python3 ${IGVM_PY_FILE} $IGVM_BUILD_VARS -o $IGVM_FILE_NAME -measurement_file $IGVM_MEASUREMENT_FILE_NAME -append "$IGVM_KERNEL_PROD_PARAMS" -svn $SVN
+	python3 ${IGVM_PY_FILE} $IGVM_BUILD_VARS -o $IGVM_DBG_FILE_NAME -measurement_file $IGVM_DBG_MEASUREMENT_FILE_NAME -append "$IGVM_KERNEL_DEBUG_PARAMS" -svn $SVN
+
+	if [ "${PWD}" -ef "$(readlink -f $OUT_DIR)" ]; then
+		echo "OUT_DIR matches with current dir, not moving build artifacts"
+	else
+		echo "Moving build artifacts to ${OUT_DIR}"
+		mv $IGVM_FILE_NAME $IGVM_DBG_FILE_NAME $IGVM_MEASUREMENT_FILE_NAME $IGVM_DBG_MEASUREMENT_FILE_NAME $OUT_DIR
+	fi
+}
diff --git a/tools/osbuilder/igvm-builder/igvm_builder.sh b/tools/osbuilder/igvm-builder/igvm_builder.sh
@@ -10,18 +10,18 @@ set -o errtrace
 
 [ -n "$DEBUG" ] && set -x
 
-script_dir="$(dirname $(readlink -f $0))"
+SCRIPT_DIR="$(dirname $(readlink -f $0))"
 
 # distro-specific config file
 typeset -r CONFIG_SH="config.sh"
 
 # Name of an optional distro-specific file which, if it exists, must implement the
-# install_igvm() function.
+# install_igvm_tool, build_igvm_files, and uninstall_igvm_tool functions.
 typeset -r LIB_SH="igvm_lib.sh"
 
-build_igvm_distro()
+load_config_distro()
 {
-	distro_config_dir="${script_dir}/${distro}"
+	distro_config_dir="${SCRIPT_DIR}/${DISTRO}"
 
 	[ -d "${distro_config_dir}" ] || die "Could not find configuration directory '${distro_config_dir}'"
 
@@ -31,50 +31,20 @@ build_igvm_distro()
 		source "${igvm_lib}"
 	fi
 
-	root_hash_file="${script_dir}/../root_hash.txt"
-
-	if [ ! -f "${root_hash_file}" ]; then
-		echo "Could no find image root hash file '${root_hash_file}', aborting"
-		exit 1
-	fi
-
-	echo "Reading image dm-verity root hash values"
-	root_hash=$(sed -e 's/Root hash:\s*//g;t;d' "${root_hash_file}")
-	salt=$(sed -e 's/Salt:\s*//g;t;d' "${root_hash_file}")
-	data_blocks=$(sed -e 's/Data blocks:\s*//g;t;d' "${root_hash_file}")
-	data_block_size=$(sed -e 's/Data block size:\s*//g;t;d' "${root_hash_file}")
-	data_sectors_per_block=$((data_block_size / 512))
-	data_sectors=$((data_blocks * data_sectors_per_block))
-	hash_block_size=$(sed -e 's/Hash block size:\s*//g;t;d' "${root_hash_file}")
-
 	# Source config.sh from distro, depends on root_hash based variables here
 	igvm_config="${distro_config_dir}/${CONFIG_SH}"
 	source "${igvm_config}"
-
-	echo "Install IGVM tool"
-	install_igvm
-
-	echo "Build IGVM (debug) file and calculate reference measurements"
-	# we could call into the installed binary '~/.local/bin/igvmgen' when adding to PATH or, better, into 'python3 -m msigvm'
-	# however, as we still need the installation directory for the ACPI tables, we leave things as is for now
-	# at the same time we seem to need to call pip3 install for invoking the tool at all
-	python3 ${igvmgen_py_file} $igvm_vars -o kata-containers-igvm.img -measurement_file igvm-measurement.cose -append "$igvm_kernel_prod_params" -svn $SVN
-	python3 ${igvmgen_py_file} $igvm_vars -o kata-containers-igvm-debug.img -measurement_file igvm-debug-measurement.cose -append "$igvm_kernel_debug_params" -svn $SVN
-
-	if [ "${PWD}" -ef "$(readlink -f $OUT_DIR)" ]; then
-		echo "OUT_DIR matches with current dir, not moving build artifacts"
-	else
-		echo "Moving build artifacts to ${OUT_DIR}"
-		mv igvm-measurement.cose kata-containers-igvm.img igvm-debug-measurement.cose kata-containers-igvm-debug.img $OUT_DIR
-	fi
 }
 
-distro="azure-linux"
+DISTRO="azure-linux"
+MODE="build"
 
-while getopts ":o:s:" OPTIONS; do
+while getopts ":o:s:iu" OPTIONS; do
 	case "${OPTIONS}" in
 		o ) OUT_DIR=$OPTARG ;;
 		s ) SVN=$OPTARG ;;
+		i ) MODE="install" ;;
+		u ) MODE="uninstall" ;;
 		\? )
 			echo "Error - Invalid Option: -$OPTARG" 1>&2
 			exit 1
@@ -89,11 +59,24 @@ done
 echo "IGVM builder script"
 echo "-- OUT_DIR -> $OUT_DIR"
 echo "-- SVN -> $SVN"
-echo "-- distro -> $distro"
+echo "-- DISTRO -> $DISTRO"
+echo "-- MODE -> $MODE"
 
-if [ -n "$distro" ]; then
-  build_igvm_distro
+if [ -n "$DISTRO" ]; then
+	load_config_distro
 else
-  echo "distro must be specified"
-  exit 1
+	echo "DISTRO must be specified"
+	exit 1
 fi
+
+case "$MODE" in
+	"install")
+		install_igvm_tool
+		;;
+	"uninstall")
+		uninstall_igvm_tool
+		;;
+	"build")
+		build_igvm_files
+		;;
+esac
diff --git a/tools/osbuilder/node-builder/azure-linux/Makefile b/tools/osbuilder/node-builder/azure-linux/Makefile
@@ -33,11 +33,23 @@ clean-confpods:
 	CONF_PODS=yes ./clean.sh
 
 .PHONY: deploy
-deploy:
+deploy: deploy-package deploy-uvm
+
+.PHONY: deploy-package
+deploy-package:
 	./package_install.sh
+
+.PHONY: deploy-uvm
+deploy-uvm:
 	./uvm_install.sh
 
 .PHONY: deploy-confpods
-deploy-confpods:
+deploy-confpods: deploy-confpods-package deploy-confpods-uvm
+
+.PHONY: deploy-confpods-package
+deploy-confpods-package:
 	CONF_PODS=yes ./package_install.sh
+
+.PHONY: deploy-confpods-uvm
+deploy-confpods-uvm:
 	CONF_PODS=yes ./uvm_install.sh
diff --git a/tools/osbuilder/node-builder/azure-linux/README.md b/tools/osbuilder/node-builder/azure-linux/README.md
@@ -76,17 +76,29 @@ sudo tee -a /etc/containerd/config.toml 2&>1 <<EOF
 EOF
 ```
 
-Restart containerd (ensureing the configuration file is intact):
+Restart containerd (ensuring the configuration file is intact):
 
 ```sudo systemctl restart containerd```
 
 # Install general build dependencies
 
 ```
 sudo dnf -y makecache
-sudo dnf -y install git vim golang rust build-essential protobuf-compiler protobuf-devel expect openssl-devel clang-devel libseccomp-devel parted qemu-img btrfs-progs-devel device-mapper-devel cmake fuse-devel jq curl kata-packages-uvm-build kernel-uvm-devel
+sudo dnf -y install git vim golang rust cargo build-essential protobuf-compiler protobuf-devel expect openssl-devel clang-devel libseccomp-devel parted qemu-img btrfs-progs-devel device-mapper-devel cmake fuse-devel jq curl kata-packages-uvm-build kernel-uvm-devel
 ```
 
+**Note:** The kernel-uvm-devel package in step above is only required for Confidential Containers and can be omitted for regular Kata Containers builds.
+
+When intending to build the components for Confidential Containers, install the IGVM tool that will be used by the build tooling to create IGVM files with their reference measurements for the ConfPods UVM.
+
+```
+pushd kata-containers/tools/osbuilder/igvm-builder
+sudo ./igvm_builder.sh -i
+popd
+```
+
+This command installs the latest release of the [IGVM tooling](https://github.com/microsoft/igvm-tooling/) using `pip3 install`. The tool can be uninstalled at any time by calling the script using the -u parameter instead.
+
 # Optional: Build and deploy the containerd fork from scratch
 
 ```
@@ -123,12 +135,12 @@ sudo make deploy-confpods
 popd
 ```
 
-The `all[-confpods]` target runs the targets `package[-confpods]` and `uvm[-confpods]` in a single step (the `uvm[-confpods]` target depends on the `package[-confpods]` target). The `deploy[-confpods]` target moves the build artifacts to proper places.
+The `all[-confpods]` target runs the targets `package[-confpods]` and `uvm[-confpods]` in a single step (the `uvm[-confpods]` target depends on the `package[-confpods]` target). The `deploy[-confpods]` target moves the build artifacts to proper places (and calls into `deploy[-confpods]-package`, `deploy[-confpods]-uvm`).
 
 Notes:
   - To retrieve more detailed build output, prefix the make commands with `DEBUG=1`.
+  - To build for Mariner 3, prefix the make commands that build artifacts with `OS_VERSION=3.0`
   - For build and deployment of both Kata and Kata-CC artifacts, first run the `make all` and `make deploy` commands to build and install the Kata Containers for AKS components followed by `make clean`, and then run `make all-confpods` and `make deploy-confpods` to build and install the Confidential Containers for AKS components - or vice versa (using `make clean-confpods`).
-  - The `uvm-confpods` target includes a step that installs the latest release of the [IGVM tooling](https://github.com/microsoft/igvm-tooling/) using `pip3 install` while the `clean-confpods` target uninstalls the `msigvm` pip package. To modify this behavior, the scripting should be adapted according to individuals' needs.
 
 # Run Kata (Confidential) Containers
 

diff --git a/tools/osbuilder/node-builder/azure-linux/clean.sh b/tools/osbuilder/node-builder/azure-linux/clean.sh
@@ -16,8 +16,6 @@ repo_dir="${script_dir}/../../../../"
 common_file="common.sh"
 source "${common_file}"
 
-agent_install_dir="${script_dir}/agent-install"
-
 pushd "${repo_dir}"
 
 echo "Clean runtime build"
@@ -30,13 +28,16 @@ pushd src/agent/
 make clean
 popd
 
-rm -rf ${agent_install_dir}
+rm -rf ${AGENT_INSTALL_DIR}
 
 echo "Clean UVM build"
 pushd tools/osbuilder/
 sudo -E PATH=$PATH make DISTRO=cbl-mariner clean
 popd
 
+echo "Clean IGVM tool installation"
+
+
 if [ "${CONF_PODS}" == "yes" ]; then
 
 	echo "Clean SNP debug shim config"