From 2f672a1907bc8607db4f679fc62d1bfcbefd53b6 Mon Sep 17 00:00:00 2001 From: Manuel Huber Date: Tue, 4 Jun 2024 13:16:51 -0700 Subject: [PATCH] tools: Improve igvm-builder and node-builder/azure-linux scripting - Support for Mariner 3 builds using OS_VERSION variable - Improvements to IGVM build process and flow as described in README - Adoption of using only cloud-hypervisor-cvm on CBL-Mariner Signed-off-by: Manuel Huber --- src/runtime/Makefile | 6 -- src/runtime/arch/amd64-options.mk | 1 - src/runtime/arch/arm64-options.mk | 1 - .../config/configuration-clh-snp.toml.in | 6 +- tools/osbuilder/.gitignore | 2 + tools/osbuilder/Makefile | 2 - .../igvm-builder/azure-linux/config.sh | 21 +++--- .../igvm-builder/azure-linux/igvm_lib.sh | 57 +++++++++++++-- tools/osbuilder/igvm-builder/igvm_builder.sh | 71 +++++++------------ .../node-builder/azure-linux/Makefile | 16 ++++- .../node-builder/azure-linux/README.md | 20 ++++-- .../node-builder/azure-linux/clean.sh | 7 +- .../node-builder/azure-linux/common.sh | 34 ++++++--- .../node-builder/azure-linux/package_build.sh | 27 +++++-- .../azure-linux/package_install.sh | 35 +++++---- .../node-builder/azure-linux/uvm_build.sh | 21 ++---- .../node-builder/azure-linux/uvm_install.sh | 2 +- 17 files changed, 203 insertions(+), 126 deletions(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 2077067e229b..7b1b5e821caf 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -175,9 +175,6 @@ QEMUVIRTIOFSPATH := $(QEMUBINDIR)/$(QEMUVIRTIOFSCMD) CLHPATH := $(CLHBINDIR)/$(CLHCMD) CLHVALIDHYPERVISORPATHS := [\"$(CLHPATH)\"] -CLHSNPPATH := $(CLHBINDIR)/$(CLHSNPCMD) -CLHSNPVALIDHYPERVISORPATHS := [\"$(CLHSNPPATH)\"] - FCPATH = $(FCBINDIR)/$(FCCMD) FCVALIDHYPERVISORPATHS := [\"$(FCPATH)\"] FCJAILERPATH = $(FCBINDIR)/$(FCJAILERCMD) @@ -566,9 +563,6 @@ USER_VARS += ACRNCTLPATH USER_VARS += ACRNVALIDCTLPATHS USER_VARS += CLHPATH USER_VARS += CLHVALIDHYPERVISORPATHS -USER_VARS += CLHSNPCMD -USER_VARS += CLHSNPPATH -USER_VARS += CLHSNPVALIDHYPERVISORPATHS USER_VARS += FIRMWAREPATH_CLH USER_VARS += FCCMD USER_VARS += FCPATH diff --git a/src/runtime/arch/amd64-options.mk b/src/runtime/arch/amd64-options.mk index 0e3ff83b54c7..940e87a41a17 100644 --- a/src/runtime/arch/amd64-options.mk +++ b/src/runtime/arch/amd64-options.mk @@ -26,7 +26,6 @@ ACRNCTLCMD := acrnctl # cloud-hypervisor binary name CLHCMD := cloud-hypervisor -CLHSNPCMD := cloud-hypervisor-snp DEFSTATICRESOURCEMGMT_CLH := false diff --git a/src/runtime/arch/arm64-options.mk b/src/runtime/arch/arm64-options.mk index bfac6071079d..895c93f82e9a 100644 --- a/src/runtime/arch/arm64-options.mk +++ b/src/runtime/arch/arm64-options.mk @@ -19,7 +19,6 @@ FCJAILERCMD := jailer # cloud-hypervisor binary name CLHCMD := cloud-hypervisor -CLHSNPCMD := cloud-hypervisor-snp DEFSTATICRESOURCEMGMT_CLH := true diff --git a/src/runtime/config/configuration-clh-snp.toml.in b/src/runtime/config/configuration-clh-snp.toml.in index 33aa41db0c8d..d98fdd31ef27 100644 --- a/src/runtime/config/configuration-clh-snp.toml.in +++ b/src/runtime/config/configuration-clh-snp.toml.in @@ -11,7 +11,7 @@ # XXX: Type: @PROJECT_TYPE@ [hypervisor.clh] -path = "@CLHSNPPATH@" +path = "@CLHPATH@" igvm = "@IGVMPATH@" image = "@IMAGEPATH@" @@ -80,8 +80,8 @@ enable_annotations = @DEFENABLEANNOTATIONS@ # List of valid annotations values for the hypervisor # Each member of the list is a path pattern as described by glob(3). # The default if not set is empty (all annotations rejected.) -# Your distribution recommends: @CLHSNPVALIDHYPERVISORPATHS@ -valid_hypervisor_paths = @CLHSNPVALIDHYPERVISORPATHS@ +# Your distribution recommends: @CLHVALIDHYPERVISORPATHS@ +valid_hypervisor_paths = @CLHVALIDHYPERVISORPATHS@ # Optional space-separated list of options to pass to the guest kernel. # For example, use `kernel_params = "vsyscall=emulate"` if you are having diff --git a/tools/osbuilder/.gitignore b/tools/osbuilder/.gitignore index becda8442895..365d32272eb4 100644 --- a/tools/osbuilder/.gitignore +++ b/tools/osbuilder/.gitignore @@ -9,3 +9,5 @@ kata-containers-initrd.img kata-containers.img rootfs-builder/centos/RPM-GPG-KEY-* typescript +node-builder/azure-linux/agent-install +igvm-builder/igvm-tooling diff --git a/tools/osbuilder/Makefile b/tools/osbuilder/Makefile index f7924a7c3497..ae7baf72054a 100644 --- a/tools/osbuilder/Makefile +++ b/tools/osbuilder/Makefile @@ -9,7 +9,6 @@ ROOTFS_BUILDER := $(MK_DIR)/rootfs-builder/rootfs.sh INITRD_BUILDER := $(MK_DIR)/initrd-builder/initrd_builder.sh IMAGE_BUILDER := $(MK_DIR)/image-builder/image_builder.sh IGVM_BUILDER := $(MK_DIR)/igvm-builder/igvm_builder.sh -IGVM_TOOL_SRC := $(MK_DIR)/igvm-tooling DISTRO ?= ubuntu BUILD_METHOD := distro @@ -222,7 +221,6 @@ install-scripts: clean: rm -rf $(TARGET_ROOTFS_MARKER) $(TARGET_ROOTFS) $(TARGET_IMAGE) $(TARGET_INITRD) $(DRACUT_OVERLAY_DIR) $(TARGET_IGVM) $(TARGET_IGVM_DEBUG) $(TARGET_IGVM_MSMT) $(TARGET_IGVM_DEBUG_MSMT) $(TARGET_IGVM_LOG) rm -rf $(IGVM_TOOL_SRC) - pip3 uninstall -y msigvm # Prints the name of the variable passed as suffix to the print- target, # E.g., if Makefile contains: diff --git a/tools/osbuilder/igvm-builder/azure-linux/config.sh b/tools/osbuilder/igvm-builder/azure-linux/config.sh index bc6986f75adb..ade604dd6046 100644 --- a/tools/osbuilder/igvm-builder/azure-linux/config.sh +++ b/tools/osbuilder/igvm-builder/azure-linux/config.sh @@ -5,16 +5,21 @@ # SPDX-License-Identifier: Apache-2.0 # this is where the kernel-uvm package installation places bzImage, see SPEC file -bzimage_bin="/usr/share/cloud-hypervisor/bzImage" +BZIMAGE_BIN="/usr/share/cloud-hypervisor/bzImage" -igvm_extract_folder="igvm-tooling" -clh_acpi_tables_dir="${igvm_extract_folder}/src/igvm/acpi/acpi-clh/" -igvmgen_py_file="${igvm_extract_folder}/src/igvm/igvmgen.py" +IGVM_EXTRACT_FOLDER="${SCRIPT_DIR}/igvm-tooling" +CLH_ACPI_TABLES_DIR="${IGVM_EXTRACT_FOLDER}/src/igvm/acpi/acpi-clh/" +IGVM_PY_FILE="${IGVM_EXTRACT_FOLDER}/src/igvm/igvmgen.py" -igvm_vars="-kernel ${bzimage_bin} -boot_mode x64 -vtl 0 -svme 1 -encrypted_page 1 -pvalidate_opt 1 -acpi ${clh_acpi_tables_dir}" +IGVM_BUILD_VARS="-kernel ${BZIMAGE_BIN} -boot_mode x64 -vtl 0 -svme 1 -encrypted_page 1 -pvalidate_opt 1 -acpi ${CLH_ACPI_TABLES_DIR}" -igvm_kernel_params_common="dm-mod.create=\"dm-verity,,,ro,0 ${data_sectors} verity 1 /dev/vda1 /dev/vda2 ${data_block_size} ${hash_block_size} ${data_blocks} 0 sha256 ${root_hash} ${salt}\" \ +IGVM_KERNEL_PARAMS_COMMON="dm-mod.create=\"dm-verity,,,ro,0 ${IMAGE_DATA_SECTORS} verity 1 /dev/vda1 /dev/vda2 ${IMAGE_DATA_BLOCK_SIZE} ${IMAGE_HASH_BLOCK_SIZE} ${IMAGE_DATA_BLOCKS} 0 sha256 ${IMAGE_ROOT_HASH} ${IMAGE_SALT}\" \ root=/dev/dm-0 rootflags=data=ordered,errors=remount-ro ro rootfstype=ext4 panic=1 no_timer_check noreplace-smp systemd.unit=kata-containers.target systemd.mask=systemd-networkd.service \ systemd.mask=systemd-networkd.socket agent.enable_signature_verification=false" -igvm_kernel_prod_params="${igvm_kernel_params_common} quiet" -igvm_kernel_debug_params="${igvm_kernel_params_common} console=hvc0 systemd.log_target=console agent.log=debug agent.debug_console agent.debug_console_vport=1026" +IGVM_KERNEL_PROD_PARAMS="${IGVM_KERNEL_PARAMS_COMMON} quiet" +IGVM_KERNEL_DEBUG_PARAMS="${IGVM_KERNEL_PARAMS_COMMON} console=hvc0 systemd.log_target=console agent.log=debug agent.debug_console agent.debug_console_vport=1026" + +IGVM_FILE_NAME="kata-containers-igvm.img" +IGVM_DBG_FILE_NAME="kata-containers-igvm-debug.img" +IGVM_MEASUREMENT_FILE_NAME="igvm-measurement.cose" +IGVM_DBG_MEASUREMENT_FILE_NAME="igvm-debug-measurement.cose" diff --git a/tools/osbuilder/igvm-builder/azure-linux/igvm_lib.sh b/tools/osbuilder/igvm-builder/azure-linux/igvm_lib.sh index 06e6d8aafe2f..e5b13307445f 100644 --- a/tools/osbuilder/igvm-builder/azure-linux/igvm_lib.sh +++ b/tools/osbuilder/igvm-builder/azure-linux/igvm_lib.sh @@ -4,10 +4,11 @@ # # SPDX-License-Identifier: Apache-2.0 -install_igvm() +install_igvm_tool() { - if [ -d ${igvm_extract_folder} ]; then - echo "${igvm_extract_folder} folder already exists, assuming tool is already installed" + echo "Installing IGVM tool" + if [ -d ${IGVM_EXTRACT_FOLDER} ]; then + echo "${IGVM_EXTRACT_FOLDER} folder already exists, assuming tool is already installed" return fi @@ -16,10 +17,54 @@ install_igvm() echo "Determining and downloading latest IGVM tooling release, and extracting including ACPI tables" IGVM_VER=$(curl -sL "https://api.github.com/repos/microsoft/igvm-tooling/releases/latest" | jq -r .tag_name | sed 's/^v//') curl -sL "https://github.com/microsoft/igvm-tooling/archive/refs/tags/${IGVM_VER}.tar.gz" | tar --no-same-owner -xz - mv igvm-tooling-${IGVM_VER} ${igvm_extract_folder} + mv igvm-tooling-${IGVM_VER} ${IGVM_EXTRACT_FOLDER} - echo "Installing IGVM module msigvm via pip3" - pushd ${igvm_extract_folder}/src + echo "Installing IGVM module msigvm (${IGVM_VER}) via pip3" + pushd ${IGVM_EXTRACT_FOLDER}/src pip3 install --no-deps ./ popd } + +uninstall_igvm_tool() +{ + echo "Uninstalling IGVM tool" + + rm -rf ${IGVM_EXTRACT_FOLDER} + pip3 uninstall -y msigvm +} + +build_igvm_files() +{ + echo "Reading Kata image dm_verity root hash information from root_hash file" + ROOT_HASH_FILE="${SCRIPT_DIR}/../root_hash.txt" + + if [ ! -f "${ROOT_HASH_FILE}" ]; then + echo "Could no find image root hash file '${ROOT_HASH_FILE}', aborting" + exit 1 + fi + + IMAGE_ROOT_HASH=$(sed -e 's/Root hash:\s*//g;t;d' "${ROOT_HASH_FILE}") + IMAGE_SALT=$(sed -e 's/Salt:\s*//g;t;d' "${ROOT_HASH_FILE}") + IMAGE_DATA_BLOCKS=$(sed -e 's/Data blocks:\s*//g;t;d' "${ROOT_HASH_FILE}") + IMAGE_DATA_BLOCK_SIZE=$(sed -e 's/Data block size:\s*//g;t;d' "${ROOT_HASH_FILE}") + IMAGE_DATA_SECTORS_PER_BLOCK=$((IMAGE_DATA_BLOCK_SIZE / 512)) + IMAGE_DATA_SECTORS=$((IMAGE_DATA_BLOCKS * IMAGE_DATA_SECTORS_PER_BLOCK)) + IMAGE_HASH_BLOCK_SIZE=$(sed -e 's/Hash block size:\s*//g;t;d' "${ROOT_HASH_FILE}") + + # reloading the config file as various variables depend on above values + load_config_distro + + echo "Building (debug) IGVM files and creating their reference measurement files" + # we could call into the installed binary '~/.local/bin/igvmgen' when adding to PATH or, better, into 'python3 -m msigvm' + # however, as we still need the installation directory for the ACPI tables, we leave things as is for now + # at the same time we seem to need to call pip3 install for invoking the tool at all + python3 ${IGVM_PY_FILE} $IGVM_BUILD_VARS -o $IGVM_FILE_NAME -measurement_file $IGVM_MEASUREMENT_FILE_NAME -append "$IGVM_KERNEL_PROD_PARAMS" -svn $SVN + python3 ${IGVM_PY_FILE} $IGVM_BUILD_VARS -o $IGVM_DBG_FILE_NAME -measurement_file $IGVM_DBG_MEASUREMENT_FILE_NAME -append "$IGVM_KERNEL_DEBUG_PARAMS" -svn $SVN + + if [ "${PWD}" -ef "$(readlink -f $OUT_DIR)" ]; then + echo "OUT_DIR matches with current dir, not moving build artifacts" + else + echo "Moving build artifacts to ${OUT_DIR}" + mv $IGVM_FILE_NAME $IGVM_DBG_FILE_NAME $IGVM_MEASUREMENT_FILE_NAME $IGVM_DBG_MEASUREMENT_FILE_NAME $OUT_DIR + fi +} diff --git a/tools/osbuilder/igvm-builder/igvm_builder.sh b/tools/osbuilder/igvm-builder/igvm_builder.sh index 58bd38b63990..8e539f69d941 100755 --- a/tools/osbuilder/igvm-builder/igvm_builder.sh +++ b/tools/osbuilder/igvm-builder/igvm_builder.sh @@ -10,18 +10,18 @@ set -o errtrace [ -n "$DEBUG" ] && set -x -script_dir="$(dirname $(readlink -f $0))" +SCRIPT_DIR="$(dirname $(readlink -f $0))" # distro-specific config file typeset -r CONFIG_SH="config.sh" # Name of an optional distro-specific file which, if it exists, must implement the -# install_igvm() function. +# install_igvm_tool, build_igvm_files, and uninstall_igvm_tool functions. typeset -r LIB_SH="igvm_lib.sh" -build_igvm_distro() +load_config_distro() { - distro_config_dir="${script_dir}/${distro}" + distro_config_dir="${SCRIPT_DIR}/${DISTRO}" [ -d "${distro_config_dir}" ] || die "Could not find configuration directory '${distro_config_dir}'" @@ -31,50 +31,20 @@ build_igvm_distro() source "${igvm_lib}" fi - root_hash_file="${script_dir}/../root_hash.txt" - - if [ ! -f "${root_hash_file}" ]; then - echo "Could no find image root hash file '${root_hash_file}', aborting" - exit 1 - fi - - echo "Reading image dm-verity root hash values" - root_hash=$(sed -e 's/Root hash:\s*//g;t;d' "${root_hash_file}") - salt=$(sed -e 's/Salt:\s*//g;t;d' "${root_hash_file}") - data_blocks=$(sed -e 's/Data blocks:\s*//g;t;d' "${root_hash_file}") - data_block_size=$(sed -e 's/Data block size:\s*//g;t;d' "${root_hash_file}") - data_sectors_per_block=$((data_block_size / 512)) - data_sectors=$((data_blocks * data_sectors_per_block)) - hash_block_size=$(sed -e 's/Hash block size:\s*//g;t;d' "${root_hash_file}") - # Source config.sh from distro, depends on root_hash based variables here igvm_config="${distro_config_dir}/${CONFIG_SH}" source "${igvm_config}" - - echo "Install IGVM tool" - install_igvm - - echo "Build IGVM (debug) file and calculate reference measurements" - # we could call into the installed binary '~/.local/bin/igvmgen' when adding to PATH or, better, into 'python3 -m msigvm' - # however, as we still need the installation directory for the ACPI tables, we leave things as is for now - # at the same time we seem to need to call pip3 install for invoking the tool at all - python3 ${igvmgen_py_file} $igvm_vars -o kata-containers-igvm.img -measurement_file igvm-measurement.cose -append "$igvm_kernel_prod_params" -svn $SVN - python3 ${igvmgen_py_file} $igvm_vars -o kata-containers-igvm-debug.img -measurement_file igvm-debug-measurement.cose -append "$igvm_kernel_debug_params" -svn $SVN - - if [ "${PWD}" -ef "$(readlink -f $OUT_DIR)" ]; then - echo "OUT_DIR matches with current dir, not moving build artifacts" - else - echo "Moving build artifacts to ${OUT_DIR}" - mv igvm-measurement.cose kata-containers-igvm.img igvm-debug-measurement.cose kata-containers-igvm-debug.img $OUT_DIR - fi } -distro="azure-linux" +DISTRO="azure-linux" +MODE="build" -while getopts ":o:s:" OPTIONS; do +while getopts ":o:s:iu" OPTIONS; do case "${OPTIONS}" in o ) OUT_DIR=$OPTARG ;; s ) SVN=$OPTARG ;; + i ) MODE="install" ;; + u ) MODE="uninstall" ;; \? ) echo "Error - Invalid Option: -$OPTARG" 1>&2 exit 1 @@ -89,11 +59,24 @@ done echo "IGVM builder script" echo "-- OUT_DIR -> $OUT_DIR" echo "-- SVN -> $SVN" -echo "-- distro -> $distro" +echo "-- DISTRO -> $DISTRO" +echo "-- MODE -> $MODE" -if [ -n "$distro" ]; then - build_igvm_distro +if [ -n "$DISTRO" ]; then + load_config_distro else - echo "distro must be specified" - exit 1 + echo "DISTRO must be specified" + exit 1 fi + +case "$MODE" in + "install") + install_igvm_tool + ;; + "uninstall") + uninstall_igvm_tool + ;; + "build") + build_igvm_files + ;; +esac diff --git a/tools/osbuilder/node-builder/azure-linux/Makefile b/tools/osbuilder/node-builder/azure-linux/Makefile index 26c28bd18cde..6ef571a55fa2 100644 --- a/tools/osbuilder/node-builder/azure-linux/Makefile +++ b/tools/osbuilder/node-builder/azure-linux/Makefile @@ -33,11 +33,23 @@ clean-confpods: CONF_PODS=yes ./clean.sh .PHONY: deploy -deploy: +deploy: deploy-package deploy-uvm + +.PHONY: deploy-package +deploy-package: ./package_install.sh + +.PHONY: deploy-uvm +deploy-uvm: ./uvm_install.sh .PHONY: deploy-confpods -deploy-confpods: +deploy-confpods: deploy-confpods-package deploy-confpods-uvm + +.PHONY: deploy-confpods-package +deploy-confpods-package: CONF_PODS=yes ./package_install.sh + +.PHONY: deploy-confpods-uvm +deploy-confpods-uvm: CONF_PODS=yes ./uvm_install.sh diff --git a/tools/osbuilder/node-builder/azure-linux/README.md b/tools/osbuilder/node-builder/azure-linux/README.md index 693f9a8ac9e2..16a9b84dd5a9 100644 --- a/tools/osbuilder/node-builder/azure-linux/README.md +++ b/tools/osbuilder/node-builder/azure-linux/README.md @@ -76,7 +76,7 @@ sudo tee -a /etc/containerd/config.toml 2&>1 <