From eef2029fe7373f4e29101e8d9b50001fd88a28b7 Mon Sep 17 00:00:00 2001 From: Manuel Huber Date: Tue, 19 Nov 2024 16:38:30 -0800 Subject: [PATCH] node-builder: Deploy-only for AzL3 VMs - split deployment sections in node-builder README.md - install jq, curl dependencies within IGVM script - add path parameter to UVM install script Signed-off-by: Manuel Huber --- .../node-builder/azure-linux/README.md | 168 +++++++++++------- .../node-builder/azure-linux/common.sh | 4 +- .../azure-linux/package_tools_install.sh | 1 + .../node-builder/azure-linux/uvm_install.sh | 2 + 4 files changed, 107 insertions(+), 68 deletions(-) diff --git a/tools/osbuilder/node-builder/azure-linux/README.md b/tools/osbuilder/node-builder/azure-linux/README.md index 8e9611b83ba7..ff7c7955a9b7 100644 --- a/tools/osbuilder/node-builder/azure-linux/README.md +++ b/tools/osbuilder/node-builder/azure-linux/README.md @@ -1,58 +1,42 @@ # Overview -This guide serves as a reference on how to build and evaluate the underlying software stack for *Kata VM Isolated Containers on AKS* and for *Confidential Containers on AKS* using Azure Linux. -The underlying software stack referred to in this guide will stretch from containerd to lower layers, for instance, enabling to deploy Kata (Confidential) Containers via the OCI interface, or deploying a local kubelet, or leveraging AKS' Kubernetes solution. +This guide serves as a reference on how to build and install the underlying software stack for *Pod Sandboxing with AKS* and for *Confidential Containers on AKS* using Azure Linux. +This enables running Kata (Confidential) Containers via the OCI interface, or via a local kubelet, or leveraging AKS' Kubernetes solution. -In the following, the terms *Kata* and *Kata-CC* refer to *Kata VM Isolated Containers on AKS* and *Confidential Containers on AKS*, respectively. Note that, *Kata VM Isolated Containers on AKS* is also referred to as *Pod Sandboxing with AKS* in the public. +In the following, the terms *Kata* and *Kata-CC* refer to *Pod Sandboxing with AKS* and *Confidential Containers on AKS*, respectively. +The term *building* refers to build the components from source, whereas the term *installing* refers to utilizing components released by the Azure Linux team for straightforward evaluation. -# Pre-requirements +The guide provides the steps for two different environments: +- Azure Linux 3 based systems, such as Azure VMs + - Variant I: Utilize released components + - Variant II: Build components from source +- AKS nodes (based on Azure Linux 2 as of today) -While build can happen in any Azure Linux based environment, the stack can only be evaluated in Azure Linux environments on top of AMD SEV-SNP - the details here are omitted: +# Steps for Azure Linux 3 based environments + +## Set up environment + +While build can happen in any Azure Linux 3 based environment, the stack can only be evaluated on environments with proper virtualization support and, for Kata-CC, on top of AMD SEV-SNP. An example of such environment are Azure Linux 3 based Azure VMs using a proper SKU: - Deploy an Azure Linux 3 VM via `az vm create` using a [CC vm size SKU](https://learn.microsoft.com/en-us/azure/virtual-machines/dcasccv5-dcadsccv5-series) - Example: `az vm create --resource-group --name --os-disk-size-gb --public-ip-sku Standard --size --admin-username azureuser --ssh-key-values --image ` -- Deploy a [Confidential Containers for AKS cluster](https://learn.microsoft.com/en-us/azure/aks/deploy-confidential-containers-default-policy) via `az aks create` (using `AzureLinux` as `os-sku`). Note, this way the bits built in this guide will already be present on the cluster's Azure Linux based nodes. The current version is Azure Linux 2. - - Deploy a debugging pod onto one of the nodes, SSH onto the node. -- Not validated for evaluation: Install [Azure Linux 3](https://github.com/microsoft/azurelinux) on a bare metal machine supporting AMD SEV-SNP. +- SSH onto the VM -To merely build the stack, we refer to the official [Azure Linux GitHub page](https://github.com/microsoft/azurelinux) to set up Azure Linux. +Not validated for evaluation: Install [Azure Linux 3](https://github.com/microsoft/azurelinux) on a bare metal machine supporting AMD SEV-SNP. -The following steps assume the user has direct console access on the environment that was set up. +To merely build the stack, we refer to the official [Azure Linux GitHub page](https://github.com/microsoft/azurelinux) to set up an Azure Linux 3 environment. -# Deploy required virtualization packages (e.g., VMM, SEV-SNP capable kernel and Microsoft Hypervisor) +## Deploy required host packages (incl. VMM, SEV-SNP capable kernel and Microsoft Hypervisor) and extend containerd configuration -Note: This step can be skipped if your environment was set up through `az aks create` - -Install relevant packages: +Install relevant packages, append a configuration snippet to `/etc/containerd/config.toml` to register the Kata(-CC) handlers, then reboot the system: ``` sudo dnf -y makecache sudo dnf -y install kata-packages-host -``` - -Azure Linux 2 only: modify the grub configuration to boot into the SEV-SNP capable kernel `kernel-mshv` upon next reboot: -``` -boot_uuid=$(sudo grep -o -m 1 '[0-9a-f]\{8\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{4\}-[0-9a-f]\{12\}' /boot/efi/boot/grub2/grub.cfg) - -sudo sed -i -e 's@load_env -f \$bootprefix\/mariner.cfg@load_env -f \$bootprefix\/mariner-mshv.cfg\nload_env -f $bootprefix\/mariner.cfg\n@' /boot/grub2/grub.cfg - -sudo sed -i -e 's@menuentry "CBL-Mariner"@menuentry "Dom0" {\n search --no-floppy --set=root --file /HvLoader.efi\n chainloader /HvLoader.efi lxhvloader.dll MSHV_ROOT=\\\\Windows MSHV_ENABLE=TRUE MSHV_SCHEDULER_TYPE=ROOT MSHV_X2APIC_POLICY=ENABLE MSHV_SEV_SNP=TRUE MSHV_LOAD_OPTION=INCLUDETRACEMETADATA=1\n boot\n search --no-floppy --fs-uuid '"$boot_uuid"' --set=root\n linux $bootprefix/$mariner_linux_mshv $mariner_cmdline_mshv $systemd_cmdline root=$rootdevice\n if [ -f $bootprefix/$mariner_initrd_mshv ]; then\n initrd $bootprefix/$mariner_initrd_mshv\n fi\n}\n\nmenuentry "CBL-Mariner"@' /boot/grub2/grub.cfg -``` - -Reboot the system: -```sudo reboot``` - -Note: We currently use a [forked version](https://github.com/microsoft/confidential-containers-containerd/tree/tardev-v1.7.7) of `containerd` called `containerd-cc` which is installed as part of the `kata-packages-host` package. This containerd version is based on stock containerd with patches to support the Confidential Containers on AKS use case and conflicts with the `containerd` package. - -# Add Kata(-CC) handler configuration snippets to containerd configuration -Note: This step can be skipped if your environment was set up through `az aks create`. - -Append the following containerd configuration snippet to `/etc/containerd/config.toml` to register the Kata(-CC) handlers, for example, using this command: - -``` sudo tee -a /etc/containerd/config.toml 2&>1 <1 <` - For build and deployment of both Kata and Kata-CC artifacts, first run the `make all` and `make deploy` commands to build and install the Kata Containers for AKS components followed by `make clean`, and then run `make all-confpods` and `make deploy-confpods` to build and install the Confidential Containers for AKS components - or vice versa (using `make clean-confpods`). -## Debug build +## Debug builds This section describes how to build and deploy in debug mode. @@ -192,6 +220,21 @@ command: sudo make BUILD_TYPE=debug SHIM_REDEPLOY_CONFIG=no all-confpods deploy-confpods ``` +## Optional build step: Build and deploy the containerd fork from scratch + +``` +git clone --depth 1 --branch tardev-v1.7.7 https://github.com/microsoft/confidential-containers-containerd.git +pushd confidential-containers-containerd/ +GODEBUG=1 make +popd +``` + +Overwrite existing containerd binary, restart service: +``` +sudo cp -a --backup=numbered confidential-containers-containerd/bin/containerd /usr/bin/containerd +sudo systemctl restart containerd +``` + # Run Kata (Confidential) Containers ## Run via CRI or via containerd API @@ -201,19 +244,13 @@ Use e.g. `crictl` (or `ctr`) to schedule Kata (Confidential) containers, referen Note: On Kubernetes nodes, pods created via `crictl` will be deleted by the control plane. The following instructions serve as a general reference: -- Install `crictl`, set runtime endpoint in `crictl` configuration: +- Install `crictl`, `cni` binaries, and set runtime endpoint in `crictl` configuration: ``` - sudo dnf -y install cri-tools + sudo dnf -y install cri-tools cni sudo crictl config --set runtime-endpoint=unix:///run/containerd/containerd.sock ``` -- Install CNI binaries: - - ``` - sudo dnf -y install cni - ``` - - Set a proper CNI configuration and create a sample pod manifest: This step is omitted as it depends on the individual needs. - Run pods with `crictl`, for example: @@ -231,9 +268,8 @@ For further usage we refer to the upstream `crictl` (or `ctr`) and CNI documenta ## Run via Kubernetes If your environment was set up through `az aks create` the respective node is ready to run Kata (Confidential) Containers as AKS Kubernetes pods. -Other types of Kubernetes clusters should work as well - but this document doesn't cover how to set-up those clusters. - -Next, apply the kata and kata-cc runtime classes on the machine that holds your kubeconfig file, for example: +Other types of Kubernetes clusters should work as well. While this document doesn't cover how to set-up those clusters, you can +apply the kata and kata-cc runtime classes to your cluster from the machine that holds your kubeconfig file, for example: ``` cat << EOF > runtimeClass-kata-cc.yaml kind: RuntimeClass diff --git a/tools/osbuilder/node-builder/azure-linux/common.sh b/tools/osbuilder/node-builder/azure-linux/common.sh index 30c32e102cbc..e76376971c9a 100755 --- a/tools/osbuilder/node-builder/azure-linux/common.sh +++ b/tools/osbuilder/node-builder/azure-linux/common.sh @@ -16,7 +16,7 @@ if [ "${CONF_PODS}" == "yes" ]; then INSTALL_PATH_PREFIX="/opt/confidential-containers" UVM_TOOLS_PATH_OSB="${INSTALL_PATH_PREFIX}/uvm/tools/osbuilder" UVM_TOOLS_PATH_SRC="${INSTALL_PATH_PREFIX}/uvm/src" - UVM_PATH="${INSTALL_PATH_PREFIX}/share/kata-containers" + UVM_PATH_DEFAULT="${INSTALL_PATH_PREFIX}/share/kata-containers" IMG_FILE_NAME="kata-containers.img" IGVM_FILE_NAME="kata-containers-igvm.img" IGVM_DBG_FILE_NAME="kata-containers-igvm-debug.img" @@ -34,7 +34,7 @@ else INSTALL_PATH_PREFIX="/usr" UVM_TOOLS_PATH_OSB="/opt/kata-containers/uvm/tools/osbuilder" UVM_TOOLS_PATH_SRC="/opt/kata-containers/uvm/src" - UVM_PATH="${INSTALL_PATH_PREFIX}/share/kata-containers" + UVM_PATH_DEFAULT="${INSTALL_PATH_PREFIX}/share/kata-containers" IMG_FILE_NAME="kata-containers.img" SHIM_CONFIG_PATH="${INSTALL_PATH_PREFIX}/share/defaults/kata-containers" SHIM_CONFIG_FILE_NAME="configuration-clh.toml" diff --git a/tools/osbuilder/node-builder/azure-linux/package_tools_install.sh b/tools/osbuilder/node-builder/azure-linux/package_tools_install.sh index a52981e19e70..8bf306bce1ac 100755 --- a/tools/osbuilder/node-builder/azure-linux/package_tools_install.sh +++ b/tools/osbuilder/node-builder/azure-linux/package_tools_install.sh @@ -45,6 +45,7 @@ cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/Makefile "${PRE cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/clean.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/" cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/common.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/" cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/uvm_build.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/" +cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/uvm_install.sh "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/" echo "Installing agent binary and service files" cp -a --backup=numbered tools/osbuilder/node-builder/azure-linux/agent-install/usr/bin/kata-agent "${PREFIX}/${UVM_TOOLS_PATH_OSB}/node-builder/azure-linux/agent-install/usr/bin/" diff --git a/tools/osbuilder/node-builder/azure-linux/uvm_install.sh b/tools/osbuilder/node-builder/azure-linux/uvm_install.sh index bf385ea6bdb9..09e2cfa386eb 100755 --- a/tools/osbuilder/node-builder/azure-linux/uvm_install.sh +++ b/tools/osbuilder/node-builder/azure-linux/uvm_install.sh @@ -18,6 +18,8 @@ repo_dir="${script_dir}/../../../../" common_file="common.sh" source "${common_file}" +UVM_PATH=${UVM_PATH:-${UVM_PATH_DEFAULT}} + pushd "${repo_dir}" pushd tools/osbuilder