From 35d4c62522a963f3f043af97cdaf6a3e4588a38f Mon Sep 17 00:00:00 2001 From: Manuel Huber Date: Thu, 2 Jan 2025 22:49:49 +0000 Subject: [PATCH] runtime: Allocate default workload vcpus - similar to the static_sandbox_default_workload_mem option, assign a default number of vcpus to the VM when no limits are given, 1 vcpu in this case - similar to commit c7b8ee9, do not allocate additional vcpus when limits are provided Signed-off-by: Manuel Huber --- src/runtime/Makefile | 6 ++- .../config/configuration-clh-snp.toml.in | 7 +++- src/runtime/config/configuration-clh.toml.in | 7 +++- .../pkg/katautils/config-settings.go.in | 2 +- src/runtime/pkg/katautils/config.go | 38 ++++++++++--------- src/runtime/pkg/oci/utils.go | 12 +++++- src/runtime/virtcontainers/hypervisor.go | 2 +- src/runtime/virtcontainers/sandbox.go | 2 + .../node-builder/azure-linux/package_build.sh | 2 +- 9 files changed, 51 insertions(+), 27 deletions(-) diff --git a/src/runtime/Makefile b/src/runtime/Makefile index 7b1b5e821caf..909ef5716c32 100644 --- a/src/runtime/Makefile +++ b/src/runtime/Makefile @@ -189,7 +189,7 @@ STRATOVIRTPATH = $(STRATOVIRTBINDIR)/$(STRATOVIRTCMD) STRATOVIRTVALIDHYPERVISORPATHS := [\"$(STRATOVIRTPATH)\"] # Default number of vCPUs -DEFVCPUS := 1 +DEFVCPUS ?= 1 # Default maximum number of vCPUs DEFMAXVCPUS := 0 # Default memory size in MiB @@ -263,8 +263,9 @@ DEFSANDBOXCGROUPONLY ?= false DEFSTATICRESOURCEMGMT ?= false DEFSTATICRESOURCEMGMT_TEE = true -# Default memory for use for workloads within the sandbox if no specific workload memory value is requested +# Default memory and vcpus for use for workloads within the sandbox if no specific workload values are requested DEFSTATICSANDBOXWORKLOADMEM ?= 2048 +DEFSTATICSANDBOXWORKLOADVCPUS ?= 1 DEFBINDMOUNTS := [] @@ -694,6 +695,7 @@ USER_VARS += DEFSTATICRESOURCEMGMT_FC USER_VARS += DEFSTATICRESOURCEMGMT_STRATOVIRT USER_VARS += DEFSTATICRESOURCEMGMT_TEE USER_VARS += DEFSTATICSANDBOXWORKLOADMEM +USER_VARS += DEFSTATICSANDBOXWORKLOADVCPUS USER_VARS += DEFBINDMOUNTS USER_VARS += DEFSERVICEOFFLOAD USER_VARS += DEFVFIOMODE diff --git a/src/runtime/config/configuration-clh-snp.toml.in b/src/runtime/config/configuration-clh-snp.toml.in index d98fdd31ef27..994605bb8046 100644 --- a/src/runtime/config/configuration-clh-snp.toml.in +++ b/src/runtime/config/configuration-clh-snp.toml.in @@ -100,7 +100,7 @@ kernel_params = "@KERNELPARAMS@" # < 0 --> will be set to the actual number of physical cores # > 0 <= number of physical cores --> will be set to the specified number # > number of physical cores --> will be set to the actual number of physical cores -default_vcpus = 1 +default_vcpus = @DEFVCPUS@ # Default maximum number of vCPUs per SB/VM: # unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number @@ -398,6 +398,11 @@ static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_TEE@ # default amount of memory available within the sandbox. static_sandbox_default_workload_mem=@DEFSTATICSANDBOXWORKLOADMEM@ +# If set, the runtime will use the value as the default number of vcpus for the sandbox when no workload vcpu request is passed +# down to the shim via the OCI when static sandbox resource management is enabled. With this, we ensure that workloads have a proper +# default amount of vcpus available within the sandbox. +static_sandbox_default_workload_vcpus=@DEFSTATICSANDBOXWORKLOADVCPUS@ + # If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path. # This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory. # If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts` diff --git a/src/runtime/config/configuration-clh.toml.in b/src/runtime/config/configuration-clh.toml.in index 290ed53cee65..4e587d962dd1 100644 --- a/src/runtime/config/configuration-clh.toml.in +++ b/src/runtime/config/configuration-clh.toml.in @@ -95,7 +95,7 @@ kernel_params = "@KERNELPARAMS@" # < 0 --> will be set to the actual number of physical cores # > 0 <= number of physical cores --> will be set to the specified number # > number of physical cores --> will be set to the actual number of physical cores -default_vcpus = 1 +default_vcpus = @DEFVCPUS@ # Default maximum number of vCPUs per SB/VM: # unspecified or == 0 --> will be set to the actual number of physical cores or to the maximum number @@ -418,6 +418,11 @@ static_sandbox_resource_mgmt=@DEFSTATICRESOURCEMGMT_CLH@ # default amount of memory available within the sandbox. static_sandbox_default_workload_mem=@DEFSTATICSANDBOXWORKLOADMEM@ +# If set, the runtime will use the value as the default number of vcpus for the sandbox when no workload vcpu request is passed +# down to the shim via the OCI when static sandbox resource management is enabled. With this, we ensure that workloads have a proper +# default amount of vcpus available within the sandbox. +static_sandbox_default_workload_vcpus=@DEFSTATICSANDBOXWORKLOADVCPUS@ + # If specified, sandbox_bind_mounts identifieds host paths to be mounted (ro) into the sandboxes shared path. # This is only valid if filesystem sharing is utilized. The provided path(s) will be bindmounted into the shared fs directory. # If defaults are utilized, these mounts should be available in the guest at `/run/kata-containers/shared/containers/sandbox-mounts` diff --git a/src/runtime/pkg/katautils/config-settings.go.in b/src/runtime/pkg/katautils/config-settings.go.in index a6d2ece25ad7..34ecc75417bc 100644 --- a/src/runtime/pkg/katautils/config-settings.go.in +++ b/src/runtime/pkg/katautils/config-settings.go.in @@ -59,7 +59,7 @@ var systemdUnitName = "kata-containers.target" const defaultKernelParams = "" const defaultMachineType = "q35" -const defaultVCPUCount uint32 = 1 +const defaultVCPUCount uint32 = 0 const defaultMaxVCPUCount uint32 = 0 const defaultMemSize uint32 = 2048 // MiB const defaultMemSlots uint32 = 10 diff --git a/src/runtime/pkg/katautils/config.go b/src/runtime/pkg/katautils/config.go index 6e1251543521..18935d05405d 100644 --- a/src/runtime/pkg/katautils/config.go +++ b/src/runtime/pkg/katautils/config.go @@ -170,24 +170,25 @@ type hypervisor struct { } type runtime struct { - InterNetworkModel string `toml:"internetworking_model"` - JaegerEndpoint string `toml:"jaeger_endpoint"` - JaegerUser string `toml:"jaeger_user"` - JaegerPassword string `toml:"jaeger_password"` - VfioMode string `toml:"vfio_mode"` - GuestSeLinuxLabel string `toml:"guest_selinux_label"` - SandboxBindMounts []string `toml:"sandbox_bind_mounts"` - Experimental []string `toml:"experimental"` - Tracing bool `toml:"enable_tracing"` - DisableNewNetNs bool `toml:"disable_new_netns"` - DisableGuestSeccomp bool `toml:"disable_guest_seccomp"` - EnableVCPUsPinning bool `toml:"enable_vcpus_pinning"` - Debug bool `toml:"enable_debug"` - SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` - StaticSandboxResourceMgmt bool `toml:"static_sandbox_resource_mgmt"` - EnablePprof bool `toml:"enable_pprof"` - DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"` - StaticSandboxWorkloadDefaultMem uint32 `toml:"static_sandbox_default_workload_mem"` + InterNetworkModel string `toml:"internetworking_model"` + JaegerEndpoint string `toml:"jaeger_endpoint"` + JaegerUser string `toml:"jaeger_user"` + JaegerPassword string `toml:"jaeger_password"` + VfioMode string `toml:"vfio_mode"` + GuestSeLinuxLabel string `toml:"guest_selinux_label"` + SandboxBindMounts []string `toml:"sandbox_bind_mounts"` + Experimental []string `toml:"experimental"` + Tracing bool `toml:"enable_tracing"` + DisableNewNetNs bool `toml:"disable_new_netns"` + DisableGuestSeccomp bool `toml:"disable_guest_seccomp"` + EnableVCPUsPinning bool `toml:"enable_vcpus_pinning"` + Debug bool `toml:"enable_debug"` + SandboxCgroupOnly bool `toml:"sandbox_cgroup_only"` + StaticSandboxResourceMgmt bool `toml:"static_sandbox_resource_mgmt"` + EnablePprof bool `toml:"enable_pprof"` + DisableGuestEmptyDir bool `toml:"disable_guest_empty_dir"` + StaticSandboxWorkloadDefaultMem uint32 `toml:"static_sandbox_default_workload_mem"` + StaticSandboxWorkloadDefaultVcpus float32 `toml:"static_sandbox_default_workload_vcpus"` } type agent struct { @@ -1588,6 +1589,7 @@ func LoadConfiguration(configPath string, ignoreLogging bool) (resolvedConfigPat config.GuestSeLinuxLabel = tomlConf.Runtime.GuestSeLinuxLabel config.StaticSandboxResourceMgmt = tomlConf.Runtime.StaticSandboxResourceMgmt config.StaticSandboxWorkloadDefaultMem = tomlConf.Runtime.StaticSandboxWorkloadDefaultMem + config.StaticSandboxWorkloadDefaultVcpus = tomlConf.Runtime.StaticSandboxWorkloadDefaultVcpus config.SandboxCgroupOnly = tomlConf.Runtime.SandboxCgroupOnly config.DisableNewNetNs = tomlConf.Runtime.DisableNewNetNs config.EnablePprof = tomlConf.Runtime.EnablePprof diff --git a/src/runtime/pkg/oci/utils.go b/src/runtime/pkg/oci/utils.go index d00bbea813de..4b5106113db5 100644 --- a/src/runtime/pkg/oci/utils.go +++ b/src/runtime/pkg/oci/utils.go @@ -148,6 +148,9 @@ type RuntimeConfig struct { // Memory to allocate for workloads within the sandbox when workload memory is unspecified StaticSandboxWorkloadDefaultMem uint32 + // vcpus to allocate for workloads within the sandbox when workload vcpus is unspecified + StaticSandboxWorkloadDefaultVcpus float32 + // Determines if create a netns for hypervisor process DisableNewNetNs bool @@ -996,6 +999,8 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st StaticWorkloadDefaultMem: runtime.StaticSandboxWorkloadDefaultMem, + StaticWorkloadDefaultVcpus: runtime.StaticSandboxWorkloadDefaultVcpus, + ShmSize: shmSize, VfioMode: runtime.VfioMode, @@ -1022,11 +1027,14 @@ func SandboxConfig(ocispec specs.Spec, runtime RuntimeConfig, bundlePath, cid st // with the base number of CPU/memory (which is equal to the default CPU/memory specified for the runtime // configuration or annotations) as well as any specified workload resources. if sandboxConfig.StaticResourceMgmt { - // If no Limits are set in pod config, use StaticWorkloadDefaultMem to ensure the containers generally - // have a reasonable amount of memory available + // If no Limits are set in pod config, use StaticWorkloadDefaultMem/Vcpus to ensure the containers generally + // have a reasonable amount of resources available if sandboxConfig.SandboxResources.WorkloadMemMB == 0 { sandboxConfig.SandboxResources.WorkloadMemMB = sandboxConfig.StaticWorkloadDefaultMem } + if sandboxConfig.SandboxResources.WorkloadCPUs == 0 { + sandboxConfig.SandboxResources.WorkloadCPUs = sandboxConfig.StaticWorkloadDefaultVcpus + } sandboxConfig.SandboxResources.BaseCPUs = sandboxConfig.HypervisorConfig.NumVCPUsF sandboxConfig.SandboxResources.BaseMemMB = sandboxConfig.HypervisorConfig.MemorySize diff --git a/src/runtime/virtcontainers/hypervisor.go b/src/runtime/virtcontainers/hypervisor.go index c7b1ff505dac..4eec4b0f83aa 100644 --- a/src/runtime/virtcontainers/hypervisor.go +++ b/src/runtime/virtcontainers/hypervisor.go @@ -65,7 +65,7 @@ const ( procCPUInfo = "/proc/cpuinfo" - defaultVCPUs = float32(1) + defaultVCPUs = float32(0) // 2 GiB defaultMemSzMiB = 2048 diff --git a/src/runtime/virtcontainers/sandbox.go b/src/runtime/virtcontainers/sandbox.go index a210371c0127..f76daeb8f6a3 100644 --- a/src/runtime/virtcontainers/sandbox.go +++ b/src/runtime/virtcontainers/sandbox.go @@ -165,6 +165,8 @@ type SandboxConfig struct { StaticWorkloadDefaultMem uint32 + StaticWorkloadDefaultVcpus float32 + // Memory to allocate for workloads within the sandbox when workload memory is unspecified ShmSize uint64 diff --git a/tools/osbuilder/node-builder/azure-linux/package_build.sh b/tools/osbuilder/node-builder/azure-linux/package_build.sh index 42ffb61b6ec8..93cedb855470 100755 --- a/tools/osbuilder/node-builder/azure-linux/package_build.sh +++ b/tools/osbuilder/node-builder/azure-linux/package_build.sh @@ -21,7 +21,7 @@ source "${common_file}" # these options ensure we produce the proper CLH config file runtime_make_flags="SKIP_GO_VERSION_CHECK=1 QEMUCMD= FCCMD= ACRNCMD= STRATOVIRTCMD= DEFAULT_HYPERVISOR=cloud-hypervisor - DEFMEMSZ=0 DEFSTATICSANDBOXWORKLOADMEM=512 DEFVIRTIOFSDAEMON=${VIRTIOFSD_BINARY_LOCATION} PREFIX=${INSTALL_PATH_PREFIX}" + DEFMEMSZ=0 DEFSTATICSANDBOXWORKLOADMEM=512 DEFVCPUS=0 DEFSTATICSANDBOXWORKLOADVCPUS=1 DEFVIRTIOFSDAEMON=${VIRTIOFSD_BINARY_LOCATION} PREFIX=${INSTALL_PATH_PREFIX}" # - for vanilla Kata we use the kernel binary. For ConfPods we use IGVM, so no need to provide kernel path. # - for vanilla Kata we explicitly set DEFSTATICRESOURCEMGMT_CLH. For ConfPods,