From a8e0f51c52391ba60417450378e0a82c7cb8465c Mon Sep 17 00:00:00 2001 From: Helin Guo Date: Tue, 7 Feb 2023 11:46:56 +0800 Subject: [PATCH 1/3] dragonball: extend DeviceOpContext In order to support virtio-mem and virtio-balloon devices, we need to extend DeviceOpContext with VmConfigInfo and InstanceInfo. Fixes: #6719 Signed-off-by: Helin Guo --- src/dragonball/src/device_manager/mod.rs | 46 ++++++++++++++++++++---- src/dragonball/src/vm/mod.rs | 4 +-- 2 files changed, 41 insertions(+), 9 deletions(-) diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 49e9666ac4aa..324cf41fcaa7 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -7,7 +7,7 @@ use std::collections::HashMap; use std::io; -use std::sync::{Arc, Mutex, MutexGuard}; +use std::sync::{Arc, Mutex, MutexGuard, RwLock}; use arc_swap::ArcSwap; use dbs_address_space::AddressSpace; @@ -45,9 +45,10 @@ use dbs_upcall::{ use dbs_virtio_devices::vsock::backend::VsockInnerConnector; use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::api::v1::InstanceInfo; use crate::error::StartMicroVmError; use crate::resource_manager::ResourceManager; -use crate::vm::{KernelConfigInfo, Vm}; +use crate::vm::{KernelConfigInfo, Vm, VmConfigInfo}; use crate::IoManagerCached; /// Virtual machine console device manager. @@ -248,6 +249,8 @@ pub struct DeviceOpContext { upcall_client: Option>>, #[cfg(feature = "dbs-virtio-devices")] virtio_devices: Vec>, + vm_config: Option, + shared_info: Arc>, } impl DeviceOpContext { @@ -257,6 +260,8 @@ impl DeviceOpContext { vm_as: Option, address_space: Option, is_hotplug: bool, + vm_config: Option, + shared_info: Arc>, ) -> Self { let irq_manager = device_mgr.irq_manager.clone(); let res_manager = device_mgr.res_manager.clone(); @@ -282,11 +287,21 @@ impl DeviceOpContext { upcall_client: None, #[cfg(feature = "dbs-virtio-devices")] virtio_devices: Vec::new(), + vm_config, + shared_info, } } pub(crate) fn create_boot_ctx(vm: &Vm, epoll_mgr: Option) -> Self { - Self::new(epoll_mgr, vm.device_manager(), None, None, false) + Self::new( + epoll_mgr, + vm.device_manager(), + None, + None, + false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), + ) } pub(crate) fn get_vm_as(&self) -> Result { @@ -386,6 +401,8 @@ impl DeviceOpContext { Some(vm_as), vm.vm_address_space().cloned(), true, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), ); ctx.upcall_client = vm.upcall_client().clone(); ctx @@ -463,7 +480,7 @@ pub struct DeviceManager { res_manager: Arc, vm_fd: Arc, pub(crate) logger: slog::Logger, - + pub(crate) shared_info: Arc>, pub(crate) con_manager: ConsoleManager, pub(crate) legacy_manager: Option, #[cfg(target_arch = "aarch64")] @@ -490,6 +507,7 @@ impl DeviceManager { res_manager: Arc, epoll_manager: EpollManager, logger: &slog::Logger, + shared_info: Arc>, ) -> Self { DeviceManager { io_manager: Arc::new(ArcSwap::new(Arc::new(IoManager::new()))), @@ -498,6 +516,7 @@ impl DeviceManager { res_manager, vm_fd, logger: logger.new(slog::o!()), + shared_info, con_manager: ConsoleManager::new(epoll_manager, logger), legacy_manager: None, @@ -636,9 +655,9 @@ impl DeviceManager { vm_as: GuestAddressSpaceImpl, epoll_mgr: EpollManager, kernel_config: &mut KernelConfigInfo, - com1_sock_path: Option, dmesg_fifo: Option>, address_space: Option<&AddressSpace>, + vm_config: &VmConfigInfo, ) -> std::result::Result<(), StartMicroVmError> { let mut ctx = DeviceOpContext::new( Some(epoll_mgr), @@ -646,8 +665,12 @@ impl DeviceManager { Some(vm_as), address_space.cloned(), false, + Some(vm_config.clone()), + self.shared_info.clone(), ); + let com1_sock_path = vm_config.serial_path.clone(); + self.create_legacy_devices(&mut ctx)?; self.init_legacy_devices(dmesg_fifo, com1_sock_path, &mut ctx)?; @@ -710,6 +733,8 @@ impl DeviceManager { Some(vm_as), address_space.cloned(), true, + None, + self.shared_info.clone(), ); #[cfg(feature = "virtio-blk")] @@ -1030,6 +1055,10 @@ mod tests { let epoll_manager = EpollManager::default(); let res_manager = Arc::new(ResourceManager::new(None)); let logger = slog_scope::logger().new(slog::o!()); + let shared_info = Arc::new(RwLock::new(InstanceInfo::new( + String::from("dragonball"), + String::from("1"), + ))); DeviceManager { vm_fd: Arc::clone(&vm_fd), @@ -1052,6 +1081,7 @@ mod tests { mmio_device_info: HashMap::new(), logger, + shared_info, } } } @@ -1091,7 +1121,7 @@ mod tests { }, vpmu_feature: 0, }; - vm.set_vm_config(vm_config); + vm.set_vm_config(vm_config.clone()); vm.init_guest_memory().unwrap(); vm.setup_interrupt_controller().unwrap(); let vm_as = vm.vm_as().cloned().unwrap(); @@ -1117,8 +1147,8 @@ mod tests { event_mgr.epoll_manager(), &mut cmdline, None, - None, address_space.as_ref(), + &vm_config, ) .unwrap(); let guard = mgr.io_manager.load(); @@ -1142,6 +1172,8 @@ mod tests { Some(vm.vm_as().unwrap().clone()), vm.vm_address_space().cloned(), true, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), ); let guest_addr = GuestAddress(0x200000000000); diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index 852d78c2b496..6e2545c6f1cb 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -222,6 +222,7 @@ impl Vm { resource_manager.clone(), epoll_manager.clone(), &logger, + api_shared_info.clone(), ); Ok(Vm { @@ -453,7 +454,6 @@ impl Vm { ) -> std::result::Result<(), StartMicroVmError> { info!(self.logger, "VM: initializing devices ..."); - let com1_sock_path = self.vm_config.serial_path.clone(); let kernel_config = self .kernel_config .as_mut() @@ -475,9 +475,9 @@ impl Vm { vm_as.clone(), epoll_manager, kernel_config, - com1_sock_path, self.dmesg_fifo.take(), self.address_space.address_space(), + &self.vm_config, )?; info!(self.logger, "VM: start devices"); From 7ed9494973b8b7e4f393f38b00fdd7fb18082165 Mon Sep 17 00:00:00 2001 From: Helin Guo Date: Wed, 31 May 2023 10:13:09 +0800 Subject: [PATCH 2/3] dragonball: introduce virtio-mem device We introduce virtio-mem device to support memory resize. virtio-mem device could hot-plug more memory blocks to guest and could also hot-unplug them from guest. Fixes: #6719 Signed-off-by: Helin Guo --- src/dragonball/Cargo.lock | 15 +- src/dragonball/Cargo.toml | 7 +- src/dragonball/src/api/v1/vmm_action.rs | 83 ++ .../src/device_manager/blk_dev_mgr.rs | 4 + .../src/device_manager/mem_dev_mgr.rs | 733 ++++++++++++++++++ .../device_manager/memory_region_handler.rs | 1 + src/dragonball/src/device_manager/mod.rs | 52 ++ 7 files changed, 885 insertions(+), 10 deletions(-) create mode 100644 src/dragonball/src/device_manager/mem_dev_mgr.rs diff --git a/src/dragonball/Cargo.lock b/src/dragonball/Cargo.lock index f2e087213b1f..3cc177fe1cf3 100644 --- a/src/dragonball/Cargo.lock +++ b/src/dragonball/Cargo.lock @@ -209,11 +209,12 @@ dependencies = [ [[package]] name = "dbs-address-space" -version = "0.2.2" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6bcc37dc0b8ffae1c5911d13ae630dc7a9020fa0de0edd178d6ab71daf56c8fc" +checksum = "95e20d28a9cd13bf00d0ecd1bd073d242242b04f0acb663d7adfc659f8879322" dependencies = [ "arc-swap", + "lazy_static", "libc", "nix 0.23.2", "thiserror", @@ -300,9 +301,9 @@ dependencies = [ [[package]] name = "dbs-upcall" -version = "0.2.0" +version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "699e62afa444ae4b00d474fd91bc37785ba050acdfbe179731c81898e32efc3f" +checksum = "ea3a78128fd0be8b8b10257675c262b378dc5d00b1e18157736a6c27e45ce4fb" dependencies = [ "anyhow", "dbs-utils", @@ -330,9 +331,9 @@ dependencies = [ [[package]] name = "dbs-virtio-devices" -version = "0.2.0" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88e5c6c48b766afb95851b04b6b193871a59d0b2a3ed19990d4f8f651ae5c668" +checksum = "24d671cc3e5f98b84ef6b6bed007d28f72f16d3aea8eb38e2d42b00b2973c1d8" dependencies = [ "byteorder", "caps", @@ -346,7 +347,7 @@ dependencies = [ "kvm-ioctls", "libc", "log", - "nix 0.23.2", + "nix 0.24.3", "nydus-api", "nydus-blobfs", "nydus-rafs", diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index f70463266e98..0c94530ca0af 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -12,16 +12,16 @@ edition = "2018" [dependencies] arc-swap = "1.5.0" bytes = "1.1.0" -dbs-address-space = "0.2.0" +dbs-address-space = "0.3.0" dbs-allocator = "0.1.0" dbs-arch = "0.2.0" dbs-boot = "0.4.0" dbs-device = "0.2.0" dbs-interrupt = { version = "0.2.0", features = ["kvm-irq"] } dbs-legacy-devices = "0.1.0" -dbs-upcall = { version = "0.2.0", optional = true } +dbs-upcall = { version = "0.3.0", optional = true } dbs-utils = "0.2.0" -dbs-virtio-devices = { version = "0.2.0", optional = true, features = ["virtio-mmio"] } +dbs-virtio-devices = { version = "0.3.1", optional = true, features = ["virtio-mmio"] } kvm-bindings = "0.6.0" kvm-ioctls = "0.12.0" lazy_static = "1.2" @@ -55,3 +55,4 @@ virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"] virtio-net = ["dbs-virtio-devices/virtio-net", "virtio-queue"] # virtio-fs only work on atomic-guest-memory virtio-fs = ["dbs-virtio-devices/virtio-fs", "virtio-queue", "atomic-guest-memory"] +virtio-mem = ["dbs-virtio-devices/virtio-mem", "virtio-queue", "atomic-guest-memory"] diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 6ad7bfcb7231..6243b82a3340 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -27,6 +27,8 @@ pub use crate::device_manager::blk_dev_mgr::{ pub use crate::device_manager::fs_dev_mgr::{ FsDeviceConfigInfo, FsDeviceConfigUpdateInfo, FsDeviceError, FsDeviceMgr, FsMountConfigInfo, }; +#[cfg(feature = "virtio-mem")] +pub use crate::device_manager::mem_dev_mgr::{MemDeviceConfigInfo, MemDeviceError}; #[cfg(feature = "virtio-net")] pub use crate::device_manager::virtio_net_dev_mgr::{ VirtioNetDeviceConfigInfo, VirtioNetDeviceConfigUpdateInfo, VirtioNetDeviceError, @@ -97,6 +99,15 @@ pub enum VmmActionError { /// The action `ResizeVcpu` Failed #[error("vcpu resize error : {0}")] ResizeVcpu(#[source] VcpuResizeError), + + /// Cannot access address space. + #[error("Cannot access address space.")] + AddressSpaceNotInitialized, + + #[cfg(feature = "virtio-mem")] + /// Mem device related errors. + #[error("virtio-mem device error: {0}")] + Mem(#[source] MemDeviceError), } /// This enum represents the public interface of the VMM. Each action contains various @@ -172,6 +183,10 @@ pub enum VmmAction { #[cfg(feature = "hotplug")] /// Resize Vcpu number in the guest. ResizeVcpu(VcpuResizeInfo), + + #[cfg(feature = "virtio-mem")] + /// Add a new mem device or update one that already exists using the `MemDeviceConfig` as input. + InsertMemDevice(MemDeviceConfigInfo), } /// The enum represents the response sent by the VMM in case of success. The response is either @@ -274,6 +289,8 @@ impl VmmService { } #[cfg(feature = "hotplug")] VmmAction::ResizeVcpu(vcpu_resize_cfg) => self.resize_vcpu(vmm, vcpu_resize_cfg), + #[cfg(feature = "virtio-mem")] + VmmAction::InsertMemDevice(mem_cfg) => self.add_mem_device(vmm, event_mgr, mem_cfg), }; debug!("send vmm response: {:?}", response); @@ -648,6 +665,32 @@ impl VmmService { Ok(VmmData::Empty) } + + #[cfg(feature = "virtio-mem")] + fn add_mem_device( + &mut self, + vmm: &mut Vmm, + event_mgr: &mut EventManager, + config: MemDeviceConfigInfo, + ) -> VmmRequestResult { + let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; + + let ctx = vm + .create_device_op_context(Some(event_mgr.epoll_manager())) + .map_err(|e| { + if let StartMicroVmError::UpcallServerNotReady = e { + VmmActionError::UpcallServerNotReady + } else { + VmmActionError::StartMicroVm(e) + } + })?; + + vm.device_manager_mut() + .mem_manager + .insert_or_update_device(ctx, config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::Mem) + } } fn handle_cpu_topology( @@ -1456,4 +1499,44 @@ mod tests { t.check_request(); } } + + #[cfg(feature = "virtio-mem")] + #[test] + fn test_vmm_action_insert_mem_device() { + skip_if_not_root!(); + + let tests = &mut [ + // hotplug unready + TestData::new( + VmmAction::InsertMemDevice(MemDeviceConfigInfo::default()), + InstanceState::Running, + &|result| { + assert!(matches!( + result, + Err(VmmActionError::StartMicroVm( + StartMicroVmError::UpcallMissVsock + )) + )); + let err_string = format!("{}", result.unwrap_err()); + let expected_err = String::from( + "failed to boot the VM: \ + the upcall client needs a virtio-vsock device for communication", + ); + assert_eq!(err_string, expected_err); + }, + ), + // success + TestData::new( + VmmAction::InsertMemDevice(MemDeviceConfigInfo::default()), + InstanceState::Uninitialized, + &|result| { + assert!(result.is_ok()); + }, + ), + ]; + + for t in tests.iter_mut() { + t.check_request(); + } + } } diff --git a/src/dragonball/src/device_manager/blk_dev_mgr.rs b/src/dragonball/src/device_manager/blk_dev_mgr.rs index f7cdfa9b0d8c..0fe10cf246c9 100644 --- a/src/dragonball/src/device_manager/blk_dev_mgr.rs +++ b/src/dragonball/src/device_manager/blk_dev_mgr.rs @@ -871,6 +871,8 @@ mod tests { Some(vm.vm_as().unwrap().clone()), None, false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), ); let dummy_file = TempFile::new().unwrap(); @@ -907,6 +909,8 @@ mod tests { Some(vm.vm_as().unwrap().clone()), None, false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), ); vm.device_manager_mut() diff --git a/src/dragonball/src/device_manager/mem_dev_mgr.rs b/src/dragonball/src/device_manager/mem_dev_mgr.rs new file mode 100644 index 000000000000..2bb68ae80f9d --- /dev/null +++ b/src/dragonball/src/device_manager/mem_dev_mgr.rs @@ -0,0 +1,733 @@ +// Copyright 2020 Alibaba Cloud. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use std::io; +use std::sync::{Arc, Mutex}; + +use dbs_address_space::{ + AddressSpace, AddressSpaceError, AddressSpaceRegion, MPOL_MF_MOVE, MPOL_PREFERRED, USABLE_END, +}; +use dbs_utils::epoll_manager::EpollManager; +use dbs_virtio_devices as virtio; +use kvm_bindings::kvm_userspace_memory_region; +use kvm_ioctls::VmFd; +use nix::sys::mman; +use serde_derive::{Deserialize, Serialize}; +use slog::{debug, error, info, warn}; +use virtio::mem::{Mem, MemRegionFactory}; +use virtio::Error as VirtIoError; +use vm_memory::{ + Address, GuestAddress, GuestAddressSpace, GuestMemory, GuestRegionMmap, GuestUsize, MmapRegion, +}; + +use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::config_manager::{ConfigItem, DeviceConfigInfo, DeviceConfigInfos}; +use crate::device_manager::DbsMmioV2Device; +use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; +use crate::vm::VmConfigInfo; + +// The flag of whether to use the shared irq. +const USE_SHARED_IRQ: bool = true; +// The flag of whether to use the generic irq. +const USE_GENERIC_IRQ: bool = false; + +const HUGE_PAGE_2M: usize = 0x200000; + +// max numa node ids on host +const MAX_NODE: u32 = 64; + +/// Errors associated with `MemDeviceConfig`. +#[derive(Debug, thiserror::Error)] +pub enum MemDeviceError { + /// The mem device was already used. + #[error("the virtio-mem ID was already added to a different device")] + MemDeviceAlreadyExists, + + /// Cannot perform the requested operation after booting the microVM. + #[error("the update operation is not allowed after boot")] + UpdateNotAllowedPostBoot, + + /// insert mem device error + #[error("cannot add virtio-mem device, {0}")] + InsertDeviceFailed(#[source] DeviceMgrError), + + /// create mem device error + #[error("cannot create virito-mem device, {0}")] + CreateMemDevice(#[source] DeviceMgrError), + + /// create mmio device error + #[error("cannot create virito-mem mmio device, {0}")] + CreateMmioDevice(#[source] DeviceMgrError), + + /// resize mem device error + #[error("failure while resizing virtio-mem device, {0}")] + ResizeFailed(#[source] VirtIoError), + + /// mem device does not exist + #[error("mem device does not exist")] + DeviceNotExist, + + /// address space region error + #[error("address space region error, {0}")] + AddressSpaceRegion(#[source] AddressSpaceError), + + /// Cannot initialize a mem device or add a device to the MMIO Bus. + #[error("failure while registering mem device: {0}")] + RegisterMemDevice(#[source] DeviceMgrError), + + /// The mem device id doesn't exist. + #[error("invalid mem device id '{0}'")] + InvalidDeviceId(String), + + /// The device manager errors. + #[error("DeviceManager error: {0}")] + DeviceManager(#[source] DeviceMgrError), +} + +/// Configuration information for a virtio-mem device. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +pub struct MemDeviceConfigInfo { + /// Unique identifier of the pmem device + pub mem_id: String, + /// Memory size mib + pub size_mib: u64, + /// Memory capacity mib + pub capacity_mib: u64, + /// Use multi_region or not + pub multi_region: bool, + /// host numa node id + pub host_numa_node_id: Option, + /// guest numa node id + pub guest_numa_node_id: Option, + /// Use shared irq + pub use_shared_irq: Option, + /// Use generic irq + pub use_generic_irq: Option, +} + +impl ConfigItem for MemDeviceConfigInfo { + type Err = MemDeviceError; + + fn id(&self) -> &str { + &self.mem_id + } + + fn check_conflicts(&self, other: &Self) -> Result<(), MemDeviceError> { + if self.mem_id.as_str() == other.mem_id.as_str() { + Err(MemDeviceError::MemDeviceAlreadyExists) + } else { + Ok(()) + } + } +} + +/// Mem Device Info +pub type MemDeviceInfo = DeviceConfigInfo; + +impl ConfigItem for MemDeviceInfo { + type Err = MemDeviceError; + + fn id(&self) -> &str { + &self.config.mem_id + } + + fn check_conflicts(&self, other: &Self) -> Result<(), MemDeviceError> { + if self.config.mem_id.as_str() == other.config.mem_id.as_str() { + Err(MemDeviceError::MemDeviceAlreadyExists) + } else { + Ok(()) + } + } +} + +/// Wrapper for the collection that holds all the Mem Devices Configs +#[derive(Clone)] +pub struct MemDeviceMgr { + /// A list of `MemDeviceConfig` objects. + info_list: DeviceConfigInfos, + pub(crate) use_shared_irq: bool, +} + +impl MemDeviceMgr { + /// Inserts `mem_cfg` in the virtio-mem device configuration list. + /// If an entry with the same id already exists, it will attempt to update + /// the existing entry. + pub fn insert_or_update_device( + &mut self, + mut ctx: DeviceOpContext, + mem_cfg: MemDeviceConfigInfo, + ) -> std::result::Result<(), MemDeviceError> { + if !cfg!(feature = "hotplug") && ctx.is_hotplug { + error!(ctx.logger(), "hotplug feature has been disabled."; + "subsystem" => "virito-mem"); + return Err(MemDeviceError::UpdateNotAllowedPostBoot); + } + + let epoll_mgr = ctx.get_epoll_mgr().map_err(MemDeviceError::DeviceManager)?; + + // If the id of the drive already exists in the list, the operation is update. + if let Some(index) = self.get_index_of_mem_dev(&mem_cfg.mem_id) { + // Update an existing memory device + if ctx.is_hotplug { + info!( + ctx.logger(), + "update memory device: {}, size: 0x{:x}MB.", + mem_cfg.mem_id, + mem_cfg.size_mib; + "subsystem" => "virito-mem" + ); + self.update_memory_size(index, mem_cfg.size_mib)?; + } + self.info_list.insert_or_update(&mem_cfg)?; + } else { + // Create a new memory device + if !ctx.is_hotplug { + self.info_list.insert_or_update(&mem_cfg)?; + return Ok(()); + } + + info!( + ctx.logger(), + "hot-add memory device: {}, size: 0x{:x}MB.", mem_cfg.mem_id, mem_cfg.size_mib; + "subsystem" => "virito-mem" + ); + + let device = Self::create_memory_device(&mem_cfg, &ctx, &epoll_mgr) + .map_err(MemDeviceError::CreateMemDevice)?; + let mmio_device = + DeviceManager::create_mmio_virtio_device_with_device_change_notification( + Box::new(device), + &mut ctx, + mem_cfg.use_shared_irq.unwrap_or(self.use_shared_irq), + mem_cfg.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(MemDeviceError::CreateMmioDevice)?; + + #[cfg(not(test))] + ctx.insert_hotplug_mmio_device(&mmio_device, None) + .map_err(|e| { + error!( + ctx.logger(), + "failed to hot-add virtio-mem device {}, {}", &mem_cfg.mem_id, e; + "subsystem" => "virito-mem" + ); + MemDeviceError::InsertDeviceFailed(e) + })?; + + let index = self.info_list.insert_or_update(&mem_cfg)?; + self.info_list[index].set_device(mmio_device); + } + + Ok(()) + } + + /// Attaches all virtio-mem devices from the MemDevicesConfig. + pub fn attach_devices( + &mut self, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), MemDeviceError> { + let epoll_mgr = ctx.get_epoll_mgr().map_err(MemDeviceError::DeviceManager)?; + + for info in self.info_list.iter_mut() { + let config = &info.config; + info!( + ctx.logger(), + "attach virtio-mem device {}, size 0x{:x}.", config.mem_id, config.size_mib; + "subsystem" => "virito-mem" + ); + // Ignore virtio-mem device with zero memory capacity. + if config.size_mib == 0 { + debug!( + ctx.logger(), + "ignore zero-sizing memory device {}.", config.mem_id; + "subsystem" => "virito-mem" + ); + continue; + } + + let device = Self::create_memory_device(config, ctx, &epoll_mgr) + .map_err(MemDeviceError::CreateMemDevice)?; + let mmio_device = + DeviceManager::create_mmio_virtio_device_with_device_change_notification( + Box::new(device), + ctx, + config.use_shared_irq.unwrap_or(self.use_shared_irq), + config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(MemDeviceError::RegisterMemDevice)?; + + info.set_device(mmio_device); + } + + Ok(()) + } + + fn get_index_of_mem_dev(&self, mem_id: &str) -> Option { + self.info_list + .iter() + .position(|info| info.config.mem_id.eq(mem_id)) + } + + fn create_memory_device( + config: &MemDeviceConfigInfo, + ctx: &DeviceOpContext, + epoll_mgr: &EpollManager, + ) -> std::result::Result, DeviceMgrError> { + let factory = Arc::new(Mutex::new(MemoryRegionFactory::new( + ctx, + config.mem_id.clone(), + config.host_numa_node_id, + )?)); + + let mut capacity_mib = config.capacity_mib; + if capacity_mib == 0 { + capacity_mib = *USABLE_END >> 20; + } + // get boot memory size for calculate alignment + let boot_mem_size = { + let boot_size = (ctx.get_vm_config()?.mem_size_mib << 20) as u64; + // increase 1G memory because of avoiding mmio hole + match boot_size { + x if x > dbs_boot::layout::MMIO_LOW_START => x + (1 << 30), + _ => boot_size, + } + }; + + virtio::mem::Mem::new( + config.mem_id.clone(), + capacity_mib, + config.size_mib, + config.multi_region, + config.guest_numa_node_id, + epoll_mgr.clone(), + factory, + boot_mem_size, + ) + .map_err(DeviceMgrError::Virtio) + } + + /// Removes all virtio-mem devices + pub fn remove_devices(&self, ctx: &mut DeviceOpContext) -> Result<(), DeviceMgrError> { + for info in self.info_list.iter() { + if let Some(device) = &info.device { + DeviceManager::destroy_mmio_virtio_device(device.clone(), ctx)?; + } + } + + Ok(()) + } + + fn update_memory_size( + &self, + index: usize, + size_mib: u64, + ) -> std::result::Result<(), MemDeviceError> { + let device = self.info_list[index] + .device + .as_ref() + .ok_or_else(|| MemDeviceError::DeviceNotExist)?; + if let Some(mmio_dev) = device.as_any().downcast_ref::() { + let guard = mmio_dev.state(); + let inner_dev = guard.get_inner_device(); + if let Some(mem_dev) = inner_dev + .as_any() + .downcast_ref::>() + { + return mem_dev + .set_requested_size(size_mib) + .map_err(MemDeviceError::ResizeFailed); + } + } + Ok(()) + } +} + +impl Default for MemDeviceMgr { + /// Create a new `MemDeviceMgr` object.. + fn default() -> Self { + MemDeviceMgr { + info_list: DeviceConfigInfos::new(), + use_shared_irq: USE_SHARED_IRQ, + } + } +} + +struct MemoryRegionFactory { + mem_id: String, + vm_as: GuestAddressSpaceImpl, + address_space: AddressSpace, + vm_config: VmConfigInfo, + vm_fd: Arc, + logger: Arc, + host_numa_node_id: Option, + instance_id: String, +} + +impl MemoryRegionFactory { + fn new( + ctx: &DeviceOpContext, + mem_id: String, + host_numa_node_id: Option, + ) -> Result { + let vm_as = ctx.get_vm_as()?; + let address_space = ctx.get_address_space()?; + let vm_config = ctx.get_vm_config()?; + let logger = Arc::new(ctx.logger().new(slog::o!())); + + let shared_info = ctx.shared_info.read().unwrap(); + let instance_id = shared_info.id.clone(); + + Ok(MemoryRegionFactory { + mem_id, + vm_as, + address_space, + vm_config, + vm_fd: ctx.vm_fd.clone(), + logger, + host_numa_node_id, + instance_id, + }) + } + + fn configure_anon_mem(&self, mmap_reg: &MmapRegion) -> Result<(), VirtIoError> { + unsafe { + mman::madvise( + mmap_reg.as_ptr() as *mut libc::c_void, + mmap_reg.size(), + mman::MmapAdvise::MADV_DONTFORK, + ) + } + .map_err(VirtIoError::Madvise)?; + + Ok(()) + } + + fn configure_numa(&self, mmap_reg: &MmapRegion, node_id: u32) -> Result<(), VirtIoError> { + let nodemask = 1_u64 + .checked_shl(node_id) + .ok_or(VirtIoError::InvalidInput)?; + let res = unsafe { + libc::syscall( + libc::SYS_mbind, + mmap_reg.as_ptr() as *mut libc::c_void, + mmap_reg.size(), + MPOL_PREFERRED, + &nodemask as *const u64, + MAX_NODE, + MPOL_MF_MOVE, + ) + }; + if res < 0 { + warn!( + self.logger, + "failed to mbind memory to host_numa_node_id {}: this may affect performance", + node_id; + "subsystem" => "virito-mem" + ); + } + Ok(()) + } + + fn configure_thp(&mut self, mmap_reg: &MmapRegion) -> Result<(), VirtIoError> { + debug!( + self.logger, + "Setting MADV_HUGEPAGE on AddressSpaceRegion addr {:x?} len {:x?}", + mmap_reg.as_ptr(), + mmap_reg.size(); + "subsystem" => "virito-mem" + ); + + // Safe because we just create the MmapRegion + unsafe { + mman::madvise( + mmap_reg.as_ptr() as *mut libc::c_void, + mmap_reg.size(), + mman::MmapAdvise::MADV_HUGEPAGE, + ) + } + .map_err(VirtIoError::Madvise)?; + + Ok(()) + } + + fn map_to_kvm( + &mut self, + slot: u32, + reg: &Arc, + mmap_reg: &MmapRegion, + ) -> Result<(), VirtIoError> { + let host_addr = mmap_reg.as_ptr() as u64; + + let flags = 0u32; + + let mem_region = kvm_userspace_memory_region { + slot, + guest_phys_addr: reg.start_addr().raw_value(), + memory_size: reg.len(), + userspace_addr: host_addr, + flags, + }; + + // Safe because the user mem region is just created, and kvm slot is allocated + // by resource allocator. + unsafe { self.vm_fd.set_user_memory_region(mem_region) } + .map_err(VirtIoError::SetUserMemoryRegion)?; + + Ok(()) + } +} + +impl MemRegionFactory for MemoryRegionFactory { + fn create_region( + &mut self, + guest_addr: GuestAddress, + region_len: GuestUsize, + kvm_slot: u32, + ) -> std::result::Result, VirtIoError> { + // create address space region + let mem_type = self.vm_config.mem_type.as_str(); + let mut mem_file_path = self.vm_config.mem_file_path.clone(); + let mem_file_name = format!( + "/virtiomem_{}_{}", + self.instance_id.as_str(), + self.mem_id.as_str() + ); + mem_file_path.push_str(mem_file_name.as_str()); + let region = Arc::new( + AddressSpaceRegion::create_default_memory_region( + guest_addr, + region_len, + self.host_numa_node_id, + mem_type, + mem_file_path.as_str(), + false, + true, + ) + .map_err(|e| { + error!(self.logger, "failed to insert address space region: {}", e); + // dbs-virtio-devices should not depend on dbs-address-space. + // So here io::Error is used instead of AddressSpaceError directly. + VirtIoError::IOError(io::Error::new( + io::ErrorKind::Other, + format!( + "invalid address space region ({0:#x}, {1:#x})", + guest_addr.0, region_len + ), + )) + })?, + ); + info!( + self.logger, + "VM: mem_type: {} mem_file_path: {}, numa_node_id: {:?} file_offset: {:?}", + mem_type, + mem_file_path, + self.host_numa_node_id, + region.file_offset(); + "subsystem" => "virito-mem" + ); + + let mmap_region = MmapRegion::build( + region.file_offset().cloned(), + region_len as usize, + region.prot_flags(), + region.perm_flags(), + ) + .map_err(VirtIoError::NewMmapRegion)?; + let host_addr: u64 = mmap_region.as_ptr() as u64; + + // thp + if mem_type == "hugeanon" || mem_type == "hugeshmem" { + self.configure_thp(&mmap_region)?; + } + + // Handle numa + if let Some(numa_node_id) = self.host_numa_node_id { + self.configure_numa(&mmap_region, numa_node_id)?; + } + + // add to guest memory mapping + self.map_to_kvm(kvm_slot, ®ion, &mmap_region)?; + + info!( + self.logger, + "kvm set user memory region: slot: {}, flags: {}, guest_phys_addr: {:X}, memory_size: {}, userspace_addr: {:X}", + kvm_slot, + 0, + guest_addr.raw_value(), + region_len, + host_addr; + "subsystem" => "virito-mem" + ); + + // All value should be valid. + let memory_region = Arc::new( + GuestRegionMmap::new(mmap_region, guest_addr).map_err(VirtIoError::InsertMmap)?, + ); + + let vm_as_new = self + .vm_as + .memory() + .insert_region(memory_region.clone()) + .map_err(VirtIoError::InsertMmap)?; + self.vm_as.lock().unwrap().replace(vm_as_new); + self.address_space.insert_region(region).map_err(|e| { + error!(self.logger, "failed to insert address space region: {}", e); + // dbs-virtio-devices should not depend on dbs-address-space. + // So here io::Error is used instead of AddressSpaceError directly. + VirtIoError::IOError(io::Error::new( + io::ErrorKind::Other, + format!( + "invalid address space region ({0:#x}, {1:#x})", + guest_addr.0, region_len + ), + )) + })?; + + Ok(memory_region) + } + + fn restore_region_addr( + &self, + guest_addr: GuestAddress, + ) -> std::result::Result<*mut u8, VirtIoError> { + let memory = self.vm_as.memory(); + // NOTE: We can't clone `GuestRegionMmap` reference directly!!! + // + // Since an important role of the member `mapping` (type is + // `MmapRegion`) in `GuestRegionMmap` is to mmap the memory during + // construction and munmap the memory during drop. However, when the + // life time of cloned data is over, the drop operation will be + // performed, which will munmap the origional mmap memory, which will + // cause some memory in dragonall to be inaccessable. And remember the + // data structure that was cloned is still alive now, when its life time + // is over, it will perform the munmap operation again, which will cause + // a memory exception! + memory + .get_host_address(guest_addr) + .map_err(VirtIoError::GuestMemory) + } + + fn get_host_numa_node_id(&self) -> Option { + self.host_numa_node_id + } + + fn set_host_numa_node_id(&mut self, host_numa_node_id: Option) { + self.host_numa_node_id = host_numa_node_id; + } +} + +#[cfg(test)] +mod tests { + use vm_memory::GuestMemoryRegion; + + use super::*; + use crate::test_utils::tests::create_vm_for_test; + + impl Default for MemDeviceConfigInfo { + fn default() -> Self { + MemDeviceConfigInfo { + mem_id: "".to_string(), + size_mib: 0, + capacity_mib: 1024, + multi_region: true, + host_numa_node_id: None, + guest_numa_node_id: None, + use_generic_irq: None, + use_shared_irq: None, + } + } + } + + #[test] + fn test_mem_config_check_conflicts() { + let config = MemDeviceConfigInfo::default(); + let mut config2 = MemDeviceConfigInfo::default(); + assert!(config.check_conflicts(&config2).is_err()); + config2.mem_id = "dummy_mem".to_string(); + assert!(config.check_conflicts(&config2).is_ok()); + } + + #[test] + fn test_create_mem_devices_configs() { + let mgr = MemDeviceMgr::default(); + assert_eq!(mgr.info_list.len(), 0); + assert_eq!(mgr.get_index_of_mem_dev(""), None); + } + + #[test] + fn test_mem_insert_or_update_device() { + // Init vm for test. + let mut vm = create_vm_for_test(); + + // We don't need to use virtio-mem before start vm + // Test for standard config with hotplug + let device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + vm.vm_address_space().cloned(), + true, + Some(VmConfigInfo::default()), + vm.shared_info().clone(), + ); + + let dummy_mem_device = MemDeviceConfigInfo::default(); + vm.device_manager_mut() + .mem_manager + .insert_or_update_device(device_op_ctx, dummy_mem_device) + .unwrap(); + assert_eq!(vm.device_manager().mem_manager.info_list.len(), 1); + } + + #[test] + fn test_mem_attach_device() { + // Init vm and insert mem config for test. + let mut vm = create_vm_for_test(); + let dummy_mem_device = MemDeviceConfigInfo::default(); + vm.device_manager_mut() + .mem_manager + .info_list + .insert_or_update(&dummy_mem_device) + .unwrap(); + assert_eq!(vm.device_manager().mem_manager.info_list.len(), 1); + + // Test for standard config + let mut device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + vm.vm_address_space().cloned(), + false, + Some(VmConfigInfo::default()), + vm.shared_info().clone(), + ); + vm.device_manager_mut() + .mem_manager + .attach_devices(&mut device_op_ctx) + .unwrap(); + assert_eq!(vm.device_manager().mem_manager.info_list.len(), 1); + } + + #[test] + fn test_mem_create_region() { + let vm = create_vm_for_test(); + let ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + vm.vm_address_space().cloned(), + true, + Some(VmConfigInfo::default()), + vm.shared_info().clone(), + ); + let mem_id = String::from("mem0"); + let guest_addr = GuestAddress(0x1_0000_0000); + let region_len = 0x1000_0000; + let kvm_slot = 2; + + // no vfio manager, no numa node + let mut factory = MemoryRegionFactory::new(&ctx, mem_id, None).unwrap(); + let region_opt = factory.create_region(guest_addr, region_len, kvm_slot); + assert_eq!(region_opt.unwrap().len(), region_len); + } +} diff --git a/src/dragonball/src/device_manager/memory_region_handler.rs b/src/dragonball/src/device_manager/memory_region_handler.rs index 2be149ef97f6..fbf5aa20cbf0 100644 --- a/src/dragonball/src/device_manager/memory_region_handler.rs +++ b/src/dragonball/src/device_manager/memory_region_handler.rs @@ -55,6 +55,7 @@ impl DeviceVirtioRegionHandler { None, file_offset, region.flags(), + region.prot(), false, )); diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 324cf41fcaa7..69112e3ae98e 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -90,6 +90,12 @@ mod memory_region_handler; #[cfg(feature = "virtio-fs")] pub use self::memory_region_handler::*; +#[cfg(feature = "virtio-mem")] +/// Device manager for virtio-mem devices. +pub mod mem_dev_mgr; +#[cfg(feature = "virtio-mem")] +use self::mem_dev_mgr::MemDeviceMgr; + macro_rules! info( ($l:expr, $($args:tt)+) => { slog::info!($l, $($args)+; slog::o!("subsystem" => "device_manager")) @@ -311,6 +317,27 @@ impl DeviceOpContext { } } + pub(crate) fn get_vm_config(&self) -> Result { + match self.vm_config.as_ref() { + Some(v) => Ok(v.clone()), + None => Err(DeviceMgrError::InvalidOperation), + } + } + + pub(crate) fn get_address_space(&self) -> Result { + match self.address_space.as_ref() { + Some(v) => Ok(v.clone()), + None => Err(DeviceMgrError::InvalidOperation), + } + } + + pub(crate) fn get_epoll_mgr(&self) -> Result { + match self.epoll_mgr.as_ref() { + Some(v) => Ok(v.clone()), + None => Err(DeviceMgrError::InvalidOperation), + } + } + pub(crate) fn logger(&self) -> &slog::Logger { &self.logger } @@ -498,6 +525,9 @@ pub struct DeviceManager { #[cfg(feature = "virtio-fs")] fs_manager: Arc>, + + #[cfg(feature = "virtio-mem")] + pub(crate) mem_manager: MemDeviceMgr, } impl DeviceManager { @@ -530,6 +560,8 @@ impl DeviceManager { virtio_net_manager: VirtioNetDeviceMgr::default(), #[cfg(feature = "virtio-fs")] fs_manager: Arc::new(Mutex::new(FsDeviceMgr::default())), + #[cfg(feature = "virtio-mem")] + mem_manager: MemDeviceMgr::default(), } } @@ -899,6 +931,24 @@ impl DeviceManager { ) } + /// Create an Virtio MMIO transport layer device for the virtio backend device with configure + /// change notification enabled. + pub fn create_mmio_virtio_device_with_device_change_notification( + device: DbsVirtioDevice, + ctx: &mut DeviceOpContext, + use_shared_irq: bool, + use_generic_irq: bool, + ) -> std::result::Result, DeviceMgrError> { + let features = DRAGONBALL_FEATURE_PER_QUEUE_NOTIFY; + DeviceManager::create_mmio_virtio_device_with_features( + device, + ctx, + Some(features), + use_shared_irq, + use_generic_irq, + ) + } + /// Create an Virtio MMIO transport layer device for the virtio backend device with specified /// features. pub fn create_mmio_virtio_device_with_features( @@ -1077,6 +1127,8 @@ mod tests { virtio_net_manager: VirtioNetDeviceMgr::default(), #[cfg(feature = "virtio-vsock")] vsock_manager: VsockDeviceMgr::default(), + #[cfg(feature = "virtio-mem")] + mem_manager: MemDeviceMgr::default(), #[cfg(target_arch = "aarch64")] mmio_device_info: HashMap::new(), From 8fb7ab75186b3077e556c566e2beb951571c4035 Mon Sep 17 00:00:00 2001 From: Helin Guo Date: Sun, 29 Jan 2023 16:18:06 +0800 Subject: [PATCH 3/3] dragonball: introduce virtio-balloon device We introduce virtio-balloon device to support memory resize. virtio-balloon device could reclaim memory from guest to host. Fixes: #6719 Signed-off-by: Helin Guo --- src/dragonball/Cargo.lock | 124 ++++-- src/dragonball/Cargo.toml | 1 + src/dragonball/src/api/v1/vmm_action.rs | 87 +++- .../src/device_manager/balloon_dev_mgr.rs | 419 ++++++++++++++++++ src/dragonball/src/device_manager/mod.rs | 13 + src/dragonball/src/error.rs | 5 + src/dragonball/src/vm/mod.rs | 13 + 7 files changed, 632 insertions(+), 30 deletions(-) create mode 100644 src/dragonball/src/device_manager/balloon_dev_mgr.rs diff --git a/src/dragonball/Cargo.lock b/src/dragonball/Cargo.lock index 3cc177fe1cf3..0ed990c0becd 100644 --- a/src/dragonball/Cargo.lock +++ b/src/dragonball/Cargo.lock @@ -180,9 +180,9 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.7" +version = "0.5.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf2b3e8478797446514c91ef04bafcb59faba183e621ad488df88983cc14128c" +checksum = "a33c2bf77f2df06183c3aa30d1e96c0695a313d4f9c453cc3762a6db39f99200" dependencies = [ "cfg-if", "crossbeam-utils", @@ -445,13 +445,13 @@ dependencies = [ [[package]] name = "errno" -version = "0.2.8" +version = "0.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f639046355ee4f37944e44f60642c6f3a7efa3cf6b78c78a0d989a8ce6c396a1" +checksum = "4bcfec3a70f97c962c307b2d2c56e358cf1d00b558d74262b5f929ee8cc7e73a" dependencies = [ "errno-dragonfly", "libc", - "winapi", + "windows-sys 0.48.0", ] [[package]] @@ -483,7 +483,7 @@ dependencies = [ "cfg-if", "libc", "redox_syscall", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -619,9 +619,9 @@ dependencies = [ [[package]] name = "getrandom" -version = "0.2.8" +version = "0.2.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c05aeb6a22b8f62540c194aac980f2115af067bfe15a0734d7277a768d396b31" +checksum = "c85e1d9ab2eadba7e5040d4e09cbd6d072b76a557ad64e797c2cb9d4da21d7e4" dependencies = [ "cfg-if", "libc", @@ -671,7 +671,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1abeb7a0dd0f8181267ff8adc397075586500b81b28a73e8a0208b00fc170fb3" dependencies = [ "libc", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -842,7 +842,7 @@ dependencies = [ "libc", "log", "wasi", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1045,7 +1045,7 @@ dependencies = [ "libc", "redox_syscall", "smallvec", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1121,16 +1121,16 @@ checksum = "7ef03e0a2b150c7a90d01faf6254c9c48a41e95fb2a8c2ac1c6f0d2b9aefc342" [[package]] name = "rustix" -version = "0.36.8" +version = "0.36.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43abb88211988493c1abb44a70efa56ff0ce98f233b7b276146f1f3f7ba9644" +checksum = "14e4d67015953998ad0eb82887a0eb0129e18a7e2f3b7b0f6c422fddcd503d62" dependencies = [ "bitflags", "errno", "io-lifetimes", "libc", "linux-raw-sys", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1168,18 +1168,18 @@ dependencies = [ [[package]] name = "serde" -version = "1.0.152" +version = "1.0.156" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bb7d1f0d3021d347a83e556fc4683dea2ea09d87bccdf88ff5c12545d89d5efb" +checksum = "314b5b092c0ade17c00142951e50ced110ec27cea304b1037c6969246c2469a4" dependencies = [ "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.152" +version = "1.0.156" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af487d118eecd09402d70a5d72551860e788df87b464af30e5ea6a38c75c541e" +checksum = "d7e29c4601e36bcec74a223228dce795f4cd3616341a4af93520ca1a837c087d" dependencies = [ "proc-macro2", "quote", @@ -1188,9 +1188,9 @@ dependencies = [ [[package]] name = "serde_json" -version = "1.0.93" +version = "1.0.96" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cad406b69c91885b5107daf2c29572f6c8cdb3c66826821e286c533490c0bc76" +checksum = "057d394a50403bcac12672b2b18fb387ab6d289d957dab67dd201875391e52f1" dependencies = [ "itoa", "ryu", @@ -1423,7 +1423,7 @@ dependencies = [ "pin-project-lite", "socket2", "tokio-macros", - "windows-sys", + "windows-sys 0.45.0", ] [[package]] @@ -1611,7 +1611,16 @@ version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets", + "windows-targets 0.42.1", +] + +[[package]] +name = "windows-sys" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +dependencies = [ + "windows-targets 0.48.0", ] [[package]] @@ -1620,13 +1629,28 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e2522491fbfcd58cc84d47aeb2958948c4b8982e9a2d8a2a35bbaed431390e7" dependencies = [ - "windows_aarch64_gnullvm", - "windows_aarch64_msvc", - "windows_i686_gnu", - "windows_i686_msvc", - "windows_x86_64_gnu", - "windows_x86_64_gnullvm", - "windows_x86_64_msvc", + "windows_aarch64_gnullvm 0.42.1", + "windows_aarch64_msvc 0.42.1", + "windows_i686_gnu 0.42.1", + "windows_i686_msvc 0.42.1", + "windows_x86_64_gnu 0.42.1", + "windows_x86_64_gnullvm 0.42.1", + "windows_x86_64_msvc 0.42.1", +] + +[[package]] +name = "windows-targets" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b1eb6f0cd7c80c79759c929114ef071b87354ce476d9d94271031c0497adfd5" +dependencies = [ + "windows_aarch64_gnullvm 0.48.0", + "windows_aarch64_msvc 0.48.0", + "windows_i686_gnu 0.48.0", + "windows_i686_msvc 0.48.0", + "windows_x86_64_gnu 0.48.0", + "windows_x86_64_gnullvm 0.48.0", + "windows_x86_64_msvc 0.48.0", ] [[package]] @@ -1635,42 +1659,84 @@ version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8c9864e83243fdec7fc9c5444389dcbbfd258f745e7853198f365e3c4968a608" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91ae572e1b79dba883e0d315474df7305d12f569b400fcf90581b06062f7e1bc" + [[package]] name = "windows_aarch64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4c8b1b673ffc16c47a9ff48570a9d85e25d265735c503681332589af6253c6c7" +[[package]] +name = "windows_aarch64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2ef27e0d7bdfcfc7b868b317c1d32c641a6fe4629c171b8928c7b08d98d7cf3" + [[package]] name = "windows_i686_gnu" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de3887528ad530ba7bdbb1faa8275ec7a1155a45ffa57c37993960277145d640" +[[package]] +name = "windows_i686_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "622a1962a7db830d6fd0a69683c80a18fda201879f0f447f065a3b7467daa241" + [[package]] name = "windows_i686_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf4d1122317eddd6ff351aa852118a2418ad4214e6613a50e0191f7004372605" +[[package]] +name = "windows_i686_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4542c6e364ce21bf45d69fdd2a8e455fa38d316158cfd43b3ac1c5b1b19f8e00" + [[package]] name = "windows_x86_64_gnu" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c1040f221285e17ebccbc2591ffdc2d44ee1f9186324dd3e84e99ac68d699c45" +[[package]] +name = "windows_x86_64_gnu" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ca2b8a661f7628cbd23440e50b05d705db3686f894fc9580820623656af974b1" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "628bfdf232daa22b0d64fdb62b09fcc36bb01f05a3939e20ab73aaf9470d0463" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7896dbc1f41e08872e9d5e8f8baa8fdd2677f29468c4e156210174edc7f7b953" + [[package]] name = "windows_x86_64_msvc" version = "0.42.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "447660ad36a13288b1db4d4248e857b510e8c3a225c822ba4fb748c0aafecffd" +[[package]] +name = "windows_x86_64_msvc" +version = "0.48.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a515f5799fe4961cb532f983ce2b23082366b898e52ffbce459c86f67c8378a" + [[package]] name = "xattr" version = "0.2.3" diff --git a/src/dragonball/Cargo.toml b/src/dragonball/Cargo.toml index 0c94530ca0af..b0e03047d84b 100644 --- a/src/dragonball/Cargo.toml +++ b/src/dragonball/Cargo.toml @@ -56,3 +56,4 @@ virtio-net = ["dbs-virtio-devices/virtio-net", "virtio-queue"] # virtio-fs only work on atomic-guest-memory virtio-fs = ["dbs-virtio-devices/virtio-fs", "virtio-queue", "atomic-guest-memory"] virtio-mem = ["dbs-virtio-devices/virtio-mem", "virtio-queue", "atomic-guest-memory"] +virtio-balloon = ["dbs-virtio-devices/virtio-balloon", "virtio-queue"] diff --git a/src/dragonball/src/api/v1/vmm_action.rs b/src/dragonball/src/api/v1/vmm_action.rs index 6243b82a3340..247daf10d51a 100644 --- a/src/dragonball/src/api/v1/vmm_action.rs +++ b/src/dragonball/src/api/v1/vmm_action.rs @@ -19,6 +19,8 @@ use crate::vmm::Vmm; use self::VmConfigError::*; use self::VmmActionError::MachineConfig; +#[cfg(feature = "virtio-balloon")] +pub use crate::device_manager::balloon_dev_mgr::{BalloonDeviceConfigInfo, BalloonDeviceError}; #[cfg(feature = "virtio-blk")] pub use crate::device_manager::blk_dev_mgr::{ BlockDeviceConfigInfo, BlockDeviceConfigUpdateInfo, BlockDeviceError, BlockDeviceMgr, @@ -36,7 +38,6 @@ pub use crate::device_manager::virtio_net_dev_mgr::{ }; #[cfg(feature = "virtio-vsock")] pub use crate::device_manager::vsock_dev_mgr::{VsockDeviceConfigInfo, VsockDeviceError}; - #[cfg(feature = "hotplug")] pub use crate::vcpu::{VcpuResizeError, VcpuResizeInfo}; @@ -108,6 +109,11 @@ pub enum VmmActionError { /// Mem device related errors. #[error("virtio-mem device error: {0}")] Mem(#[source] MemDeviceError), + + #[cfg(feature = "virtio-balloon")] + /// Balloon device related errors. + #[error("virtio-balloon device error: {0}")] + Balloon(#[source] BalloonDeviceError), } /// This enum represents the public interface of the VMM. Each action contains various @@ -187,6 +193,11 @@ pub enum VmmAction { #[cfg(feature = "virtio-mem")] /// Add a new mem device or update one that already exists using the `MemDeviceConfig` as input. InsertMemDevice(MemDeviceConfigInfo), + + #[cfg(feature = "virtio-balloon")] + /// Add a new balloon device or update one that already exists using the `BalloonDeviceConfig` + /// as input. + InsertBalloonDevice(BalloonDeviceConfigInfo), } /// The enum represents the response sent by the VMM in case of success. The response is either @@ -291,6 +302,10 @@ impl VmmService { VmmAction::ResizeVcpu(vcpu_resize_cfg) => self.resize_vcpu(vmm, vcpu_resize_cfg), #[cfg(feature = "virtio-mem")] VmmAction::InsertMemDevice(mem_cfg) => self.add_mem_device(vmm, event_mgr, mem_cfg), + #[cfg(feature = "virtio-balloon")] + VmmAction::InsertBalloonDevice(balloon_cfg) => { + self.add_balloon_device(vmm, event_mgr, balloon_cfg) + } }; debug!("send vmm response: {:?}", response); @@ -691,6 +706,36 @@ impl VmmService { .map(|_| VmmData::Empty) .map_err(VmmActionError::Mem) } + + #[cfg(feature = "virtio-balloon")] + fn add_balloon_device( + &mut self, + vmm: &mut Vmm, + event_mgr: &mut EventManager, + config: BalloonDeviceConfigInfo, + ) -> VmmRequestResult { + let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?; + + if config.size_mib != 0 { + info!("add_balloon_device: wait prealloc"); + vm.stop_prealloc().map_err(VmmActionError::StartMicroVm)?; + } + let ctx = vm + .create_device_op_context(Some(event_mgr.epoll_manager())) + .map_err(|e| { + if let StartMicroVmError::UpcallServerNotReady = e { + VmmActionError::UpcallServerNotReady + } else { + VmmActionError::StartMicroVm(e) + } + })?; + + vm.device_manager_mut() + .balloon_manager + .insert_or_update_device(ctx, config) + .map(|_| VmmData::Empty) + .map_err(VmmActionError::Balloon) + } } fn handle_cpu_topology( @@ -1539,4 +1584,44 @@ mod tests { t.check_request(); } } + + #[cfg(feature = "virtio-balloon")] + #[test] + fn test_vmm_action_insert_balloon_device() { + skip_if_not_root!(); + + let tests = &mut [ + // hotplug unready + TestData::new( + VmmAction::InsertBalloonDevice(BalloonDeviceConfigInfo::default()), + InstanceState::Running, + &|result| { + assert!(matches!( + result, + Err(VmmActionError::StartMicroVm( + StartMicroVmError::UpcallMissVsock + )) + )); + let err_string = format!("{}", result.unwrap_err()); + let expected_err = String::from( + "failed to boot the VM: \ + the upcall client needs a virtio-vsock device for communication", + ); + assert_eq!(err_string, expected_err); + }, + ), + // success + TestData::new( + VmmAction::InsertBalloonDevice(BalloonDeviceConfigInfo::default()), + InstanceState::Uninitialized, + &|result| { + assert!(result.is_ok()); + }, + ), + ]; + + for t in tests.iter_mut() { + t.check_request(); + } + } } diff --git a/src/dragonball/src/device_manager/balloon_dev_mgr.rs b/src/dragonball/src/device_manager/balloon_dev_mgr.rs new file mode 100644 index 000000000000..b0ee2bd37e29 --- /dev/null +++ b/src/dragonball/src/device_manager/balloon_dev_mgr.rs @@ -0,0 +1,419 @@ +// Copyright 2020 Alibaba Cloud. All Rights Reserved. +// SPDX-License-Identifier: Apache-2.0 + +use dbs_virtio_devices as virtio; +use serde_derive::{Deserialize, Serialize}; +use slog::{error, info}; +use virtio::balloon::{Balloon, BalloonConfig}; +use virtio::Error as VirtIoError; + +use crate::address_space_manager::GuestAddressSpaceImpl; +use crate::config_manager::{ConfigItem, DeviceConfigInfo, DeviceConfigInfos}; +use crate::device_manager::DbsMmioV2Device; +use crate::device_manager::{DeviceManager, DeviceMgrError, DeviceOpContext}; + +// The flag of whether to use the shared irq. +const USE_SHARED_IRQ: bool = true; +// The flag of whether to use the generic irq. +const USE_GENERIC_IRQ: bool = false; + +/// Errors associated with `BalloonDeviceConfig`. +#[derive(Debug, thiserror::Error)] +pub enum BalloonDeviceError { + /// The balloon device was already used. + #[error("the virtio-balloon ID was already added to a different device")] + BalloonDeviceAlreadyExists, + + /// Cannot perform the requested operation after booting the microVM. + #[error("the update operation is not allowed after boot")] + UpdateNotAllowedPostBoot, + + /// guest memory error + #[error("failed to access guest memory, {0}")] + GuestMemoryError(#[source] vm_memory::mmap::Error), + + /// create balloon device error + #[error("failed to create virtio-balloon device, {0}")] + CreateBalloonDevice(#[source] virtio::Error), + + /// hotplug balloon device error + #[error("cannot hotplug virtio-balloon device, {0}")] + HotplugDeviceFailed(#[source] DeviceMgrError), + + /// create mmio device error + #[error("cannot create virtio-balloon mmio device, {0}")] + CreateMmioDevice(#[source] DeviceMgrError), + + /// Cannot initialize a balloon device or add a device to the MMIO Bus. + #[error("failure while registering balloon device: {0}")] + RegisterBalloonDevice(#[source] DeviceMgrError), + + /// resize balloon device error + #[error("failure while resizing virtio-balloon device, {0}")] + ResizeFailed(#[source] VirtIoError), + + /// The balloon device id doesn't exist. + #[error("invalid balloon device id '{0}'")] + InvalidDeviceId(String), + + /// balloon device does not exist + #[error("balloon device does not exist")] + NotExist, + + /// The device manager errors. + #[error("DeviceManager error: {0}")] + DeviceManager(#[source] DeviceMgrError), +} + +/// Configuration information for a virtio-balloon device. +#[derive(Clone, Debug, Deserialize, PartialEq, Eq, Serialize)] +pub struct BalloonDeviceConfigInfo { + /// Unique identifier of the balloon device + pub balloon_id: String, + /// Resize balloon size in mib + pub size_mib: u64, + /// Use shared irq + pub use_shared_irq: Option, + /// Use generic irq + pub use_generic_irq: Option, + /// VIRTIO_BALLOON_F_DEFLATE_ON_OOM + pub f_deflate_on_oom: bool, + /// VIRTIO_BALLOON_F_REPORTING + pub f_reporting: bool, +} + +impl ConfigItem for BalloonDeviceConfigInfo { + type Err = BalloonDeviceError; + + fn id(&self) -> &str { + &self.balloon_id + } + + fn check_conflicts(&self, other: &Self) -> Result<(), BalloonDeviceError> { + if self.balloon_id.as_str() == other.balloon_id.as_str() { + Err(BalloonDeviceError::BalloonDeviceAlreadyExists) + } else { + Ok(()) + } + } +} + +/// Balloon Device Info +pub type BalloonDeviceInfo = DeviceConfigInfo; + +impl ConfigItem for BalloonDeviceInfo { + type Err = BalloonDeviceError; + + fn id(&self) -> &str { + &self.config.balloon_id + } + + fn check_conflicts(&self, other: &Self) -> Result<(), BalloonDeviceError> { + if self.config.balloon_id.as_str() == other.config.balloon_id.as_str() { + Err(BalloonDeviceError::BalloonDeviceAlreadyExists) + } else { + Ok(()) + } + } +} + +/// Wrapper for the collection that holds all the Balloon Devices Configs +#[derive(Clone)] +pub struct BalloonDeviceMgr { + /// A list of `BalloonDeviceConfig` objects. + info_list: DeviceConfigInfos, + pub(crate) use_shared_irq: bool, +} + +impl BalloonDeviceMgr { + /// Inserts `balloon_cfg` in the virtio-balloon device configuration list. + /// If an entry with the same id already exists, it will attempt to update + /// the existing entry. + pub fn insert_or_update_device( + &mut self, + mut ctx: DeviceOpContext, + balloon_cfg: BalloonDeviceConfigInfo, + ) -> std::result::Result<(), BalloonDeviceError> { + if !cfg!(feature = "hotplug") && ctx.is_hotplug { + error!(ctx.logger(), "hotplug feature has been disabled."; + "subsystem" => "balloon_dev_mgr",); + return Err(BalloonDeviceError::UpdateNotAllowedPostBoot); + } + + let epoll_mgr = ctx + .get_epoll_mgr() + .map_err(BalloonDeviceError::DeviceManager)?; + + // If the id of the drive already exists in the list, the operation is update. + if let Some(index) = self.get_index_of_balloon_dev(&balloon_cfg.balloon_id) { + // Update an existing balloon device + if ctx.is_hotplug { + info!(ctx.logger(), "resize virtio balloon size to {:?}", balloon_cfg.size_mib; "subsystem" => "balloon_dev_mgr"); + self.update_balloon_size(index, balloon_cfg.size_mib)?; + } + self.info_list.insert_or_update(&balloon_cfg)?; + } else { + // Create a new balloon device + if !self.info_list.is_empty() { + error!(ctx.logger(), "only support one balloon device!"; "subsystem" => "balloon_dev_mgr"); + return Err(BalloonDeviceError::BalloonDeviceAlreadyExists); + } + + if !ctx.is_hotplug { + self.info_list.insert_or_update(&balloon_cfg)?; + return Ok(()); + } + + info!(ctx.logger(), "hotplug balloon device: {}", balloon_cfg.balloon_id; "subsystem" => "balloon_dev_mgr"); + let device = Box::new( + virtio::balloon::Balloon::new( + epoll_mgr, + BalloonConfig { + f_deflate_on_oom: balloon_cfg.f_deflate_on_oom, + f_reporting: balloon_cfg.f_reporting, + }, + ) + .map_err(BalloonDeviceError::CreateBalloonDevice)?, + ); + + let mmio_dev = + DeviceManager::create_mmio_virtio_device_with_device_change_notification( + device, + &mut ctx, + balloon_cfg.use_shared_irq.unwrap_or(self.use_shared_irq), + balloon_cfg.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(BalloonDeviceError::CreateMmioDevice)?; + ctx.insert_hotplug_mmio_device(&mmio_dev, None) + .map_err(|e| { + error!( + ctx.logger(), + "hotplug balloon device {} error: {}", + &balloon_cfg.balloon_id, e; + "subsystem" => "balloon_dev_mgr" + ); + BalloonDeviceError::HotplugDeviceFailed(e) + })?; + let index = self.info_list.insert_or_update(&balloon_cfg)?; + self.info_list[index].set_device(mmio_dev); + } + Ok(()) + } + + /// Attaches all virtio-balloon devices from the BalloonDevicesConfig. + pub fn attach_devices( + &mut self, + ctx: &mut DeviceOpContext, + ) -> std::result::Result<(), BalloonDeviceError> { + let epoll_mgr = ctx + .get_epoll_mgr() + .map_err(BalloonDeviceError::DeviceManager)?; + + for info in self.info_list.iter_mut() { + info!(ctx.logger(), "attach balloon device: {}", info.config.balloon_id; "subsystem" => "balloon_dev_mgr"); + + let device = Balloon::new( + epoll_mgr.clone(), + BalloonConfig { + f_deflate_on_oom: info.config.f_deflate_on_oom, + f_reporting: info.config.f_reporting, + }, + ) + .map_err(BalloonDeviceError::CreateBalloonDevice)?; + let mmio_dev = + DeviceManager::create_mmio_virtio_device_with_device_change_notification( + Box::new(device), + ctx, + info.config.use_shared_irq.unwrap_or(self.use_shared_irq), + info.config.use_generic_irq.unwrap_or(USE_GENERIC_IRQ), + ) + .map_err(BalloonDeviceError::RegisterBalloonDevice)?; + info.set_device(mmio_dev); + } + + Ok(()) + } + + fn update_balloon_size( + &self, + index: usize, + size_mib: u64, + ) -> std::result::Result<(), BalloonDeviceError> { + let device = self.info_list[index] + .device + .as_ref() + .ok_or_else(|| BalloonDeviceError::NotExist)?; + if let Some(mmio_dev) = device.as_any().downcast_ref::() { + let guard = mmio_dev.state(); + let inner_dev = guard.get_inner_device(); + if let Some(balloon_dev) = inner_dev + .as_any() + .downcast_ref::>() + { + return balloon_dev + .set_size(size_mib) + .map_err(BalloonDeviceError::ResizeFailed); + } + } + Ok(()) + } + + fn get_index_of_balloon_dev(&self, balloon_id: &str) -> Option { + self.info_list + .iter() + .position(|info| info.config.balloon_id.eq(balloon_id)) + } +} + +impl Default for BalloonDeviceMgr { + /// Create a new `BalloonDeviceMgr` object.. + fn default() -> Self { + BalloonDeviceMgr { + info_list: DeviceConfigInfos::new(), + use_shared_irq: USE_SHARED_IRQ, + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::test_utils::tests::create_vm_for_test; + + impl Default for BalloonDeviceConfigInfo { + fn default() -> Self { + BalloonDeviceConfigInfo { + balloon_id: "".to_string(), + size_mib: 0, + use_generic_irq: None, + use_shared_irq: None, + f_deflate_on_oom: false, + f_reporting: false, + } + } + } + + #[test] + fn test_balloon_config_check_conflicts() { + let config = BalloonDeviceConfigInfo::default(); + let mut config2 = BalloonDeviceConfigInfo::default(); + assert!(config.check_conflicts(&config2).is_err()); + config2.balloon_id = "dummy_balloon".to_string(); + assert!(config.check_conflicts(&config2).is_ok()); + } + + #[test] + fn test_create_balloon_devices_configs() { + let mgr = BalloonDeviceMgr::default(); + assert_eq!(mgr.info_list.len(), 0); + assert_eq!(mgr.get_index_of_balloon_dev(""), None); + } + + #[test] + fn test_balloon_insert_or_update_device() { + //Init vm for test. + let mut vm = create_vm_for_test(); + + // Test for standard config + let device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), + ); + + let dummy_balloon_device = BalloonDeviceConfigInfo::default(); + vm.device_manager_mut() + .balloon_manager + .insert_or_update_device(device_op_ctx, dummy_balloon_device) + .unwrap(); + assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1); + } + + #[test] + fn test_balloon_attach_device() { + //Init vm and insert balloon config for test. + let mut vm = create_vm_for_test(); + let device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), + ); + + let dummy_balloon_device = BalloonDeviceConfigInfo::default(); + vm.device_manager_mut() + .balloon_manager + .insert_or_update_device(device_op_ctx, dummy_balloon_device) + .unwrap(); + assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1); + + // Test for standard config + let mut device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), + ); + assert!(vm + .device_manager_mut() + .balloon_manager + .attach_devices(&mut device_op_ctx) + .is_ok()); + assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1); + } + + #[test] + fn test_balloon_update_device() { + //Init vm for test. + let mut vm = create_vm_for_test(); + let device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), + ); + + let dummy_balloon_device = BalloonDeviceConfigInfo::default(); + vm.device_manager_mut() + .balloon_manager + .insert_or_update_device(device_op_ctx, dummy_balloon_device) + .unwrap(); + assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1); + + let mut device_op_ctx = DeviceOpContext::new( + Some(vm.epoll_manager().clone()), + vm.device_manager(), + Some(vm.vm_as().unwrap().clone()), + None, + false, + Some(vm.vm_config().clone()), + vm.shared_info().clone(), + ); + + assert!(vm + .device_manager_mut() + .balloon_manager + .attach_devices(&mut device_op_ctx) + .is_ok()); + assert_eq!(vm.device_manager().balloon_manager.info_list.len(), 1); + + assert!(vm + .device_manager() + .balloon_manager + .update_balloon_size(0, 200) + .is_ok()); + } +} diff --git a/src/dragonball/src/device_manager/mod.rs b/src/dragonball/src/device_manager/mod.rs index 69112e3ae98e..b007270c94df 100644 --- a/src/dragonball/src/device_manager/mod.rs +++ b/src/dragonball/src/device_manager/mod.rs @@ -96,6 +96,12 @@ pub mod mem_dev_mgr; #[cfg(feature = "virtio-mem")] use self::mem_dev_mgr::MemDeviceMgr; +#[cfg(feature = "virtio-balloon")] +/// Device manager for virtio-balloon devices. +pub mod balloon_dev_mgr; +#[cfg(feature = "virtio-balloon")] +use self::balloon_dev_mgr::BalloonDeviceMgr; + macro_rules! info( ($l:expr, $($args:tt)+) => { slog::info!($l, $($args)+; slog::o!("subsystem" => "device_manager")) @@ -528,6 +534,9 @@ pub struct DeviceManager { #[cfg(feature = "virtio-mem")] pub(crate) mem_manager: MemDeviceMgr, + + #[cfg(feature = "virtio-balloon")] + pub(crate) balloon_manager: BalloonDeviceMgr, } impl DeviceManager { @@ -562,6 +571,8 @@ impl DeviceManager { fs_manager: Arc::new(Mutex::new(FsDeviceMgr::default())), #[cfg(feature = "virtio-mem")] mem_manager: MemDeviceMgr::default(), + #[cfg(feature = "virtio-balloon")] + balloon_manager: BalloonDeviceMgr::default(), } } @@ -1129,6 +1140,8 @@ mod tests { vsock_manager: VsockDeviceMgr::default(), #[cfg(feature = "virtio-mem")] mem_manager: MemDeviceMgr::default(), + #[cfg(feature = "virtio-balloon")] + balloon_manager: BalloonDeviceMgr::default(), #[cfg(target_arch = "aarch64")] mmio_device_info: HashMap::new(), diff --git a/src/dragonball/src/error.rs b/src/dragonball/src/error.rs index 66a24f5629ea..e0aab17fbe0d 100644 --- a/src/dragonball/src/error.rs +++ b/src/dragonball/src/error.rs @@ -188,6 +188,11 @@ pub enum StartMicroVmError { /// Virtio-fs errors. #[error("virtio-fs errors: {0}")] FsDeviceError(#[source] device_manager::fs_dev_mgr::FsDeviceError), + + #[cfg(feature = "virtio-balloon")] + /// Virtio-balloon errors. + #[error("virtio-balloon errors: {0}")] + BalloonDeviceError(#[source] device_manager::balloon_dev_mgr::BalloonDeviceError), } /// Errors associated with starting the instance. diff --git a/src/dragonball/src/vm/mod.rs b/src/dragonball/src/vm/mod.rs index 6e2545c6f1cb..9af046c43b10 100644 --- a/src/dragonball/src/vm/mod.rs +++ b/src/dragonball/src/vm/mod.rs @@ -386,6 +386,19 @@ impl Vm { (dragonball_version, instance_id) } + + pub(crate) fn stop_prealloc(&mut self) -> std::result::Result<(), StartMicroVmError> { + if self.address_space.is_initialized() { + return self + .address_space + .wait_prealloc(true) + .map_err(StartMicroVmError::AddressManagerError); + } + + Err(StartMicroVmError::AddressManagerError( + AddressManagerError::GuestMemoryNotInitialized, + )) + } } impl Vm {