dragonball: introduce virtio-mem device

We introduce virtio-mem device to support memory resize. virtio-mem
device could hot-plug more memory blocks to guest and could also
hot-unplug them from guest.

Fixes: kata-containers#6719

Signed-off-by: Helin Guo <[email protected]>

src/dragonball/Cargo.toml

@@ -12,16 +12,16 @@ edition = "2018"
 [dependencies]
 arc-swap = "1.5.0"
 bytes = "1.1.0"
-dbs-address-space = "0.2.0"
+dbs-address-space = "0.3.0"
 dbs-allocator = "0.1.0"
 dbs-arch = "0.2.0"
 dbs-boot = "0.4.0"
 dbs-device = "0.2.0"
 dbs-interrupt = { version = "0.2.0", features = ["kvm-irq"] }
 dbs-legacy-devices = "0.1.0"
-dbs-upcall = { version = "0.2.0", optional = true }
+dbs-upcall = { version = "0.3.0", optional = true }
 dbs-utils = "0.2.0"
-dbs-virtio-devices = { version = "0.2.0", optional = true, features = ["virtio-mmio"] }
+dbs-virtio-devices = { version = "0.3.1", optional = true, features = ["virtio-mmio"] }
 kvm-bindings = "0.6.0"
 kvm-ioctls = "0.12.0"
 lazy_static = "1.2"
@@ -55,3 +55,4 @@ virtio-blk = ["dbs-virtio-devices/virtio-blk", "virtio-queue"]
 virtio-net = ["dbs-virtio-devices/virtio-net", "virtio-queue"]
 # virtio-fs only work on atomic-guest-memory
 virtio-fs = ["dbs-virtio-devices/virtio-fs", "virtio-queue", "atomic-guest-memory"]
+virtio-mem = ["dbs-virtio-devices/virtio-mem", "virtio-queue", "atomic-guest-memory"]

src/dragonball/src/api/v1/vmm_action.rs

@@ -27,6 +27,8 @@ pub use crate::device_manager::blk_dev_mgr::{
 pub use crate::device_manager::fs_dev_mgr::{
     FsDeviceConfigInfo, FsDeviceConfigUpdateInfo, FsDeviceError, FsDeviceMgr, FsMountConfigInfo,
 };
+#[cfg(feature = "virtio-mem")]
+pub use crate::device_manager::mem_dev_mgr::{MemDeviceConfigInfo, MemDeviceError};
 #[cfg(feature = "virtio-net")]
 pub use crate::device_manager::virtio_net_dev_mgr::{
     VirtioNetDeviceConfigInfo, VirtioNetDeviceConfigUpdateInfo, VirtioNetDeviceError,
@@ -97,6 +99,15 @@ pub enum VmmActionError {
     /// The action `ResizeVcpu` Failed
     #[error("vcpu resize error : {0}")]
     ResizeVcpu(#[source] VcpuResizeError),
+
+    /// Cannot access address space.
+    #[error("Cannot access address space.")]
+    AddressSpaceNotInitialized,
+
+    #[cfg(feature = "virtio-mem")]
+    /// Mem device related errors.
+    #[error("virtio-mem device error: {0}")]
+    Mem(#[source] MemDeviceError),
 }
 
 /// This enum represents the public interface of the VMM. Each action contains various
@@ -172,6 +183,10 @@ pub enum VmmAction {
     #[cfg(feature = "hotplug")]
     /// Resize Vcpu number in the guest.
     ResizeVcpu(VcpuResizeInfo),
+
+    #[cfg(feature = "virtio-mem")]
+    /// Add a new mem device or update one that already exists using the `MemDeviceConfig` as input.
+    InsertMemDevice(MemDeviceConfigInfo),
 }
 
 /// The enum represents the response sent by the VMM in case of success. The response is either
@@ -274,6 +289,8 @@ impl VmmService {
             }
             #[cfg(feature = "hotplug")]
             VmmAction::ResizeVcpu(vcpu_resize_cfg) => self.resize_vcpu(vmm, vcpu_resize_cfg),
+            #[cfg(feature = "virtio-mem")]
+            VmmAction::InsertMemDevice(mem_cfg) => self.add_mem_device(vmm, event_mgr, mem_cfg),
         };
 
         debug!("send vmm response: {:?}", response);
@@ -648,6 +665,32 @@ impl VmmService {
 
         Ok(VmmData::Empty)
     }
+
+    #[cfg(feature = "virtio-mem")]
+    fn add_mem_device(
+        &mut self,
+        vmm: &mut Vmm,
+        event_mgr: &mut EventManager,
+        config: MemDeviceConfigInfo,
+    ) -> VmmRequestResult {
+        let vm = vmm.get_vm_mut().ok_or(VmmActionError::InvalidVMID)?;
+
+        let ctx = vm
+            .create_device_op_context(Some(event_mgr.epoll_manager()))
+            .map_err(|e| {
+                if let StartMicroVmError::UpcallServerNotReady = e {
+                    VmmActionError::UpcallServerNotReady
+                } else {
+                    VmmActionError::StartMicroVm(e)
+                }
+            })?;
+
+        vm.device_manager_mut()
+            .mem_manager
+            .insert_or_update_device(ctx, config)
+            .map(|_| VmmData::Empty)
+            .map_err(VmmActionError::Mem)
+    }
 }
 
 fn handle_cpu_topology(
@@ -1456,4 +1499,44 @@ mod tests {
             t.check_request();
         }
     }
+
+    #[cfg(feature = "virtio-mem")]
+    #[test]
+    fn test_vmm_action_insert_mem_device() {
+        skip_if_not_root!();
+
+        let tests = &mut [
+            // hotplug unready
+            TestData::new(
+                VmmAction::InsertMemDevice(MemDeviceConfigInfo::default()),
+                InstanceState::Running,
+                &|result| {
+                    assert!(matches!(
+                        result,
+                        Err(VmmActionError::StartMicroVm(
+                            StartMicroVmError::UpcallMissVsock
+                        ))
+                    ));
+                    let err_string = format!("{}", result.unwrap_err());
+                    let expected_err = String::from(
+                        "failed to boot the VM: \
+                        the upcall client needs a virtio-vsock device for communication",
+                    );
+                    assert_eq!(err_string, expected_err);
+                },
+            ),
+            // success
+            TestData::new(
+                VmmAction::InsertMemDevice(MemDeviceConfigInfo::default()),
+                InstanceState::Uninitialized,
+                &|result| {
+                    assert!(result.is_ok());
+                },
+            ),
+        ];
+
+        for t in tests.iter_mut() {
+            t.check_request();
+        }
+    }
 }

src/dragonball/src/device_manager/blk_dev_mgr.rs

@@ -871,6 +871,8 @@ mod tests {
             Some(vm.vm_as().unwrap().clone()),
             None,
             false,
+            Some(vm.vm_config().clone()),
+            vm.shared_info().clone(),
         );
 
         let dummy_file = TempFile::new().unwrap();
@@ -907,6 +909,8 @@ mod tests {
             Some(vm.vm_as().unwrap().clone()),
             None,
             false,
+            Some(vm.vm_config().clone()),
+            vm.shared_info().clone(),
         );
 
         vm.device_manager_mut()