From 3cb71535c8378a4f95e5bbb464a8554c5c14024e Mon Sep 17 00:00:00 2001 From: Jiri Appl Date: Tue, 17 Dec 2024 21:27:36 -0800 Subject: [PATCH] Verity signing support --- src/tardev-snapshotter/Cargo.toml | 6 +- src/tardev-snapshotter/src/snapshotter.rs | 717 ++++++++++++++++------ src/tardev-snapshotter/verity/src/lib.rs | 5 +- 3 files changed, 537 insertions(+), 191 deletions(-) diff --git a/src/tardev-snapshotter/Cargo.toml b/src/tardev-snapshotter/Cargo.toml index a0f8aa70a180..f17b916fae56 100644 --- a/src/tardev-snapshotter/Cargo.toml +++ b/src/tardev-snapshotter/Cargo.toml @@ -25,4 +25,8 @@ devicemapper = "0.33.1" anyhow = "=1.0.58" zerocopy = "0.6.1" uuid = { version = "1.0", features = ["v4"] } -nix = "0.24.2" \ No newline at end of file +nix = "0.24.2" + +# YAML file serialization/deserialization. +serde = { version = "1.0.159", features = ["derive"] } +hex = { version = "0.4.3" } \ No newline at end of file diff --git a/src/tardev-snapshotter/src/snapshotter.rs b/src/tardev-snapshotter/src/snapshotter.rs index a941f9aa460c..6c6017037f65 100644 --- a/src/tardev-snapshotter/src/snapshotter.rs +++ b/src/tardev-snapshotter/src/snapshotter.rs @@ -1,31 +1,109 @@ +use base64::engine::general_purpose; use base64::prelude::{Engine, BASE64_STANDARD}; use containerd_client::{services::v1::ReadContentRequest, tonic::Request, with_namespace, Client}; use containerd_snapshots::{api, Info, Kind, Snapshotter, Usage}; -use log::{debug, info, trace}; +use log::{debug, error, info, trace}; use sha2::{Digest, Sha256}; +use std::os::unix::fs::PermissionsExt; use std::path::{Path, PathBuf}; -use std::{collections::HashMap, io, os::unix::ffi::OsStrExt}; +use std::{ + collections::HashMap, fs, fs::File, fs::OpenOptions, io, io::Read, io::Seek, + os::unix::ffi::OsStrExt, process::Command, +}; use tokio::io::{AsyncSeekExt, AsyncWriteExt}; use tokio::sync::RwLock; use tonic::Status; -use anyhow::{anyhow, Context, Result}; -use std::fs::{self, File, OpenOptions}; -use std::io::{Read, Seek}; -use zerocopy::AsBytes; -use std::process::Command; use uuid::Uuid; +//use nix::unistd::{chown, Gid, Uid}; +use anyhow::{anyhow, Context, Result}; use nix::mount::MsFlags; +use serde::{Deserialize, Serialize}; +use std::io::Write; +use std::process::Stdio; +use zerocopy::AsBytes; const ROOT_HASH_LABEL: &str = "io.katacontainers.dm-verity.root-hash"; +const SALT_LABEL: &str = "io.katacontainers.dm-verity.salt"; const TARGET_LAYER_DIGEST_LABEL: &str = "containerd.io/snapshot/cri.layer-digest"; +const TARGET_LAYER_SNAPSHOT_REF_LABEL: &str = "containerd.io/snapshot.ref"; + +#[derive(Serialize, Deserialize)] +struct ImageInfo { + name: String, + layers: Vec, +} + +#[derive(Serialize, Deserialize)] +struct LayerInfo { + diff_id: String, + root_hash: String, + signature: String, + salt: String, +} + +#[derive(PartialEq)] +struct LayerInfoInternal { + salt: String, + signature: Vec, +} struct Store { root: PathBuf, + signatures: Option>, // root_hash to info + salts: Option>>, // diff_id to salt } +const SIGNATURE_STORE: &str = "/var/lib/containerd/io.containerd.snapshotter.v1.tardev/signatures"; + impl Store { fn new(root: &Path) -> Self { - Self { root: root.into() } + Self { + root: root.into(), + signatures: None, + salts: None, + } + } + + fn lazy_read_signatures(&mut self) -> Result<()> { + if self.signatures == None { + info!("Loading signatures"); + self.read_signatures() + .context("Failed to read signatures")?; + } + + Ok(()) + } + + fn read_signatures(&mut self) -> Result<()> { + let paths = std::fs::read_dir(Path::new(SIGNATURE_STORE))?; + let mut signatures = HashMap::new(); + let mut salts = HashMap::new(); + for signatures_json_path in paths { + let signatures_json = std::fs::read_to_string(signatures_json_path?.path())?; + let image_info_list = serde_json::from_str::>(signatures_json.as_str())?; + for image_info in image_info_list { + for layer_info in image_info.layers { + signatures.insert( + layer_info.root_hash, + LayerInfoInternal { + signature: BASE64_STANDARD + .decode(layer_info.signature) + .context("Failed to decode signature")?, + salt: layer_info.salt.clone(), + }, + ); + salts.insert( + layer_info.diff_id, + hex::decode(layer_info.salt).context("Failed to decode salt")?, + ); + } + } + } + + self.signatures = Some(signatures); + self.salts = Some(salts); + + Ok(()) } /// Creates the name of the directory that containerd can use to extract a layer into. @@ -79,8 +157,11 @@ impl Store { /// Reads the information from storage for the given snapshot name. fn read_snapshot(&self, name: &str) -> Result { + info!("jiria 1 {}", name); let path = self.snapshot_path(name, false)?; + info!("jiria 2 {:#?}", path); let file = fs::File::open(path)?; + info!("jiria 3"); serde_json::from_reader(file).map_err(|_| Status::unknown("unable to read snapshot")) } @@ -102,25 +183,103 @@ impl Store { ..Info::default() }; let name = self.snapshot_path(&info.name, true)?; + info!("jiria name: {:#?}", name); // TODO: How to specify the file mode (e.g., 0600)? let file = OpenOptions::new().write(true).create_new(true).open(name)?; - serde_json::to_writer_pretty(file, &info) - .map_err(|_| Status::internal("unable to write snapshot")) + info!("here"); + let foo = serde_json::to_writer_pretty(file, &info) + .map_err(|_| Status::internal("unable to write snapshot")); + foo?; + info!("there"); + Ok(()) + } + + fn get_salt(&self, diff_id: &str) -> Result> { + self.salts + .as_ref() + .context("salts not loaded")? + .get(diff_id) + .context("missing signature for the requested layer") + .cloned() + } + + fn load_signature(&self, hash: &str) -> Result<(String, String)> { + let signature_name = format!("verity:{hash}"); + + let (signature, salt) = match &self.signatures { + None => panic!("signatures were not loaded"), + Some(signatures) => match signatures.get(hash) { + Some(layer_info) => (&layer_info.signature, layer_info.salt.clone()), + None => return Err(anyhow::anyhow!("missing signature for root hash {hash}")), + }, + }; + + // https://lkml.org/lkml/2019/7/17/762 + // https://www.kernel.org/doc/html/latest/admin-guide/device-mapper/verity.html + let keyctl = Command::new("keyctl") + .stdin(Stdio::piped()) + .arg("padd") + .arg("user") + .arg(&signature_name) + .arg("@u") + .spawn()?; + keyctl.stdin.as_ref().unwrap().write_all(&signature)?; // TODO do not unwrap + // write!(keyctl.stdin.as_ref().unwrap(), "{}", signature)?; // TODO do not unwrap + let output = keyctl.wait_with_output()?; + if !output.status.success() { + return Err(anyhow::anyhow!("failed to load signature")); + } + + Ok((signature_name, salt)) + } + + /// Creates a new snapshot for use. + /// + /// It checks that the parent chain exists and that all ancestors are committed and consist of + /// layers before writing the new snapshot. + fn prepare_snapshot_for_use( + &mut self, + kind: Kind, + key: String, + parent: String, + labels: HashMap, + ) -> Result, Status> { + info!( + "jiria: prepare_snapshot_for_use kind {:#?} key {} parent {} labels {:#?}", + kind, key, parent, labels + ); + if let Err(e) = self.lazy_read_signatures() { + error!("Failing to read signatures: {e}"); + return Err(Status::internal(format!("Failed read signatures: {:?}", e))); + } + let mounts = self.mounts_from_snapshot(&parent, false)?; + self.write_snapshot(kind, key, parent, labels)?; + Ok(mounts) } - // ported over from kata agent - fn prepare_dm_target(&self, path: &str, hash: &str) -> Result<(u64, u64, String, String)> { - info!(" started prepare_dm_target"); + // ported over from kata agent + // prepares a dm-verity target configuration by reading metadata from a file (block/loop device) + // and returning the parameters required to set up the device-mapper verity target + fn prepare_dm_target( + &self, + path: &str, + hash: &str, + signature_name: &str, + salt: &String, + ) -> Result<(u64, u64, String, String)> { + info!("prepare_dm_target for loop device"); let mut file = File::open(path)?; let size = file.seek(std::io::SeekFrom::End(0))?; if size < 4096 { - return Err(anyhow!(" Block device ({path}) is too small: {size}")); + return Err(anyhow!("loop device ({path}) is too small: {size}")); } - + + // last 4096 bytes of loop device is superblock file.seek(std::io::SeekFrom::End(-4096))?; let mut buf = [0u8; 4096]; file.read_exact(&mut buf)?; - + + // parse super block let mut sb = verity::SuperBlock::default(); sb.as_bytes_mut() .copy_from_slice(&buf[4096 - 512..][..std::mem::size_of::()]); @@ -130,164 +289,134 @@ impl Store { .data_block_count .get() .checked_mul(data_block_size) - .ok_or_else(|| anyhow!(" Invalid data size"))?; + .ok_or_else(|| anyhow!("Invalid data size"))?; if data_size > size { return Err(anyhow!( - " Data size ({data_size}) is greater than device size ({size}) for device {path}" + "Data size ({data_size}) is greater than device size ({size}) for device {path}" )); } - - // TODO: Store other parameters in super block: version, hash type, salt. - Ok(( - 0, - data_size / 512, - "verity".into(), - format!( - "1 {path} {path} {data_block_size} {hash_block_size - } {} {} sha256 {hash} 0000000000000000000000000000000000000000000000000000000000000000", + + // generate dm-verity table, use all zero salt + // TODO: Store other parameters in super block: version, hash type, + // salt. + let construction_parameters = format!( + "1 {path} {path} {data_block_size} {hash_block_size} {} {} sha256 {hash} {salt} 2 root_hash_sig_key_desc {signature_name}", data_size / data_block_size, - (data_size + hash_block_size - 1) / hash_block_size - ), - )) + (data_size + hash_block_size - 1) / hash_block_size, + ); + trace!("dm-verity construction params: {construction_parameters}"); + Ok((0, data_size / 512, "verity".into(), construction_parameters)) } - // created for runc support + // Creates dm-verity device for a given layer file fn create_dm_verity_device(&self, layer_path: &str, root_hash: &str) -> Result { - info!(" started create_dm_verity_device"); - let dm = devicemapper::DM::new()?; - let base_name = Path::new(layer_path) + let dm = devicemapper::DM::new()?; + let layer_name = Path::new(layer_path) .file_name() - .ok_or_else(|| anyhow!(" Unable to get file name from layer path"))? + .ok_or_else(|| anyhow!("Unable to get file name from layer path"))? .to_str() - .ok_or_else(|| anyhow!(" Unable to convert file name to UTF-8 string"))?; - - info!(" basename: {}", base_name); - let unique_id = Uuid::new_v4(); - let layer_name = format!("{}_{}", name_to_hash(base_name), unique_id); - info!(" layername: {}", layer_name); + .ok_or_else(|| anyhow!("Unable to convert file name to UTF-8 string"))?; + info!("create_dm_verity_device for layer: {}", layer_name); + let name = devicemapper::DmName::new(&layer_name)?; let opts = devicemapper::DmOptions::default().set_flags(devicemapper::DmFlags::DM_READONLY); - info!(" before dm.device_create"); + if let Err(e) = dm.device_create(name, None, opts) { - info!(" Failed to create Device Mapper device: {:?}", e); + info!("Failed to create Device Mapper device: {:?}", e); return Err(e.into()); } - - info!(" before devicemapper::DevId::Name"); let id = devicemapper::DevId::Name(name); let result = (|| { - // Prepare DM-Verity target - info!(" layer_path: {}", layer_path); - info!(" root_hash: {}", root_hash); - // Step 1: Set up loop device for the given layer_path - info!(" Setting up loop device for {}", layer_path); let setup_output = Command::new("losetup") .arg("-fP") .arg(layer_path) .output() - .expect(" Failed to execute losetup command to create loop device"); - + .expect("Failed to execute losetup command to create loop device"); + if !setup_output.status.success() { info!( - " Failed to set up loop device: {:?}", + "Failed to set up loop device: {:?}", String::from_utf8_lossy(&setup_output.stderr) ); return Err(anyhow::anyhow!( - " Failed to set up loop device: {:?}", + "Failed to set up loop device: {:?}", String::from_utf8_lossy(&setup_output.stderr) )); } - + info!("set up loop device"); + // Step 2: Find the loop device associated with the file let loop_output = Command::new("losetup") .arg("-a") .output() - .expect(" Failed to list loop devices"); - + .expect("Failed to list loop devices"); + let loop_output_str = String::from_utf8_lossy(&loop_output.stdout); let loop_device = loop_output_str .lines() .find(|line| line.contains(layer_path)) .and_then(|line| line.split(":").next()) - .ok_or_else(|| anyhow::anyhow!(" Could not find loop device for {}", layer_path))?; - - info!(" Using loop device: {}", loop_device); - + .ok_or_else(|| anyhow::anyhow!("Could not find loop device for {}", layer_path))?; + + info!("selected newly created loop device: {}", loop_device); + // Use the loop device path for DM-Verity let device_path = loop_device; - + + let (signature_name, salt) = self.load_signature(root_hash)?; + // Step 3: Prepare DM-Verity target - info!(" before prepare_dm_target"); - let target = self.prepare_dm_target(device_path, root_hash)?; - + let target = self.prepare_dm_target(device_path, root_hash, &signature_name, &salt)?; + // Step 4: Load the DM table for DM-Verity - info!(" target before table_load: {:?}", target); - info!(" before dm.table_load"); dm.table_load(&id, &[target], opts) - .context(" Unable to load DM-Verity table")?; - + .context("Unable to load DM-Verity table")?; + info!("loaded DM table for DM-Verity"); + // Step 5: Suspend the DM device to make it active - info!(" before dm.device_suspend"); dm.device_suspend(&id, opts) - .context(" Unable to suspend DM device")?; - + .context("Unable to suspend DM device")?; + info!("suspended DM device for activation"); + // Step 6: Return success, with the path of the DM-Verity device - info!(" create_dm_verity_device return success"); Ok(format!("/dev/mapper/{}", layer_name)) })(); - + // If there is an error, remove the DM device and clean up the loop device result.map_err(|e| { // Remove the DM device if it was created if let Err(remove_err) = dm.device_remove(&id, devicemapper::DmOptions::default()) { info!( - " Unable to remove DM device ({}): {:?}", - layer_name, - remove_err + "Unable to remove DM device ({}): {:?}", + layer_name, remove_err ); } - + // Clean up the loop device - info!(" Cleaning up loop device: {}", layer_path); + info!("Cleaning up loop device: {}", layer_path); let detach_output = Command::new("losetup") .arg("-d") .arg(layer_path) .output() - .expect(" Failed to execute losetup detach command"); - + .expect("Failed to execute losetup detach command"); + if !detach_output.status.success() { info!( - " Failed to detach loop device: {:?}", + "Failed to detach loop device: {:?}", String::from_utf8_lossy(&detach_output.stderr) ); } else { - info!(" Successfully detached loop device: {}", layer_path); + info!("Successfully detached loop device: {}", layer_path); } - - info!(" Error occurred during DM-Verity setup: {:?}", e); + + info!("Error occurred during DM-Verity setup: {:?}", e); e }) } - /// Creates a new snapshot for use. - /// - /// It checks that the parent chain exists and that all ancestors are committed and consist of - /// layers before writing the new snapshot. - fn prepare_snapshot_for_use( - &mut self, - kind: Kind, - key: String, - parent: String, - labels: HashMap, - ) -> Result, Status> { - let mounts = self.mounts_from_snapshot(&parent)?; - self.write_snapshot(kind, key, parent, labels)?; - Ok(mounts) - } - - /// Mounts a DM-Verity device to a specified path using integrated `baremount` logic. + /// Mounts a DM-Verity device to a specified path. fn mount_dm_verity_device( &self, source: &str, @@ -299,11 +428,9 @@ impl Store { if source.is_empty() { return Err(anyhow!("Source path for mounting cannot be empty.")); } - if target.is_empty() { return Err(anyhow!("Target path for mounting cannot be empty.")); } - if fstype.is_empty() { return Err(anyhow!("Filesystem type cannot be empty.")); } @@ -317,13 +444,6 @@ impl Store { .with_context(|| format!("Failed to create mount point: {}", target))?; } - //info!("mounting storage, mount-source:{}, mount-destination:{}, mount-fstype:{}, mount-options:{}", - // source_path, - // target_path, - // fstype, - // options, - //); - // Attempt the mount operation nix::mount::mount( Some(source_path), @@ -332,30 +452,45 @@ impl Store { flags, Some(options), ) - .map_err(|e| { - anyhow!( - "Failed to mount {} to {} with error: {}", - source, - target, - e - ) - })?; + .map_err(|e| anyhow!("Failed to mount {} to {} with error: {}", source, target, e))?; Ok(()) } - fn mounts_from_snapshot(&self, parent: &str) -> Result, Status> { + fn mounts_from_snapshot( + &self, + parent: &str, + do_mount: bool, + ) -> Result, Status> { + info!("jiria 3"); + const PREFIX: &str = "io.katacontainers.fs-opt"; + + // Get chain of layers. let mut next_parent = Some(parent.to_string()); - let mut lower_dirs = Vec::new(); - + let mut layers = Vec::new(); + let mut opts = vec![format!( + "{PREFIX}.layer-src-prefix={}", + self.root.join("layers").to_string_lossy() + )]; + let src_prefix = self.root.join("layers"); + let mut mounted_layers = Vec::new(); while let Some(p) = next_parent { - let info = self.read_snapshot(&p)?; + info!("jiria: in while {}", p); + let infor = self.read_snapshot(&p); + let info = match infor { + Ok(a) => a, + Err(b) => { + error!("iled to read snoasoht {}", b); + return Err(b); + } + }; + info!("jiria: in while2 {} {:#?}", p, info); if info.kind != Kind::Committed { return Err(Status::failed_precondition( "parent snapshot is not committed", )); } - + let root_hash = if let Some(rh) = info.labels.get(ROOT_HASH_LABEL) { rh } else { @@ -363,60 +498,224 @@ impl Store { "parent snapshot has no root hash stored", )); }; - - let layer_path = self.layer_path(&p); - let dm_verity_device = self.create_dm_verity_device(layer_path.to_str().unwrap(), root_hash) - .map_err(|_| Status::internal("unable to create DM-Verity device"))?; - - // Mount the DM-Verity device - let mount_path = format!("/var/lib/containerd/io.containerd.snapshotter.v1.tardev/mounts/{}", p); - if let Err(e) = self.mount_dm_verity_device( - &dm_verity_device, - &mount_path, - "tar", // Assume tarfs as the filesystem type; adjust as needed - "ro", - MsFlags::MS_RDONLY, - ) { - return Err(Status::internal(format!( - "Failed to mount DM-Verity device: {:?}", e - ))); + + let name = name_to_hash(&p); + let layer_info = format!( + "{name},tar,ro,{PREFIX}.block_device=file,{PREFIX}.is-layer,{PREFIX}.root-hash={root_hash}"); + info!( + "mounts_from_snapshot(): processing snapshots: {}, layername: {}", + &info.name, &name + ); + + if do_mount { + info!("mounts_from_snapshot(): performing tarfs mounting via dm-verity"); + // Extract layer information + let mut fields = layer_info.split(','); + let src = if let Some(p) = fields.next() { + if !p.is_empty() && p.as_bytes()[0] != b'/' { + src_prefix.join(Path::new(p)) + } else { + Path::new(p).to_path_buf() + } + } else { + return Err(Status::invalid_argument( + "Missing source path in layer info", + )); + }; + info!("src: {}", src.display()); + + let fs_type = fields.next().ok_or_else(|| { + Status::invalid_argument("Missing filesystem type in layer info") + })?; + info!("fs_type: {}", fs_type); + + let fs_opts = fields + .filter(|o| !o.starts_with("io.katacontainers.")) + .fold(String::new(), |a, b| { + if a.is_empty() { + b.into() + } else { + format!("{a},{b}") + } + }); + info!("fs_opts: {}", fs_opts); + + let mount_path = self.root.join("mounts").join(&name); + info!("mount_path: {}", mount_path.display()); + std::fs::create_dir_all(&mount_path)?; + + // Step 0: Check if the dm-verity device already exists + let dm_verity_device = format!("/dev/mapper/{}", name); + if Path::new(&dm_verity_device).exists() { + info!( + "dm-verity device already exists for layer {}: {}", + name, dm_verity_device + ); + } else { + // Step 1: Create a dm-verity device for the tarfs layer + let created_dm_verity_device = self + .create_dm_verity_device(src.to_str().unwrap(), root_hash) + .map_err(|e| { + Status::internal(format!( + "Failed to create dm-verity device for source {:?}: {:?}", + src, e + )) + })?; + info!( + "created dm-verity device for layer {}: {}", + name, created_dm_verity_device + ); + } + + // Step 2: Check if the mount path is already mounted + let mount_status = Command::new("mountpoint") + .arg("-q") + .arg(&mount_path) + .status()?; + if mount_status.success() { + info!( + "Mount path {:?} is already mounted, skipping mounting.", + mount_path + ); + } else { + // Mount the dm-verity device to the mount path + let flags = MsFlags::MS_RDONLY; // Read-only to ensure integrity + self.mount_dm_verity_device( + &dm_verity_device, + mount_path.to_str().unwrap(), + fs_type, + &fs_opts, + flags, + ) + .map_err(|e| { + Status::internal(format!( + "Failed to mount dm-verity device {} to {:?}: {:?}", + dm_verity_device, mount_path, e + )) + })?; + info!( + "mounted single layer dm-verity device {} to {:?}", + dm_verity_device, mount_path + ); + } + + mounted_layers.push(mount_path.clone()); } - - info!(" Mounted DM-Verity device at {}", mount_path); - lower_dirs.push(mount_path); - + + layers.push(name); + + opts.push(format!( + "{PREFIX}.layer={}", + BASE64_STANDARD.encode(layer_info.as_bytes()) + )); + next_parent = (!info.parent.is_empty()).then_some(info.parent); } - - let work_dir = format!( - "/var/lib/containerd/io.containerd.snapshotter.v1.tardev/work/{}", - parent - ); - let upper_dir = format!( - "/var/lib/containerd/io.containerd.snapshotter.v1.tardev/upper/{}", - parent - ); - - if let Err(e) = fs::create_dir_all(&work_dir) { - return Err(Status::internal(format!("Failed to create workdir: {:?}", e))); - } - if let Err(e) = fs::create_dir_all(&upper_dir) { - return Err(Status::internal(format!("Failed to create upperdir: {:?}", e))); + + if do_mount { + info!("mounts_from_snapshot(): perform overlay mounting"); + let overlay_target = self.root.join("overlay").join(Uuid::new_v4().to_string()); + let overlay_upper = overlay_target.join("upper"); + let overlay_work = overlay_target.join("work"); + + std::fs::create_dir_all(&overlay_upper)?; + std::fs::create_dir_all(&overlay_work)?; + std::fs::create_dir_all(&overlay_target)?; + fs::set_permissions(&overlay_upper, fs::Permissions::from_mode(0o755))?; + fs::set_permissions(&overlay_work, fs::Permissions::from_mode(0o755))?; + + // Prepare the list of lowerdirs from mounted dm-verity layers + let lowerdirs = mounted_layers + .iter() + .map(|layer| layer.to_string_lossy().into_owned()) + .collect::>() + .join(":"); + info!( + "Combining dm-verity layers into overlay lowerdirs: {}", + lowerdirs + ); + + // Perform an overlay mount + let status = Command::new("mount") + .arg("none") + .arg(&overlay_target) + .args(&[ + "-t", + "overlay", + "-o", + &format!( + "lowerdir={},upperdir={},workdir={}", + lowerdirs, + overlay_upper.to_string_lossy(), + overlay_work.to_string_lossy() + ), + ]) + .status()?; + if !status.success() { + return Err(Status::internal(format!( + "Failed to perform overlay mount at {:?}", + overlay_target + ))); + } + info!("Overlay mount completed at {:?}", overlay_target); + mounted_layers.clear(); + + // Clean up dm-verity and loop devices + /*for layer_path in &mounted_layers { + // Unmount dm-verity device + info!("unmounting dm-verity layer at {:?}", layer_path); + let status = Command::new("umount").arg(layer_path).status()?; + if !status.success() { + error!("Failed to unmount dm-verity layer at {:?}, status: {status}", layer_path); + } else { + info!("Successfully unmounted dm-verity layer at {:?}", layer_path); + // Remove dm-verity device + let dm_name = Path::new(layer_path) + .file_name() + .and_then(|f| f.to_str()) + .ok_or_else(|| tonic::Status::internal(format!("Invalid dm-verity device path: {:?}", layer_path)))?; + let status = Command::new("dmsetup").arg("remove").arg(dm_name).status()?; + if !status.success() { + error!("Failed to remove dm-verity device: {}", dm_name); + } else { + info!("Successfully removed dm-verity device: {}", dm_name); + } + + // Detach loop device + let status = Command::new("losetup").arg("-d").arg(layer_path).status()?; + if !status.success() { + error!("Failed to detach loop device for layer {:?}", layer_path); + } else { + info!("Successfully detached loop device for layer {:?}", layer_path); + } + } + }*/ + + // Return a mount structure for `runc` + let overlay_mount = api::types::Mount { + r#type: "bind".into(), + source: overlay_target.to_string_lossy().into(), + target: "/".into(), + options: vec!["bind".into(), "rw".into()], + }; + + info!( + "mounts_from_snapshot(): returning mount struct for runc: type={}, source={}, target={}, options={:?}", + overlay_mount.r#type, overlay_mount.source, overlay_mount.target, overlay_mount.options + ); + + return Ok(vec![overlay_mount]); } - - let options = vec![ - format!("lowerdir={}", lower_dirs.join(":")), - format!("upperdir={}", upper_dir), - format!("workdir={}", work_dir), - ]; - - info!(" Preparing overlayFS"); - Ok(vec![api::types::Mount { - r#type: "overlay".into(), - source: "overlay".into(), + + opts.push(format!("{PREFIX}.overlay-rw")); + opts.push(format!("lowerdir={}", layers.join(":"))); + + return Ok(vec![api::types::Mount { + r#type: "fuse3.kata-overlay".into(), + source: "/".into(), target: String::new(), - options, - }]) + options: opts, + }]); } } @@ -447,9 +746,13 @@ impl TarDevSnapshotter { ) -> Result, Status> { let extract_dir; { + info!("jiria store"); let mut store = self.store.write().await; + info!("jiria extract"); extract_dir = store.extract_dir_to_write(&key)?; + info!("jiria write {:#?}", extract_dir); store.write_snapshot(Kind::Active, key, parent, labels)?; + info!("jiria done"); } Ok(vec![api::types::Mount { r#type: "bind".into(), @@ -504,16 +807,23 @@ impl TarDevSnapshotter { key: String, parent: String, mut labels: HashMap, + salt: Vec, ) -> Result<(), Status> { + trace!("in prepaer_image_lyer"); let dir = self.store.read().await.staging_dir()?; + trace!("in prepaer_image_lyer2"); + + let salt2 = salt.clone(); { let Some(digest_str) = labels.get(TARGET_LAYER_DIGEST_LABEL) else { + error!("abort"); return Err(Status::invalid_argument( "missing target layer digest label", )); }; + trace!("in prepaer_image_lyer3"); let name = dir.path().join(name_to_hash(&key)); let mut gzname = name.clone(); gzname.set_extension("gz"); @@ -539,7 +849,7 @@ impl TarDevSnapshotter { tarindex::append_index(&mut file)?; trace!("Appending dm-verity tree to {:?}", &name); - let root_hash = verity::append_tree::(&mut file)?; + let root_hash = verity::append_tree::(&mut file, &salt)?; trace!("Root hash for {:?} is {:x}", &name, root_hash); Ok(root_hash) @@ -549,6 +859,10 @@ impl TarDevSnapshotter { // Store a label with the root hash so that we can recall it later when mounting. labels.insert(ROOT_HASH_LABEL.into(), format!("{:x}", root_hash)); + labels.insert( + SALT_LABEL.into(), + format!("{}", general_purpose::STANDARD.encode(&salt2)), + ); } // Move file to its final location and write the snapshot. @@ -624,8 +938,11 @@ impl Snapshotter for TarDevSnapshotter { options: Vec::new(), }]) } else { - info!("mounts(): snapshot: {}, ready to use, preparing itself and parents ", &info.name); - store.mounts_from_snapshot(&info.parent) + info!( + "mounts(): snapshot: {}, ready to use, preparing itself and parents ", + &info.name + ); + store.mounts_from_snapshot(&info.parent, true) } } @@ -672,21 +989,47 @@ impl Snapshotter for TarDevSnapshotter { ) -> Result<(), Self::Error> { trace!("commit({}, {}, {:?})", name, key, labels); - let info; - { - let store = self.store.write().await; - info = store.read_snapshot(&key)?; + let (salt, parent) = { + // Needs to be in the closure to release the lock + let mut store = self.store.write().await; + let info = store.read_snapshot(&key)?; if info.kind != Kind::Active { return Err(Status::failed_precondition("snapshot is not active")); } - } - if info.labels.get(TARGET_LAYER_DIGEST_LABEL).is_some() { - self.prepare_image_layer(name, info.parent, labels).await - } else { - Err(Status::unimplemented( - "no support for commiting arbitrary snapshots", - )) + if info.labels.get(TARGET_LAYER_DIGEST_LABEL).is_none() { + return Err(Status::unimplemented( + "no support for commiting arbitrary snapshots", + )); + } + + let diff_id_result = info.labels.get(TARGET_LAYER_SNAPSHOT_REF_LABEL); + let diff_id = match diff_id_result { + None => return Err(Status::failed_precondition("Missing expected label")), + Some(diff_id) => diff_id, + }; + + let result = store.lazy_read_signatures(); + match result { + Err(e) => { + return Err(Status::failed_precondition(format!( + "signature read failure: {e}" + ))) + } + Ok(()) => (), + }; + + trace!("Looking up salt for {diff_id}"); + let salt = store.get_salt(diff_id); + + (salt, info.parent) + }; + + match salt { + Err(e) => Err(Status::failed_precondition(format!( + "signature missing: {e}" + ))), + Ok(salt) => self.prepare_image_layer(name, parent, labels, salt).await, } } diff --git a/src/tardev-snapshotter/verity/src/lib.rs b/src/tardev-snapshotter/verity/src/lib.rs index 30c59e87cfda..8bc174589636 100644 --- a/src/tardev-snapshotter/verity/src/lib.rs +++ b/src/tardev-snapshotter/verity/src/lib.rs @@ -1,5 +1,5 @@ use generic_array::{typenum::Unsigned, GenericArray}; -use sha2::{digest::OutputSizeUser, Digest}; +use sha2::Digest; use std::fs::File; use std::io::{self, Read, Seek, SeekFrom, Write}; use zerocopy::byteorder::{LE, U32, U64}; @@ -228,11 +228,10 @@ pub fn write_to(f: &mut File) -> impl FnMut(&mut File, &[u8], u64) -> io::Result pub fn append_tree( file: &mut File, + salt: &[u8], ) -> io::Result> { let file_size = file.seek(io::SeekFrom::End(0))?; file.rewind()?; - let mut salt = Vec::new(); - salt.resize(::OutputSize::USIZE, 0); let verity = Verity::::new(file_size, 4096, 4096, &salt, file_size)?; traverse_file(file, 0, true, verity, &mut |f, data, offset| { f.seek(SeekFrom::Start(offset))?;