Add a clone option to downstairs create (#1249)

When creating a region, add the ability to optionally populate that
region from an existing and running downstairs using the clone ability.

This makes use of the already written clone subcommand functions, but
calls them directly after creating the region.

The current `clone` subcommand still exists and we can still use it
directly if so desired.

Changed some clap `name` to `value_name` as that is the proper arg for
how we are using them.

---------

Co-authored-by: Alan Hanson <[email protected]>

Author: leftwo

downstairs/src/main.rs

@@ -47,7 +47,7 @@ enum Args {
     /// the region here will be replaced.
     Clone {
         /// Directory where the region is located.
-        #[clap(short, long, name = "DIRECTORY", action)]
+        #[clap(short, long, value_name = "DIRECTORY", action)]
         data: PathBuf,
 
         /// Source IP:Port where the extent files will come from.
@@ -58,26 +58,50 @@ enum Args {
         trace_endpoint: Option<String>,
     },
     Create {
+        /// Block size.
         #[clap(long, default_value = "512", action)]
         block_size: u64,
 
-        #[clap(short, long, name = "DIRECTORY", action)]
+        /// Directory where the region files we be located.
+        #[clap(short, long, value_name = "DIRECTORY", action)]
         data: PathBuf,
 
+        /// Number of blocks per extent file.
         #[clap(long, default_value = "100", action)]
         extent_size: u64,
 
+        /// Number of extent files.
         #[clap(long, default_value = "15", action)]
         extent_count: u64,
 
-        #[clap(short, long, name = "FILE", action)]
+        /// Import data for the extent from this file.
+        #[clap(
+            short,
+            long,
+            value_name = "FILE",
+            action,
+            conflicts_with = "clone_source"
+        )]
         import_path: Option<PathBuf>,
 
-        #[clap(short, long, name = "UUID", action)]
+        /// UUID for the region.
+        #[clap(short, long, value_name = "UUID", action)]
         uuid: Uuid,
 
+        /// Will the region be encrypted.
         #[clap(long, action)]
         encrypted: bool,
+
+        /// Clone another downstairs after creating.
+        ///
+        /// IP:Port where the extent files will come from.
+        #[clap(
+            long,
+            value_name = "SOURCE",
+            action,
+            conflicts_with = "import_path"
+        )]
+        clone_source: Option<SocketAddr>,
     },
     /*
      * Dump region information.
@@ -90,7 +114,7 @@ enum Args {
         /*
          * Directories containing a region.
          */
-        #[clap(short, long, name = "DIRECTORY", action)]
+        #[clap(short, long, value_name = "DIRECTORY", action)]
         data: Vec<PathBuf>,
 
         /*
@@ -119,16 +143,16 @@ enum Args {
         /*
          * Number of blocks to export.
          */
-        #[clap(long, default_value = "0", name = "COUNT", action)]
+        #[clap(long, default_value = "0", value_name = "COUNT", action)]
         count: u64,
 
-        #[clap(short, long, name = "DIRECTORY", action)]
+        #[clap(short, long, value_name = "DIRECTORY", action)]
         data: PathBuf,
 
-        #[clap(short, long, name = "OUT_FILE", action)]
+        #[clap(short, long, value_name = "OUT_FILE", action)]
         export_path: PathBuf,
 
-        #[clap(short, long, default_value = "0", name = "SKIP", action)]
+        #[clap(short, long, default_value = "0", value_name = "SKIP", action)]
         skip: u64,
     },
     Run {
@@ -137,13 +161,13 @@ enum Args {
             short,
             long,
             default_value = "0.0.0.0",
-            name = "ADDRESS",
+            value_name = "ADDRESS",
             action
         )]
         address: IpAddr,
 
         /// Directory where the region is located.
-        #[clap(short, long, name = "DIRECTORY", action)]
+        #[clap(short, long, value_name = "DIRECTORY", action)]
         data: PathBuf,
 
         /// Test option, makes the search for new work sleep and sometimes
@@ -156,7 +180,7 @@ enum Args {
          * oximeter server, the downstairs will publish stats.
          */
         /// Use this address:port to send stats to an Oximeter server.
-        #[clap(long, name = "OXIMETER_ADDRESS:PORT", action)]
+        #[clap(long, value_name = "OXIMETER_ADDRESS:PORT", action)]
         oximeter: Option<SocketAddr>,
 
         /// Listen on this port for the upstairs to connect to us.
@@ -204,7 +228,7 @@ enum Args {
         #[clap(long, default_value_t = 512)]
         block_size: u64,
 
-        #[clap(short, long, name = "DIRECTORY")]
+        #[clap(short, long, value_name = "DIRECTORY")]
         data: PathBuf,
 
         #[clap(long, default_value_t = 100)]
@@ -287,10 +311,11 @@ async fn main() -> Result<()> {
             import_path,
             uuid,
             encrypted,
+            clone_source,
         } => {
             let mut region = create_region(
                 block_size,
-                data,
+                data.clone(),
                 extent_size,
                 extent_count,
                 uuid,
@@ -306,6 +331,13 @@ async fn main() -> Result<()> {
                  * new data and inital flush number is written to disk.
                  */
                 region.region_flush(1, 0, &None, JobId(0), None).await?;
+            } else if let Some(ref clone_source) = clone_source {
+                info!(log, "Cloning from: {:?}", clone_source);
+                let d = Downstairs::new_builder(&data, false)
+                    .set_logger(log.clone())
+                    .build()
+                    .await?;
+                clone_region(d, *clone_source).await?
             }
 
             info!(log, "UUID: {:?}", region.def().uuid());
@@ -315,6 +347,7 @@ async fn main() -> Result<()> {
                 region.def().extent_size().value,
                 region.def().extent_count(),
             );
+            // Now, clone it!
             Ok(())
         }
         Args::Dump {

integration_tests/src/lib.rs

@@ -38,6 +38,7 @@ mod test {
     }
 
     impl TestDownstairs {
+        #[allow(clippy::too_many_arguments)]
         pub async fn new(
             address: IpAddr,
             encrypted: bool,
@@ -46,6 +47,7 @@ mod test {
             extent_count: u32,
             problematic: bool,
             backend: Backend,
+            clone_source: Option<SocketAddr>,
         ) -> Result<Self> {
             let tempdir = tempfile::Builder::new()
                 .prefix(&"downstairs-")
@@ -74,6 +76,10 @@ mod test {
                 .build()
                 .await?;
 
+            if let Some(ref clone_source) = clone_source {
+                clone_region(downstairs.clone(), *clone_source).await?
+            }
+
             let _join_handle = start_downstairs(
                 downstairs.clone(),
                 address,
@@ -252,6 +258,7 @@ mod test {
                 extent_count,
                 problematic,
                 backend,
+                None,
             )
             .await?;
             let downstairs2 = TestDownstairs::new(
@@ -262,6 +269,7 @@ mod test {
                 extent_count,
                 problematic,
                 backend,
+                None,
             )
             .await?;
             let downstairs3 = TestDownstairs::new(
@@ -272,6 +280,7 @@ mod test {
                 extent_count,
                 problematic,
                 backend,
+                None,
             )
             .await?;
 
@@ -360,6 +369,7 @@ mod test {
                 self.extent_count,
                 false,
                 Backend::RawFile,
+                None,
             )
             .await
         }
@@ -2657,6 +2667,9 @@ mod test {
         // Clone a read only downstairs to the new downstairs
         // Make a new volume with two old and the one new downstairs and verify
         // the original data we wrote.
+        // Create a new downstairs directly from the clone.
+        // Make a new volume with two old and the one new downstairs and verify
+        // the original data we wrote.
         const BLOCK_SIZE: usize = 512;
 
         // boot three downstairs, write some data to them, then change to
@@ -2708,6 +2721,7 @@ mod test {
             2,
             false,
             Backend::RawFile,
+            None,
         )
         .await
         .unwrap();
@@ -2751,6 +2765,56 @@ mod test {
 
         assert_eq!(&buffer[BLOCK_SIZE..], &random_buffer[BLOCK_SIZE..]);
 
+        // Clone an original downstairs directly to our new downstairs
+        let clone_source =
+            test_downstairs_set.downstairs1.repair_address().await;
+        let mut new_ds = TestDownstairs::new(
+            "127.0.0.1".parse().unwrap(),
+            true,
+            true,
+            5,
+            2,
+            false,
+            Backend::RawFile,
+            Some(clone_source),
+        )
+        .await
+        .unwrap();
+
+        // Restart the new downstairs read only.
+        new_ds.reboot_read_only().await.unwrap();
+        let new_target = new_ds.address().await;
+
+        // Take our original opts, and replace a target with the downstairs
+        // we just cloned.
+        let mut new_opts = test_downstairs_set.opts();
+        new_opts.target[0] = new_target;
+
+        let mut volume = Volume::new(BLOCK_SIZE as u64, csl());
+        volume
+            .add_subvolume_create_guest(
+                new_opts,
+                volume::RegionExtentInfo {
+                    block_size: BLOCK_SIZE as u64,
+                    blocks_per_extent: test_downstairs_set.blocks_per_extent(),
+                    extent_count: test_downstairs_set.extent_count(),
+                },
+                3,
+                None,
+            )
+            .await?;
+
+        volume.activate().await?;
+
+        // Verify our volume still has the random data we wrote.
+        let volume_size = volume.total_size().await? as usize;
+        let mut buffer = Buffer::new(volume_size / BLOCK_SIZE, BLOCK_SIZE);
+        volume
+            .read(Block::new(0, BLOCK_SIZE.trailing_zeros()), &mut buffer)
+            .await?;
+
+        assert_eq!(&buffer[BLOCK_SIZE..], &random_buffer[BLOCK_SIZE..]);
+
         Ok(())
     }
 
@@ -2823,6 +2887,7 @@ mod test {
             2,
             false,
             Backend::RawFile,
+            None,
         )
         .await
         .unwrap();
@@ -2884,6 +2949,7 @@ mod test {
             2,
             false,
             Backend::RawFile,
+            None,
         )
         .await
         .unwrap();
@@ -2918,6 +2984,7 @@ mod test {
             2,
             false,
             Backend::RawFile,
+            None,
         )
         .await
         .unwrap();
@@ -2951,6 +3018,7 @@ mod test {
             3,
             false,
             Backend::RawFile,
+            None,
         )
         .await
         .unwrap();
@@ -2966,6 +3034,7 @@ mod test {
             2, // <- Different than above
             false,
             Backend::RawFile,
+            None,
         )
         .await
         .unwrap();
@@ -2987,6 +3056,7 @@ mod test {
             2,
             false,
             Backend::RawFile,
+            None,
         )
         .await
         .unwrap();
@@ -3002,6 +3072,7 @@ mod test {
             2,
             false,
             Backend::RawFile,
+            None,
         )
         .await
         .unwrap();
@@ -3023,6 +3094,7 @@ mod test {
             2,
             false,
             Backend::RawFile,
+            None,
         )
         .await
         .unwrap();
@@ -3037,6 +3109,7 @@ mod test {
             2,
             false,
             Backend::RawFile,
+            None,
         )
         .await
         .unwrap();
@@ -3058,6 +3131,7 @@ mod test {
             2,
             false,
             Backend::RawFile,
+            None,
         )
         .await
         .unwrap();
@@ -3072,6 +3146,7 @@ mod test {
             2,
             false,
             Backend::RawFile,
+            None,
         )
         .await
         .unwrap();
@@ -3244,6 +3319,7 @@ mod test {
             2,
             false,
             Backend::RawFile,
+            None,
         )
         .await
         .unwrap();