Automation for CEPH-83584085: Live migration of raw data format

Signed-off-by: Aarti Sharma <[email protected]>

Author: aarsharm

ceph/rbd/workflows/migration.py

@@ -1,6 +1,8 @@
 import json
 import tempfile
 
+from ceph.rbd.utils import random_string
+from ceph.rbd.workflows.rbd import create_single_pool_and_images
 from cli.rbd.rbd import Rbd
 from utility.log import Log
 
@@ -73,3 +75,104 @@ def prepare_migration_source_spec(
     spec_file.flush()
 
     return temp_file.name
+
+
+def run_prepare_execute_commit(rbd, pool, image, **kw):
+    """
+    Function to carry out the following:
+      - Create Target/destination pool for migration
+      - Migration prepare
+      - Migration Execute
+      - Migration commit
+    Args:
+        kw: rbd object, pool, image, test data
+    Returns:
+        int: The return value. 0 for success, 1 otherwise
+
+    """
+    # Create Target Pool/ Destination Pool for migration
+    is_ec_pool = True if "ec" in kw[pool]["pool_type"] else False
+    config = kw.get("config", {})
+    target_pool = "target_pool_" + random_string(len=3)
+    target_pool_config = {}
+    if is_ec_pool:
+        data_pool_target = "data_pool_new_" + random_string(len=3)
+        target_pool_config["data_pool"] = data_pool_target
+    rc = create_single_pool_and_images(
+        config=config,
+        pool=target_pool,
+        pool_config=target_pool_config,
+        client=kw["client"],
+        cluster="ceph",
+        rbd=rbd,
+        ceph_version=int(config.get("rhbuild")[0]),
+        is_ec_pool=is_ec_pool,
+        is_secondary=False,
+        do_not_create_image=True,
+    )
+    if rc:
+        log.error(f"Creation of target pool {target_pool} failed")
+        return rc
+
+    # Adding the new pool details to config so that they are handled in cleanup
+    if kw[pool]["pool_type"] == "rep_pool_config":
+        kw["config"]["rep_pool_config"][target_pool] = {}
+    elif kw[pool]["pool_type"] == "ec_pool_config":
+        kw["config"]["ec_pool_config"][target_pool] = {"data_pool": data_pool_target}
+
+    # Prepare Migration
+    target_image = "target_image_" + random_string(len=3)
+    rbd.migration.prepare(
+        source_spec=kw[pool]["spec"],
+        dest_spec=f"{target_pool}/{target_image}",
+        client_node=kw["client"],
+    )
+    kw[pool].update({"target_pool": target_pool})
+    kw[pool].update({"target_image": target_image})
+
+    # Verify prepare migration status
+    if verify_migration_state(
+        action="prepare",
+        image_spec=f"{target_pool}/{target_image}",
+        **kw,
+    ):
+        log.error("Failed to prepare migration")
+        return 1
+    else:
+        log.info("Migration prepare status verfied successfully")
+
+    # execute migration
+    rbd.migration.action(
+        action="execute",
+        dest_spec=f"{target_pool}/{target_image}",
+        client_node=kw["client"],
+    )
+
+    # verify execute migration status
+    if verify_migration_state(
+        action="execute",
+        image_spec=f"{target_pool}/{target_image}",
+        **kw,
+    ):
+        log.error("Failed to execute migration")
+        return 1
+    else:
+        log.info("Migration executed successfully")
+
+    # commit migration
+    rbd.migration.action(
+        action="commit",
+        dest_spec=f"{target_pool}/{target_image}",
+        client_node=kw["client"],
+    )
+
+    # verify commit migration status
+    if verify_migration_state(
+        action="commit",
+        image_spec=f"{target_pool}/{target_image}",
+        **kw,
+    ):
+        log.error("Failed to commit migration")
+        return 1
+    else:
+        log.info("Migration committed successfully")

cli/rbd/rbd.py

@@ -390,3 +390,69 @@ def image_usage(self, **kw):
         cmd = f"{self.base_cmd} du {image_spec} {build_cmd_from_args(**kw_copy)}"
 
         return self.execute_as_sudo(cmd=cmd)
+
+    def lock_ls(self, **kw):
+        """
+        This method is used to get list of locked rbd images
+        Args:
+          kw(dict): Key/value pairs that needs to be provided to the installer
+          Example:
+            Supported keys:
+                image-spec(str) : [<pool-name>/[<namespace>/]]<image-name>
+        """
+        kw_copy = deepcopy(kw)
+        image_spec = kw_copy.pop("image-spec", "")
+        cmd = f"{self.base_cmd} lock ls {image_spec} {build_cmd_from_args(**kw_copy)}"
+
+        return self.execute_as_sudo(cmd=cmd)
+
+    def lock_rm(self, **kw):
+        """
+        This method is used to remove lock from rbd image
+        Args:
+          kw(dict): Key/value pairs that needs to be provided to the installer
+          Example:
+            Supported keys:
+                lock-spec(str) : locker_id locker_name
+        """
+        kw_copy = deepcopy(kw)
+        lock_spec = kw_copy.pop("lock-spec", "")
+        cmd = f"{self.base_cmd} lock rm {build_cmd_from_args(**kw_copy)} {lock_spec}"
+
+        return self.execute_as_sudo(cmd=cmd)
+
+    def blocklist_ls(self):
+        """
+        This method is used to list blocklisted clients
+        """
+        cmd = "ceph osd blocklist ls"
+
+        return self.execute_as_sudo(cmd=cmd)
+
+    def blocklist_rm(self, **kw):
+        """
+        This method is used to remove specific ip from blocklisted entries
+        Args:
+          kw(dict): Key/value pairs that needs to be provided to the installer
+          Example:
+            Supported keys:
+                ip_spec(str) : blocklist ip in form "10.0.67.42:0/3439165164"
+        """
+        kw_copy = deepcopy(kw)
+        blocklist_ip_spec = kw_copy.pop("ip-spec", "")
+        cmd = f"ceph osd blocklist rm {blocklist_ip_spec}"
+
+        return self.execute_as_sudo(cmd=cmd)
+
+    def get_raw_file_size(self, raw_file_path):
+        """
+        This method is used get size of raw file format
+        Args:
+          kw(dict): Key/value pairs that needs to be provided to the installer
+          Example:
+            Supported keys:
+                raw_file_path(str) : absolute path of the file
+        """
+        cmd = f"ls -lh {raw_file_path}"
+
+        return self.execute(cmd=cmd)

suites/squid/rbd/tier-3_rbd_migration.yaml

@@ -178,3 +178,20 @@ tests:
             Rollback after Failed Migration using migration
             abort in single ceph cluster
       polarion-id: CEPH-83584090
+
+  - test:
+      desc: Live migration of image with external data source as RAW data format in single ceph cluster
+      config:
+        rep_pool_config:
+          num_pools: 1
+          do_not_create_image: true
+        ec_pool_config:
+          num_pools: 1
+          do_not_create_image: true
+        fio:
+          size: 1G
+          fs: ext4
+          io: true
+      module: test_live_migration_with_raw_data_format.py
+      name: Live migration of raw data format
+      polarion-id: CEPH-83584085