Merge pull request #4522 from pdhiran/3az-update

Addition of site, host & OSD down scenarios for 3AZ clusters

Author: mergify[bot]

ceph/rados/core_workflows.py

@@ -602,7 +602,10 @@ def check_health_warning(self, warning: str):
         status_report = self.run_ceph_command(cmd="ceph report", client_exec=True)
         ceph_health_status = list(status_report["health"]["checks"].keys())
         if warning in ceph_health_status:
-            log.info(f"warning: {warning}  present on the cluster")
+            log.info(
+                f"warning: {warning}  present on the cluster"
+                f"all Generated warnings : {ceph_health_status}"
+            )
             log.info(
                 f"Warning: {warning} generated on the cluster : {ceph_health_status}"
             )
@@ -725,9 +728,13 @@ def collect_osd_daemon_ids(self, osd_node) -> list:
         """
         The method is used to collect the various OSD daemons present on a particular node
         :param osd_node: name of the OSD node on which osd daemon details are collected (ceph.ceph.CephNode): ceph node
+                        or host-name in string can also be sent
         :return: list of OSD ID's
         """
-        cmd = f"sudo ceph osd ls-tree {osd_node.hostname}"
+        if isinstance(osd_node, str):
+            cmd = f"sudo ceph osd ls-tree {osd_node}"
+        else:
+            cmd = f"sudo ceph osd ls-tree {osd_node.hostname}"
         return self.run_ceph_command(cmd=cmd)
 
     def enable_balancer(self, **kwargs) -> bool:
@@ -2673,7 +2680,7 @@ def list_orch_services(self, service_type=None, export=None) -> list:
         if orch_ls_op:
             return [service["service_name"] for service in orch_ls_op]
 
-    def check_host_status(self, hostname, status: str = None) -> bool:
+    def check_host_status(self, hostname, status: str) -> bool:
         """
         Checks the status of host(offline or online) using
         ceph orch host ls and return boolean
@@ -2685,13 +2692,13 @@ def check_host_status(self, hostname, status: str = None) -> bool:
         """
         host_cmd = f"ceph orch host ls --host_pattern {hostname}"
         out = self.run_ceph_command(cmd=host_cmd, client_exec=True)
-        host_status = out[0]["status"].lower().strip()
-        log.info(f"Status of the host is {host_status}")
-        if status:
+        if out:
+            host_status = out[0]["status"].lower().strip()
+            log.info("Status of the host : %s is %s", hostname, host_status)
             return True if status.lower() == host_status else False
-        elif "offline" in host_status:
+        else:
+            log.info("Host : %s is not part of the cluster", hostname)
             return False
-        return True
 
     def run_concurrent_io(self, pool_name: str, obj_name: str, obj_size: int):
         """
@@ -4524,7 +4531,7 @@ def get_osd_list(self, status: str) -> list:
                         osd_dict[key].append(entry["id"])
             log.info(f"List of {key} OSDs: {osd_dict[key]}")
 
-        return osd_dict[status]
+        return osd_dict[status.lower()]
 
     def get_osd_details(self, osd_id: int):
         """

ceph/rados/pool_workflows.py

@@ -231,7 +231,7 @@ def do_rados_put(
             put_cmd = (
                 f"rados put -p {pool} obj{i} {infile}"
                 if obj_name is None
-                else f"rados put -p {pool} {obj_name} {infile}"
+                else f"rados put -p {pool} {obj_name}-{i} {infile}"
             )
             if offset:
                 put_cmd = f"{put_cmd} --offset {offset}"

suites/squid/rados/tier-3_rados_test-3-AZ-Cluster.yaml

@@ -209,6 +209,30 @@ tests:
             - test-pool-12
       desc: Enable 3 AZ stretch mode on new pools of the cluster post upgrade
 
+  - test:
+      name: Site-down Scenarios
+      module: test_stretch_n_az_site_down_scenarios.py
+      polarion-id: CEPH-83609869
+      config:
+        pool_name: test_stretch_pool6
+        stretch_bucket: datacenter
+        rule_name: 3az_rule
+      desc: Test stretch site down scenarios in 3 AZ cluster
+
+  - test:
+      name: Maintenance mode Scenarios
+      module: test_stretch_n_az_site_down_scenarios.py
+      polarion-id: CEPH-83609871
+      config:
+        pool_name: test_stretch_pool7
+        stretch_bucket: datacenter
+        rule_name: 3az_rule
+        scenarios_to_run:
+          - scenario-8
+          - scenario-9
+          - scenario-10
+      desc: Test stretch site maintenance mode scenarios in 3 AZ cluster
+
   - test:
       name: Netsplit Scenarios data-data sites
       module: test_stretch_n-az_netsplit_scenarios.py

tests/rados/test_pool_osd_recovery.py

@@ -120,7 +120,9 @@ def run(ceph_cluster, **kw) -> int:
             while datetime.datetime.now() < timeout_time:
                 try:
                     for node in osd_nodes:
-                        assert rados_obj.check_host_status(hostname=node.hostname)
+                        assert rados_obj.check_host_status(
+                            hostname=node.hostname, status="offline"
+                        )
                     log.info("Rebooted hosts are up")
                     break
                 except AssertionError: