add UMF benchmarks for fragmentation measurements

redesign of the base class for UMF benchmarks
run each benchmark separately
benchmarks using glibc are excluded from
the fragmentation benchmark

Author: EuphoricThinking

devops/scripts/benchmarks/benches/umf.py

@@ -13,6 +13,7 @@
 import os
 import csv
 import io
+import re
 
 
 def isUMFAvailable():
@@ -40,96 +41,44 @@ def benchmarks(self) -> list[Benchmark]:
             GBenchUmfProxy(self),
             GBenchJemalloc(self),
             GBenchTbbProxy(self),
+            GBenchMemoryOverhead(self),
         ]
 
         return benches
 
 
-class ComputeUMFBenchmark(Benchmark):
-    def __init__(self, bench, name):
+class GBench(Benchmark):
+    def __init__(self, bench):
         super().__init__(bench.directory, bench)
 
         self.bench = bench
-        self.bench_name = name
+        self.bench_name = "umf-benchmark"
         self.oneapi = get_oneapi()
+        self.umf_lib = options.umf + "lib"
 
-        self.col_name = None
-        self.col_iterations = None
-        self.col_real_time = None
-        self.col_cpu_time = None
-        self.col_time_unit = None
-
-        self.col_statistics_time = None
-
-    def bin_args(self) -> list[str]:
-        return []
-
-    def extra_env_vars(self) -> dict:
-        return {}
-
-    def setup(self):
-        if not isUMFAvailable():
-            print("UMF prefix path not provided")
-            return
-
-        self.benchmark_bin = os.path.join(options.umf, "benchmark", self.bench_name)
-
-    def get_tags(self):
-        return ["UMF", "allocation", "latency", "micro"]
-
-    def run(self, env_vars) -> list[Result]:
-        command = [
-            f"{self.benchmark_bin}",
-        ]
-
-        command += self.bin_args()
-        env_vars.update(self.extra_env_vars())
-
-        result = self.run_bench(
-            command, env_vars, add_sycl=False, ld_library=[self.oneapi.tbb_lib()]
-        )
-        parsed = self.parse_output(result)
-        results = []
-        for r in parsed:
-            (config, pool, mean) = r
-            label = f"{config} {pool}"
-            results.append(
-                Result(
-                    label=label,
-                    value=mean,
-                    command=command,
-                    env=env_vars,
-                    stdout=result,
-                    unit="ns",
-                    explicit_group=config,
-                )
-            )
-        return results
-
-    # Implementation with self.col_* indices could lead to the division by None
-    def get_mean(self, datarow):
-        raise NotImplementedError()
-
-    def teardown(self):
-        return
-
-
-class GBench(ComputeUMFBenchmark):
-    def __init__(self, bench):
-        super().__init__(bench, "umf-benchmark")
+        self.num_cols_with_memory = 13
 
         self.col_name = 0
         self.col_iterations = 1
         self.col_real_time = 2
         self.col_cpu_time = 3
         self.col_time_unit = 4
+        self.col_memory_overhead = 11
 
         self.idx_pool = 0
         self.idx_config = 1
         self.name_separator = "/"
 
         self.col_statistics_time = self.col_real_time
 
+        self.is_preloaded = False
+        self.is_memory_overhead_checked = False
+
+        self.lib_to_be_replaced = None
+
+    def is_memory_statistics_included(self, data_row):
+        return len(data_row) == self.num_cols_with_memory
+
     def name(self):
         return self.bench_name
 
@@ -160,44 +109,73 @@ def get_pool_and_config(self, full_name):
     def get_mean(self, datarow):
         return float(datarow[self.col_statistics_time])
 
-    def parse_output(self, output):
-        csv_file = io.StringIO(output)
-        reader = csv.reader(csv_file)
+    def get_memory_overhead(self, datarow):
+        return float(datarow[self.col_memory_overhead])
 
-        data_row = next(reader, None)
-        if data_row is None:
-            raise ValueError("Benchmark output does not contain data.")
+    def extra_env_vars(self) -> dict:
+        return {}
 
-        results = []
-        for row in reader:
-            try:
-                full_name = row[self.col_name]
-                pool, config = self.get_pool_and_config(full_name)
-                mean = self.get_mean(row)
-                results.append((config, pool, mean))
-            except KeyError as e:
-                raise ValueError(f"Error parsing output: {e}")
+    def get_tags(self):
+        return ["UMF", "allocation", "latency", "micro"]
 
-        return results
+    def setup(self):
+        if not isUMFAvailable():
+            print("UMF prefix path not provided")
+            return
 
+        self.benchmark_bin = os.path.join(options.umf, "benchmark", self.bench_name)
 
-class GBenchPreloaded(GBench):
-    def __init__(self, bench, lib_to_be_replaced, replacing_lib):
-        super().__init__(bench)
+    def get_names_of_benchmarks_to_be_run(self, command, env_vars):
+        list_all_command = command + ["--benchmark_list_tests"]
 
-        self.lib_to_be_replaced = lib_to_be_replaced
-        self.replacing_lib = replacing_lib
+        if self.is_preloaded:
+            list_all_command += ["--benchmark_filter=" + self.lib_to_be_replaced]
 
-    def bin_args(self):
-        full_args = super().bin_args()
-        full_args.append(f"--benchmark_filter={self.lib_to_be_replaced}")
+        all_names = self.run_bench(
+            list_all_command, env_vars, add_sycl=False, ld_library=[self.umf_lib]
+        ).splitlines()
 
-        return full_args
+        if self.is_memory_overhead_checked:
+            all_names = [
+                name for name in all_names if re.search("^glibc", name) is None
+            ]
 
-    def get_preloaded_name(self, pool_name) -> str:
-        new_pool_name = pool_name.replace(self.lib_to_be_replaced, self.replacing_lib)
+        return all_names
 
-        return new_pool_name
+    def run(self, env_vars) -> list[Result]:
+        command = [f"{self.benchmark_bin}"]
+
+        all_names = self.get_names_of_benchmarks_to_be_run(command, env_vars)
+
+        command += self.bin_args()
+        env_vars.update(self.extra_env_vars())
+
+        results = []
+
+        for name in all_names:
+            specific_benchmark = command + ["--benchmark_filter=^" + name + "$"]
+
+            result = self.run_bench(
+                specific_benchmark, env_vars, add_sycl=False, ld_library=[self.umf_lib]
+            )
+
+            parsed = self.parse_output(result)
+            for r in parsed:
+                (config, pool, mean) = r
+                label = f"{config} {pool}"
+                results.append(
+                    Result(
+                        label=label,
+                        value=mean,
+                        command=command,
+                        env=env_vars,
+                        stdout=result,
+                        unit=self.unit(),
+                        explicit_group=config,
+                    )
+                )
+
+        return results
 
     def parse_output(self, output):
         csv_file = io.StringIO(output)
@@ -208,20 +186,56 @@ def parse_output(self, output):
             raise ValueError("Benchmark output does not contain data.")
 
         results = []
+
         for row in reader:
             try:
                 full_name = row[self.col_name]
                 pool, config = self.get_pool_and_config(full_name)
-                mean = self.get_mean(row)
-                updated_pool = self.get_preloaded_name(pool)
-                updated_config = self.get_preloaded_name(config)
+                statistics = None
+                is_row_matched_to_statistics_type = False
+
+                if not self.is_memory_overhead_checked:
+                    statistics = self.get_mean(row)
+
+                    is_row_matched_to_statistics_type = True
+
+                    # At this moment, preloaded benchmarks
+                    # do not support memory statitics
+                    if self.is_preloaded:
+                        pool = self.get_preloaded_pool_name(pool)
+
+                elif self.is_memory_statistics_included(row):
+                    statistics = self.get_memory_overhead(row)
+                    config = "FRAGMENTATION_" + config
+
+                    is_row_matched_to_statistics_type = True
+
+                if is_row_matched_to_statistics_type:
+                    results.append((config, pool, statistics))
 
-                results.append((updated_config, updated_pool, mean))
             except KeyError as e:
                 raise ValueError(f"Error parsing output: {e}")
 
         return results
 
+    def teardown(self):
+        return
+
+
+class GBenchPreloaded(GBench):
+    def __init__(self, bench, lib_to_be_replaced, replacing_lib):
+        super().__init__(bench)
+
+        self.is_preloaded = True
+
+        self.lib_to_be_replaced = lib_to_be_replaced
+        self.replacing_lib = replacing_lib
+
+    def get_preloaded_pool_name(self, pool_name) -> str:
+        new_pool_name = pool_name.replace(self.lib_to_be_replaced, self.replacing_lib)
+
+        return new_pool_name
+
 
 class GBenchGlibc(GBenchPreloaded):
     def __init__(self, bench, replacing_lib):
@@ -251,3 +265,13 @@ def __init__(self, bench):
 
     def extra_env_vars(self) -> dict:
         return {"LD_PRELOAD": "libtbbmalloc_proxy.so"}
+
+
+class GBenchMemoryOverhead(GBench):
+    def __init__(self, bench):
+        super().__init__(bench)
+
+        self.is_memory_overhead_checked = True
+
+    def unit(self):
+        return "%"