Fix the experiments

Author: mwydmuch

experiments/custom_utilities_methods.py

@@ -64,7 +64,7 @@ def bc_macro_min_tp_tn(
     alpha: float = 1,
     tolerance: float = 1e-6,
     init_y_pred: Union[str, np.ndarray, csr_matrix] = "random",
-    max_iter: int = 100,
+    max_iters: int = 100,
     shuffle_order: bool = True,
     verbose: bool = False,
     return_meta: bool = False,
@@ -80,7 +80,7 @@ def bc_macro_min_tp_tn(
         skip_tn=True,
         tolerance=tolerance,
         init_y_pred=init_y_pred,
-        max_iter=max_iter,
+        max_iters=max_iters,
         shuffle_order=shuffle_order,
         verbose=verbose,
         return_meta=return_meta,
@@ -94,7 +94,7 @@ def bc_micro_f1(
     alpha: float = 1,
     tolerance: float = 1e-6,
     init_y_pred: Union[str, np.ndarray, csr_matrix] = "random",
-    max_iter: int = 100,
+    max_iters: int = 100,
     shuffle_order: bool = True,
     verbose: bool = False,
     return_meta: bool = False,
@@ -109,7 +109,7 @@ def bc_macro_hmean(
     alpha: float = 1,
     tolerance: float = 1e-6,
     init_y_pred: Union[str, np.ndarray, csr_matrix] = "random",
-    max_iter: int = 100,
+    max_iters: int = 100,
     shuffle_order: bool = True,
     verbose: bool = False,
     return_meta: bool = False,
@@ -125,7 +125,7 @@ def bc_macro_hmean(
         skip_tn=True,
         tolerance=tolerance,
         init_y_pred=init_y_pred,
-        max_iter=max_iter,
+        max_iters=max_iters,
         shuffle_order=shuffle_order,
         verbose=verbose,
         return_meta=return_meta,
@@ -139,7 +139,7 @@ def bc_macro_gmean(
     alpha: float = 1,
     tolerance: float = 1e-6,
     init_y_pred: Union[str, np.ndarray, csr_matrix] = "random",
-    max_iter: int = 100,
+    max_iters: int = 100,
     shuffle_order: bool = True,
     verbose: bool = False,
     return_meta: bool = False,
@@ -155,7 +155,7 @@ def bc_macro_gmean(
         skip_tn=True,
         tolerance=tolerance,
         init_y_pred=init_y_pred,
-        max_iter=max_iter,
+        max_iters=max_iters,
         shuffle_order=shuffle_order,
         verbose=verbose,
         return_meta=return_meta,

experiments/requirements.txt

@@ -1,2 +1,3 @@
 xcolumns
 click
+tqdm

experiments/run_iclr_2024_fw_experiment.py

@@ -23,12 +23,6 @@
 from xcolumns.weighted_prediction import *
 
 
-# TODO: refactor this
-RECALCULATE_RESUTLS = False
-RECALCULATE_PREDICTION = False
-RETRAIN_MODEL = False
-
-
 def frank_wolfe_wrapper(
     Y_val,
     pred_val,
@@ -592,7 +586,25 @@ def predict_proba(self, X, top_k):
 @click.option("-s", "--seed", type=int, required=True)
 @click.option("-m", "--method", type=str, required=False, default=None)
 @click.option("-t", "--testsplit", type=float, required=False, default=0)
-def main(experiment, k, seed, method, testsplit):
+@click.option("-r", "--results_dir", type=str, required=False, default="results_fw/")
+@click.option(
+    "--recalculate_predictions", is_flag=True, type=bool, required=False, default=False
+)
+@click.option(
+    "--recalculate_results", is_flag=True, type=bool, required=False, default=False
+)
+@click.option("--retrain_model", is_flag=True, type=bool, required=False, default=False)
+def main(
+    experiment,
+    k,
+    seed,
+    method,
+    testsplit,
+    results_dir,
+    recalculate_predictions,
+    recalculate_results,
+    retrain_model,
+):
     print(experiment)
 
     if method is None:
@@ -894,7 +906,7 @@ def main(experiment, k, seed, method, testsplit):
             model = PytorchModel(model_path, model_seed, loss="asym")
 
     if isinstance(model, ModelWrapper):
-        if not os.path.exists(model_path) or RETRAIN_MODEL:
+        if not os.path.exists(model_path) or retrain_model:
             with Timer():
                 model.fit(X_train, Y_train, X_test, Y_test)
         # else:
@@ -907,7 +919,7 @@ def main(experiment, k, seed, method, testsplit):
             top_k = 100
         print("Predicting for validation set ...")
         val_pred_path = f"models_and_predictions/{experiment}_seed={model_seed}_split={1 - testsplit}_top_k={top_k}_pred_val.pkl"
-        if not os.path.exists(val_pred_path) or RETRAIN_MODEL:
+        if not os.path.exists(val_pred_path) or retrain_model:
             with Timer():
                 pred_val = model.predict_proba(X_val, top_k=top_k)
                 align_dim1(Y_train, pred_val)
@@ -920,7 +932,7 @@ def main(experiment, k, seed, method, testsplit):
 
         print("Predicting for test set ...")
         test_pred_path = f"models_and_predictions/{experiment}_seed={model_seed}_split={1 - testsplit}_top_k={top_k}_pred_test.pkl"
-        if not os.path.exists(test_pred_path) or RETRAIN_MODEL:
+        if not os.path.exists(test_pred_path) or retrain_model:
             with Timer():
                 pred_test = model.predict_proba(X_test, top_k=top_k)
                 align_dim1(Y_train, pred_test)
@@ -941,7 +953,7 @@ def main(experiment, k, seed, method, testsplit):
         pred_test = sp.csr_matrix(pred_test)
 
     print("Calculating metrics ...")
-    output_path_prefix = f"results_fw/{experiment}/"
+    output_path_prefix = f"{results_dir}/{experiment}/"
     os.makedirs(output_path_prefix, exist_ok=True)
     for method, func in methods.items():
         print(f"{experiment} - {method} @ {k}: ")
@@ -950,9 +962,9 @@ def main(experiment, k, seed, method, testsplit):
         results_path = f"{output_path}_results.json"
         pred_path = f"{output_path}_pred.pkl"
 
-        if not os.path.exists(results_path) or RECALCULATE_RESUTLS:
+        if not os.path.exists(results_path) or recalculate_results:
             results = {}
-            if not os.path.exists(pred_path) or RECALCULATE_PREDICTION:
+            if not os.path.exists(pred_path) or recalculate_predictions:
                 # results["test_log_loss"] = log_loss(Y_test, pred_test)
                 # results["val_log_loss"] = log_loss(Y_val, pred_val)
 

experiments/run_neurips_2023_bca_experiment.py

@@ -27,7 +27,7 @@
     "optimal-instance-prec": (predict_optimizing_instance_precision, {}),
     # "block-coord-instance-prec": (bc_instance_precision_at_k, {}), # This is the same as optimal-instance-prec but using block coordinate, for sanity-check purposes only
     "optimal-instance-ps-prec": (
-        predict_optimizing_instance_propensity_weighted_precision,
+        predict_optimizing_instance_propensity_scored_precision,
         {},
     ),
     "power-law-with-beta=0.75": (
@@ -136,34 +136,34 @@
     # Greedy / 1 iter variants
     "greedy-macro-prec": (
         predict_optimizing_macro_precision_using_bc,
-        {"init_y_pred": "greedy", "max_iter": 1},
+        {"init_y_pred": "greedy", "max_iters": 1},
     ),
     "greedy-macro-recall": (
         predict_optimizing_macro_precision_using_bc,
-        {"init_y_pred": "greedy", "max_iter": 1},
+        {"init_y_pred": "greedy", "max_iters": 1},
     ),
     "greedy-macro-f1": (
         predict_optimizing_macro_f1_score_using_bc,
-        {"init_y_pred": "greedy", "max_iter": 1},
+        {"init_y_pred": "greedy", "max_iters": 1},
     ),
     "greedy-cov": (
         predict_optimizing_coverage_using_bc,
-        {"init_y_pred": "greedy", "max_iter": 1},
+        {"init_y_pred": "greedy", "max_iters": 1},
     ),
     #
     "block-coord-macro-prec-iter=1": (
         predict_optimizing_macro_precision_using_bc,
-        {"max_iter": 1},
+        {"max_iters": 1},
     ),
     "block-coord-macro-recall-iter=1": (
         predict_optimizing_macro_precision_using_bc,
-        {"max_iter": 1},
+        {"max_iters": 1},
     ),
     "block-coord-macro-f1-iter=1": (
         predict_optimizing_macro_f1_score_using_bc,
-        {"max_iter": 1},
+        {"max_iters": 1},
     ),
-    "block-coord-cov-iter=1": (predict_optimizing_coverage_using_bc, {"max_iter": 1}),
+    "block-coord-cov-iter=1": (predict_optimizing_coverage_using_bc, {"max_iters": 1}),
     #
     # Similar results to the above
     # "greedy-start-block-coord-macro-prec": (predict_optimizing_macro_precision_using_bc, {"init_y_pred": "greedy"},),
@@ -227,7 +227,7 @@ def calculate_and_report_metrics(y_true, y_pred, k, metrics):
 @click.option("-m", "--method", type=str, required=False, default=None)
 @click.option("-p", "--probabilities_path", type=str, required=False, default=None)
 @click.option("-l", "--labels_path", type=str, required=False, default=None)
-@click.option("-r", "--results_dir", type=str, required=False, default="results/")
+@click.option("-r", "--results_dir", type=str, required=False, default="results_bc/")
 @click.option(
     "--recalculate_predictions", is_flag=True, type=bool, required=False, default=False
 )

experiments/utils.py

@@ -14,9 +14,11 @@
 
 
 def call_function_with_supported_kwargs(func, *args, **kwargs):
-    selected_kwargs = {
-        k: v for k, v in kwargs.items() if k in func.__code__.co_varnames
-    }
+    if hasattr(func, "__signature__"):
+        params = func.__signature__.parameters
+    else:
+        params = func.__code__.co_varnames
+    selected_kwargs = {k: v for k, v in kwargs.items() if k in params}
     return func(*args, **selected_kwargs)
 
 

xcolumns/block_coordinate.py

@@ -277,12 +277,12 @@ def predict_using_bc_with_0approx(
     binary_metric_func: Union[Callable, List[Callable]],
     k: int,
     metric_aggregation: str = "mean",  # "mean" or "sum"
-    maximize=True,
+    maximize: bool = True,
     tolerance: float = 1e-6,
     init_y_pred: Union[str, Matrix] = "random",  # "random", "top", "greedy", Matrix
-    max_iter: int = 100,
+    max_iters: int = 100,
     shuffle_order: bool = True,
-    skip_tn=False,
+    skip_tn: bool = False,
     return_meta: bool = False,
     seed: Optional[int] = None,
     verbose: bool = False,
@@ -313,7 +313,7 @@ def predict_using_bc_with_0approx(
         maximize: Whether to maximize the metric.
         tolerance: Defines the stopping condition, if the expected improvement of the metric is smaller than **tolerance** the algorithm stops.
         init_y_pred: The initial prediction matrix. It can be either "random", "top", "greedy" or a matrix of shape (n, m).
-        max_iter: The maximum number of iterations.
+        max_iters: The maximum number of iterations.
         shuffle_order: Whether to shuffle the order of instances in each iteration.
         skip_tn: Whether to skip the calculation of True Negatives in the confusion matrix, if the metric does not use the True Negatives, this can speed up the calculation, especially when using sparse matrices.
         return_meta: Whether to return the meta information.
@@ -345,7 +345,13 @@ def my_binary_f1_score_on_conf_matrix(tp, fp, fn, tn):
     """
 
     log_info(
-        f"Starting optimization of ETU metric using block coordinate {'ascent' if maximize else 'descent'} algorithm ...",
+        f"Starting optimization of ETU metric using block coordinate {'ascent (maximization)' if maximize else 'descent (minimization)'} algorithm ...",
+        verbose,
+    )
+    if k > 0:
+        log_info(f"  Budget k: {k}", verbose)
+    log_info(
+        f"  Tolerance (stopping condition): {tolerance}, max iterations: {max_iters}",
         verbose,
     )
 
@@ -378,8 +384,8 @@ def my_binary_f1_score_on_conf_matrix(tp, fp, fn, tn):
     # Initialize the instance order and set seed for shuffling
     rng = np.random.default_rng(seed)
     order = np.arange(n)
-    for j in range(1, max_iter + 1):
-        log_info(f"  Starting iteration {j}/{max_iter} ...", verbose)
+    for j in range(1, max_iters + 1):
+        log_info(f"  Starting iteration {j}/{max_iters} ...", verbose)
 
         if shuffle_order:
             rng.shuffle(order)
@@ -436,7 +442,7 @@ def my_binary_f1_score_on_conf_matrix(tp, fp, fn, tn):
         meta["utilities"].append(new_utility)
 
         log_info(
-            f"    Iteration {j}/{max_iter} finished, expected metric value: {old_utility} -> {new_utility}",
+            f"    Iteration {j}/{max_iters} finished, expected metric value: {old_utility} -> {new_utility}",
             verbose,
         )
         if (
@@ -565,7 +571,7 @@ def predict_optimizing_coverage_using_bc(
     init_y_pred: Union[
         str, np.ndarray, csr_matrix
     ] = "random",  # "random", "topk", "random", or csr_matrix
-    max_iter: int = 100,
+    max_iters: int = 100,
     shuffle_order: bool = True,
     return_meta: bool = False,
     seed: Optional[int] = None,
@@ -614,8 +620,8 @@ def predict_optimizing_coverage_using_bc(
     # Initialize the instance order and set seed for shuffling
     rng = np.random.default_rng(seed)
     order = np.arange(n)
-    for j in range(1, max_iter + 1):
-        log_info(f"  Starting iteration {j}/{max_iter} ...", verbose)
+    for j in range(1, max_iters + 1):
+        log_info(f"  Starting iteration {j}/{max_iters} ...", verbose)
 
         if shuffle_order:
             rng.shuffle(order)
@@ -642,7 +648,7 @@ def predict_optimizing_coverage_using_bc(
         meta["utilities"].append(new_cov)
 
         log_info(
-            f"    Iteration {j}/{max_iter} finished, expected coverage: {old_cov} -> {new_cov}",
+            f"    Iteration {j}/{max_iters} finished, expected coverage: {old_cov} -> {new_cov}",
             verbose,
         )
         if new_cov <= old_cov + tolerance:
@@ -759,7 +765,7 @@ def predict_optimizing_instance_precision_using_bc(
     k: int,
     tolerance: float = 1e-6,
     init_y_pred: Union[str, np.ndarray, csr_matrix] = "random",
-    max_iter: int = 100,
+    max_iters: int = 100,
     shuffle_order: bool = True,
     verbose: bool = False,
     return_meta: bool = False,
@@ -779,7 +785,7 @@ def instance_precision_with_specific_k(tp, fp, fn, tn):
         metric_aggregation="sum",
         tolerance=tolerance,
         init_y_pred=init_y_pred,
-        max_iter=max_iter,
+        max_iters=max_iters,
         shuffle_order=shuffle_order,
         verbose=verbose,
         return_meta=return_meta,
@@ -792,7 +798,7 @@ def predict_optimizing_mixed_instance_precision_and_macro_precision_using_bc(
     alpha: float = 1,
     tolerance: float = 1e-6,
     init_y_pred: Union[str, np.ndarray, csr_matrix] = "random",
-    max_iter: int = 100,
+    max_iters: int = 100,
     shuffle_order: bool = True,
     verbose: bool = False,
     return_meta: bool = False,
@@ -817,7 +823,7 @@ def mixed_utility_fn(tp, fp, fn, tn):
         skip_tn=True,
         tolerance=tolerance,
         init_y_pred=init_y_pred,
-        max_iter=max_iter,
+        max_iters=max_iters,
         shuffle_order=shuffle_order,
         verbose=verbose,
         return_meta=return_meta,
@@ -830,7 +836,7 @@ def predict_optimizing_mixed_instance_precision_and_macro_recall_using_bc(
     alpha: float = 1,
     tolerance: float = 1e-6,
     init_y_pred: Union[str, np.ndarray, csr_matrix] = "random",
-    max_iter: int = 100,
+    max_iters: int = 100,
     shuffle_order: bool = True,
     verbose: bool = False,
     return_meta: bool = False,
@@ -855,7 +861,7 @@ def mixed_utility_fn(tp, fp, fn, tn):
         skip_tn=True,
         tolerance=tolerance,
         init_y_pred=init_y_pred,
-        max_iter=max_iter,
+        max_iters=max_iters,
         shuffle_order=shuffle_order,
         verbose=verbose,
         return_meta=return_meta,
@@ -868,7 +874,7 @@ def predict_optimizing_mixed_instance_precision_and_macro_f1_score_using_bc(
     alpha: float = 1,
     tolerance: float = 1e-6,
     init_y_pred: Union[str, np.ndarray, csr_matrix] = "random",
-    max_iter: int = 100,
+    max_iters: int = 100,
     shuffle_order: bool = True,
     verbose: bool = False,
     return_meta: bool = False,
@@ -893,7 +899,7 @@ def mixed_utility_fn(tp, fp, fn, tn):
         skip_tn=True,
         tolerance=tolerance,
         init_y_pred=init_y_pred,
-        max_iter=max_iter,
+        max_iters=max_iters,
         shuffle_order=shuffle_order,
         verbose=verbose,
         return_meta=return_meta,