Merge pull request #18 from EleutherAI/interventions

import classifier at module level

Author: AlexTMallen

elk_generalization/elk/ccs.py

@@ -7,14 +7,14 @@
 
 import torch
 import torch.nn as nn
-from classifier import Classifier
 from concept_erasure import LeaceFitter
 from einops import repeat
 from torch import Tensor, optim
 from typing_extensions import override
 
 from elk_generalization.elk.burns_norm import BurnsNorm
 from elk_generalization.elk.ccs_losses import LOSSES, parse_loss
+from elk_generalization.elk.classifier import Classifier
 
 
 @dataclass

elk_generalization/elk/crc.py

@@ -1,9 +1,10 @@
 import torch
 import torch.nn.functional as F
-from classifier import Classifier
 from concept_erasure import LeaceEraser
 from torch import Tensor, nn, optim
 
+from elk_generalization.elk.classifier import Classifier
+
 
 class CrcReporter(Classifier):
     def __init__(self, in_features: int, device: torch.device, dtype: torch.dtype):

elk_generalization/elk/lda.py

@@ -1,9 +1,10 @@
 import torch
-from classifier import Classifier
 from concept_erasure.shrinkage import optimal_linear_shrinkage
 from sklearn.metrics import accuracy_score, roc_auc_score
 from torch import Tensor, nn
 
+from elk_generalization.elk.classifier import Classifier
+
 
 class LdaReporter(Classifier):
     def __init__(self, in_features: int, device: torch.device, dtype: torch.dtype):

elk_generalization/elk/lr_classifier.py

@@ -1,9 +1,10 @@
 import torch
-from classifier import Classifier
 from torch import Tensor
 from torch.nn.functional import binary_cross_entropy_with_logits as bce_with_logits
 from torch.nn.functional import cross_entropy
 
+from elk_generalization.elk.classifier import Classifier
+
 
 class LogisticRegression(Classifier):
     """Linear classifier trained with supervised learning."""

elk_generalization/elk/mean_diff.py

@@ -1,8 +1,9 @@
 import torch
-from classifier import Classifier
 from sklearn.metrics import accuracy_score, roc_auc_score
 from torch import Tensor, nn
 
+from elk_generalization.elk.classifier import Classifier
+
 
 class MeanDiffReporter(Classifier):
     def __init__(self, in_features: int, device: torch.device, dtype: torch.dtype):

elk_generalization/elk/run_transfers.py

@@ -46,6 +46,7 @@
 get_ceiling_latent_knowledge = False
 
 # code to modify models and datasets based on rank
+models = models[args.rank :: 8]
 print(ds_names, models)
 
 
@@ -61,14 +62,14 @@ def unpack_abbrev(ds_name, abbrev):
         exps = {k: ["B->B", "BE->B,BH"] for k in ["lr", "mean-diff", "lda"]}
     else:
         exps = {
-            "lr": ["A->A,B,AH,BH", "B->B,A,BH", "B->BH", "AE->AE,AH,BE,BH"],
-            "mean-diff": ["A->A,B,AH,BH", "B->B,A", "AE->AE,AH,BH"],
-            "lda": ["A->A,B,AH,BH", "B->B,A", "AE->AE,AH,BH"],
-            "lr-on-pair": ["A->A,B,AH,BH", "B->B,A", "AE->AE,AH,BH"],
+            # "lr": ["A->A,B,AH,BH", "B->B,A,BH", "B->BH", "AE->AE,AH,BE,BH"],
+            # "mean-diff": ["A->A,B,AH,BH", "B->B,A", "AE->AE,AH,BH"],
+            # "lda": ["A->A,B,AH,BH", "B->B,A", "AE->AE,AH,BH"],
+            # "lr-on-pair": ["A->A,B,AH,BH", "B->B,A", "AE->AE,AH,BH"],
             "mean-diff-on-pair": ["A->A,B,AH,BH", "B->B,A", "AE->AE,AH,BH"],
-            "ccs": ["A->A,B,AH,BH", "B->B,A", "AE->AE,AH,BH", "all->all,BH"],
-            "crc": ["A->A,B,AH,BH", "B->B,A", "AE->AE,AH,BH", "all->all,BH"],
-            "random": ["AE->AE,BH"],
+            # "ccs": ["A->A,B,AH,BH", "B->B,A", "AE->AE,AH,BH", "all->all,BH"],
+            # "crc": ["A->A,B,AH,BH", "B->B,A", "AE->AE,AH,BH", "all->all,BH"],
+            # "random": ["AE->AE,BH"],
         }
 
     experiments_dir = "../../experiments"

elk_generalization/elk/transfer.py

@@ -2,11 +2,11 @@
 from pathlib import Path
 
 import torch
-from classifier import Classifier
 from sklearn.metrics import accuracy_score, roc_auc_score
 from tqdm import tqdm
 
 from elk_generalization.elk.ccs import CcsConfig, CcsReporter
+from elk_generalization.elk.classifier import Classifier
 from elk_generalization.elk.crc import CrcReporter
 from elk_generalization.elk.lda import LdaReporter
 from elk_generalization.elk.lr_classifier import LogisticRegression

elk_generalization/results/figures.ipynb

@@ -1045,7 +1045,7 @@
     "anomaly_ds_names = ds_names.copy()\n",
     "anomaly_ds_names.remove(\"population\")  # population only has false labels on H\n",
     "\n",
-    "subtract_diag = False\n",
+    "subtract_diag = True\n",
     "root = \"../../anomaly-results/\"\n",
     "caption = \"Mechanistic anomaly detection AUROC. Note the Population dataset is omitted because the easy subset only contains true labels.\"\n",
     "if subtract_diag:\n",
@@ -1189,15 +1189,15 @@
     "model_sizes = {'Pythia': [0.41, 1, 1.4, 2.8, 6.9, 12],\n",
     "               'Llama': [7],\n",
     "               'Mistral': [7]}\n",
-    "pgr_values = {'Pythia': [0.48, 0.46, 0.56, 0.56, 0.55, 0.57],\n",
-    "              'Llama': [0.61],\n",
-    "              'Mistral': [0.61]}\n",
+    "pgr_values = {'Pythia': [0.47, 0.47, 0.55, 0.56, 0.56, 0.57],\n",
+    "              'Llama': [0.62],\n",
+    "              'Mistral': [0.63]}\n",
     "\n",
     "\n",
     "plt.figure(figsize=(4.3, 3), dpi=150)\n",
     "plt.plot(model_sizes['Pythia'], pgr_values['Pythia'], label=\"Pythia\", color=colors[0], marker=\"o\")\n",
     "plt.plot(model_sizes['Llama'], pgr_values['Llama'], label=\"Llama\", color=colors[1], marker=\"*\", markersize=10)\n",
-    "plt.plot(model_sizes['Mistral'], pgr_values['Mistral'], label=\"Mistral\", color=colors[2], marker=\"x\", markersize=10)\n",
+    "plt.plot(model_sizes['Mistral'], pgr_values['Mistral'], label=\"Mistral\", color=colors[2], marker=\"x\", markersize=8)\n",
     "plt.xlabel(\"model size\")\n",
     "plt.ylabel(\"PGR\")\n",
     "plt.legend()\n",