evaluation.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Evaluation (TODO Latex figures without Type3 fonts!!!)\n",
    "\n",
    "- Tables: Overall / By Label Class\n",
    "- Figures: By Citation Count / By Year (difference)\n",
    "- Samples (ACL only?)\n",
    "- Confusion matrix?\n",
    "\n",
    "```bash\n",
    "# Compress all result files (exclude model weights)\n",
    "tar -cvzf acl_docrel.tar.gz --exclude='*.bin' acl_docrel/*\n",
    "```\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 439,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "INFO:transformers.file_utils:PyTorch version 1.4.0 available.\n",
      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m W&B installed but not logged in.  Run `wandb login` or set the WANDB_API_KEY env variable.\n"
     ]
    }
   ],
   "source": [
    "import re\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from matplotlib import pyplot as plt\n",
    "from collections import defaultdict\n",
    "from experiments.utils import highlight_max\n",
    "import matplotlib.patches as mpatches\n",
    "from matplotlib.container import ErrorbarContainer\n",
    "\n",
    "import os\n",
    "import random\n",
    "import json\n",
    "import pickle\n",
    "import logging\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from sklearn.preprocessing import OneHotEncoder\n",
    "from sklearn.metrics import classification_report, confusion_matrix, precision_recall_fscore_support\n",
    "from IPython.core.display import display\n",
    "\n",
    "logging.basicConfig(level=logging.INFO)\n",
    "logger = logging.getLogger(__name__)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 180,
   "metadata": {},
   "outputs": [],
   "source": [
    "output_dir = './output/'\n",
    "folds = 4\n",
    "\n",
    "metric_cols = {\n",
    "    'micro avg__precision': 'Precision (micro)',\n",
    "    'micro avg__recall': 'Recall (micro)',\n",
    "    'micro avg__f1-score': 'F1 (micro)',\n",
    "    'macro avg__precision': 'Precision (macro)',\n",
    "    'macro avg__recall': 'Recall (macro)',\n",
    "    'macro avg__f1-score': 'F1 (macro)',\n",
    "}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Overall results\n",
    "\n",
    "Table: System, P, R, F micro (std) F macro (std) | for ACL-Anthology + CORD-19\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 359,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "WARNING:__main__:Skip ./output/acl_docrel/folds/1/baseline-rnn__fasttext (not all files exists, probably still running..)\n",
      "WARNING:__main__:Skip ./output/acl_docrel/folds/1/longformer-base-4096 (not all files exists, probably still running..)\n",
      "WARNING:__main__:Skip ./output/acl_docrel/folds/2/baseline-rnn__fasttext (not all files exists, probably still running..)\n",
      "WARNING:__main__:Skip ./output/acl_docrel/folds/3/baseline-rnn__fasttext (not all files exists, probably still running..)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Experiment: acl_docrel\n",
      "- Fold: 1\n",
      "   - System: roberta-base\n",
      "   - System: electra-base-discriminator\n",
      "   - System: bert-base-cased\n",
      "   - System: baseline-rnn__fasttext__custom\n",
      "   - System: xlnet-base-cased\n",
      "   - System: baseline-rnn\n",
      "   - System: scibert-scivocab-uncased\n",
      "   - System: covid_bert_base\n",
      "- Fold: 2\n",
      "   - System: roberta-base\n",
      "   - System: electra-base-discriminator\n",
      "   - System: bert-base-cased\n",
      "   - System: baseline-rnn__fasttext__custom\n",
      "   - System: xlnet-base-cased\n",
      "   - System: baseline-rnn\n",
      "   - System: scibert-scivocab-uncased\n",
      "   - System: covid_bert_base\n",
      "- Fold: 3\n",
      "   - System: roberta-base\n",
      "   - System: electra-base-discriminator\n",
      "   - System: bert-base-cased\n",
      "   - System: baseline-rnn__fasttext__custom\n",
      "   - System: xlnet-base-cased\n",
      "   - System: baseline-rnn\n",
      "   - System: scibert-scivocab-uncased\n",
      "   - System: covid_bert_base\n",
      "- Fold: 4\n",
      "   - System: roberta-base\n",
      "   - System: electra-base-discriminator\n",
      "   - System: bert-base-cased\n",
      "   - System: baseline-rnn__fasttext__custom\n",
      "   - System: xlnet-base-cased\n",
      "   - System: baseline-rnn\n",
      "   - System: scibert-scivocab-uncased\n",
      "   - System: covid_bert_base\n"
     ]
    }
   ],
   "source": [
    "overall_rows = []\n",
    "exp_names = []\n",
    "\n",
    "for exp in os.listdir(output_dir):\n",
    "    exp_dir = os.path.join(output_dir, exp)\n",
    "\n",
    "    if not exp.endswith('_docrel') or os.path.isfile(exp_dir):\n",
    "        continue\n",
    "    \n",
    "    print(f'Experiment: {exp}')\n",
    "    \n",
    "    for fold in range(1, folds+1):\n",
    "        fold_dir = os.path.join(exp_dir, 'folds', str(fold))\n",
    "        print(f'- Fold: {fold}')\n",
    "\n",
    "        if not os.path.exists(fold_dir):\n",
    "            logger.warning(f'Fold does not exist: {fold_dir}')\n",
    "            continue\n",
    "\n",
    "        for sys in os.listdir(fold_dir):\n",
    "            sys_dir = os.path.join(fold_dir, sys)\n",
    "\n",
    "            if not os.path.isdir(sys_dir) or sys.startswith('.') or sys.startswith('__'):\n",
    "                continue\n",
    "\n",
    "            try:\n",
    "                metrics = json.load(open(os.path.join(sys_dir, 'metrics.json')))\n",
    "                metrics = {k.replace('eval_', ''): v for k, v in metrics.items()}\n",
    "                exp_name = exp.replace('acl_docrel', 'ACL Anthology').replace('cord19_docrel', 'CORD-19')\n",
    "                row = {                    \n",
    "                    'system': sys,\n",
    "                    'experiment': exp_name,\n",
    "                    'fold': fold,\n",
    "                    'class': None,\n",
    "                    'precision': 0.,\n",
    "                    'recall': 0.,\n",
    "                    'f1-score': 0.,\n",
    "                    'support': 0,\n",
    "                }\n",
    "                cs = set([k.split('__')[0] for k in metrics.keys() if len(k.split('__')) == 2])\n",
    "\n",
    "                for c in cs:\n",
    "                    crow = row.copy()\n",
    "                    crow['class'] = c\n",
    "                    for m in ms:\n",
    "                        crow[m] = metrics[c + '__' + m]\n",
    "                    overall_rows.append(crow)\n",
    "                    \n",
    "                print(f'   - System: {sys}')\n",
    "\n",
    "            except FileNotFoundError:\n",
    "                logger.warning(f'Skip {sys_dir} (not all files exists, probably still running..)')\n",
    "                pass\n",
    "    #break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 360,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(overall_rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 361,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>system</th>\n",
       "      <th>experiment</th>\n",
       "      <th>fold</th>\n",
       "      <th>class</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>roberta-base</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>1</td>\n",
       "      <td>evaluation</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>253</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>roberta-base</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>1</td>\n",
       "      <td>discussion</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>296</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>roberta-base</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>1</td>\n",
       "      <td>introduction</td>\n",
       "      <td>0.508932</td>\n",
       "      <td>0.354996</td>\n",
       "      <td>0.418250</td>\n",
       "      <td>4093</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>roberta-base</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>1</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.707644</td>\n",
       "      <td>0.541440</td>\n",
       "      <td>0.613484</td>\n",
       "      <td>21646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>roberta-base</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>1</td>\n",
       "      <td>none</td>\n",
       "      <td>0.905441</td>\n",
       "      <td>0.951392</td>\n",
       "      <td>0.927848</td>\n",
       "      <td>6069</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>507</th>\n",
       "      <td>covid_bert_base</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>4</td>\n",
       "      <td>conclusion</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>508</th>\n",
       "      <td>covid_bert_base</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>4</td>\n",
       "      <td>related work</td>\n",
       "      <td>0.634146</td>\n",
       "      <td>0.556905</td>\n",
       "      <td>0.593021</td>\n",
       "      <td>3128</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>509</th>\n",
       "      <td>covid_bert_base</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>4</td>\n",
       "      <td>other</td>\n",
       "      <td>0.607843</td>\n",
       "      <td>0.589046</td>\n",
       "      <td>0.598297</td>\n",
       "      <td>5368</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>510</th>\n",
       "      <td>covid_bert_base</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>4</td>\n",
       "      <td>results</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>286</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>511</th>\n",
       "      <td>covid_bert_base</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>4</td>\n",
       "      <td>method</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>175</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>512 rows × 8 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              system     experiment  fold         class  precision    recall  \\\n",
       "0       roberta-base  ACL Anthology     1    evaluation   0.000000  0.000000   \n",
       "1       roberta-base  ACL Anthology     1    discussion   0.000000  0.000000   \n",
       "2       roberta-base  ACL Anthology     1  introduction   0.508932  0.354996   \n",
       "3       roberta-base  ACL Anthology     1     micro avg   0.707644  0.541440   \n",
       "4       roberta-base  ACL Anthology     1          none   0.905441  0.951392   \n",
       "..               ...            ...   ...           ...        ...       ...   \n",
       "507  covid_bert_base  ACL Anthology     4    conclusion   0.000000  0.000000   \n",
       "508  covid_bert_base  ACL Anthology     4  related work   0.634146  0.556905   \n",
       "509  covid_bert_base  ACL Anthology     4         other   0.607843  0.589046   \n",
       "510  covid_bert_base  ACL Anthology     4       results   0.000000  0.000000   \n",
       "511  covid_bert_base  ACL Anthology     4        method   0.000000  0.000000   \n",
       "\n",
       "     f1-score  support  \n",
       "0    0.000000      253  \n",
       "1    0.000000      296  \n",
       "2    0.418250     4093  \n",
       "3    0.613484    21646  \n",
       "4    0.927848     6069  \n",
       "..        ...      ...  \n",
       "507  0.000000      281  \n",
       "508  0.593021     3128  \n",
       "509  0.598297     5368  \n",
       "510  0.000000      286  \n",
       "511  0.000000      175  \n",
       "\n",
       "[512 rows x 8 columns]"
      ]
     },
     "execution_count": 361,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 362,
   "metadata": {},
   "outputs": [],
   "source": [
    "systems = df['system'].unique().tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 363,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "baseline-rnn\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>system</th>\n",
       "      <th>experiment</th>\n",
       "      <th>fold</th>\n",
       "      <th>class</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>83</th>\n",
       "      <td>baseline-rnn</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>1</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.760135</td>\n",
       "      <td>0.198374</td>\n",
       "      <td>0.314636</td>\n",
       "      <td>21646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>211</th>\n",
       "      <td>baseline-rnn</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>2</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.750000</td>\n",
       "      <td>0.202060</td>\n",
       "      <td>0.318351</td>\n",
       "      <td>21558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>339</th>\n",
       "      <td>baseline-rnn</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>3</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.791186</td>\n",
       "      <td>0.197907</td>\n",
       "      <td>0.316616</td>\n",
       "      <td>21500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>467</th>\n",
       "      <td>baseline-rnn</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>4</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.787380</td>\n",
       "      <td>0.203705</td>\n",
       "      <td>0.323671</td>\n",
       "      <td>21379</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           system     experiment  fold      class  precision    recall  \\\n",
       "83   baseline-rnn  ACL Anthology     1  micro avg   0.760135  0.198374   \n",
       "211  baseline-rnn  ACL Anthology     2  micro avg   0.750000  0.202060   \n",
       "339  baseline-rnn  ACL Anthology     3  micro avg   0.791186  0.197907   \n",
       "467  baseline-rnn  ACL Anthology     4  micro avg   0.787380  0.203705   \n",
       "\n",
       "     f1-score  support  \n",
       "83   0.314636    21646  \n",
       "211  0.318351    21558  \n",
       "339  0.316616    21500  \n",
       "467  0.323671    21379  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "baseline-rnn__fasttext__custom\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>system</th>\n",
       "      <th>experiment</th>\n",
       "      <th>fold</th>\n",
       "      <th>class</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>51</th>\n",
       "      <td>baseline-rnn__fasttext__custom</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>1</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.771262</td>\n",
       "      <td>0.176799</td>\n",
       "      <td>0.287658</td>\n",
       "      <td>21646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>179</th>\n",
       "      <td>baseline-rnn__fasttext__custom</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>2</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.752587</td>\n",
       "      <td>0.185546</td>\n",
       "      <td>0.297697</td>\n",
       "      <td>21558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>307</th>\n",
       "      <td>baseline-rnn__fasttext__custom</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>3</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.781591</td>\n",
       "      <td>0.183256</td>\n",
       "      <td>0.296899</td>\n",
       "      <td>21500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>435</th>\n",
       "      <td>baseline-rnn__fasttext__custom</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>4</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.759660</td>\n",
       "      <td>0.192198</td>\n",
       "      <td>0.306779</td>\n",
       "      <td>21379</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                             system     experiment  fold      class  \\\n",
       "51   baseline-rnn__fasttext__custom  ACL Anthology     1  micro avg   \n",
       "179  baseline-rnn__fasttext__custom  ACL Anthology     2  micro avg   \n",
       "307  baseline-rnn__fasttext__custom  ACL Anthology     3  micro avg   \n",
       "435  baseline-rnn__fasttext__custom  ACL Anthology     4  micro avg   \n",
       "\n",
       "     precision    recall  f1-score  support  \n",
       "51    0.771262  0.176799  0.287658    21646  \n",
       "179   0.752587  0.185546  0.297697    21558  \n",
       "307   0.781591  0.183256  0.296899    21500  \n",
       "435   0.759660  0.192198  0.306779    21379  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "bert-base-cased\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>system</th>\n",
       "      <th>experiment</th>\n",
       "      <th>fold</th>\n",
       "      <th>class</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>35</th>\n",
       "      <td>bert-base-cased</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>1</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.714614</td>\n",
       "      <td>0.588931</td>\n",
       "      <td>0.645714</td>\n",
       "      <td>21646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>163</th>\n",
       "      <td>bert-base-cased</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>2</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.717620</td>\n",
       "      <td>0.587531</td>\n",
       "      <td>0.646093</td>\n",
       "      <td>21558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>291</th>\n",
       "      <td>bert-base-cased</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>3</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.720533</td>\n",
       "      <td>0.596233</td>\n",
       "      <td>0.652516</td>\n",
       "      <td>21500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>419</th>\n",
       "      <td>bert-base-cased</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>4</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.718741</td>\n",
       "      <td>0.587492</td>\n",
       "      <td>0.646523</td>\n",
       "      <td>21379</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              system     experiment  fold      class  precision    recall  \\\n",
       "35   bert-base-cased  ACL Anthology     1  micro avg   0.714614  0.588931   \n",
       "163  bert-base-cased  ACL Anthology     2  micro avg   0.717620  0.587531   \n",
       "291  bert-base-cased  ACL Anthology     3  micro avg   0.720533  0.596233   \n",
       "419  bert-base-cased  ACL Anthology     4  micro avg   0.718741  0.587492   \n",
       "\n",
       "     f1-score  support  \n",
       "35   0.645714    21646  \n",
       "163  0.646093    21558  \n",
       "291  0.652516    21500  \n",
       "419  0.646523    21379  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "covid_bert_base\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>system</th>\n",
       "      <th>experiment</th>\n",
       "      <th>fold</th>\n",
       "      <th>class</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>115</th>\n",
       "      <td>covid_bert_base</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>1</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.713653</td>\n",
       "      <td>0.565555</td>\n",
       "      <td>0.631031</td>\n",
       "      <td>21646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>243</th>\n",
       "      <td>covid_bert_base</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>2</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.712580</td>\n",
       "      <td>0.575703</td>\n",
       "      <td>0.636870</td>\n",
       "      <td>21558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>371</th>\n",
       "      <td>covid_bert_base</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>3</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.715581</td>\n",
       "      <td>0.583814</td>\n",
       "      <td>0.643016</td>\n",
       "      <td>21500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>covid_bert_base</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>4</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.707958</td>\n",
       "      <td>0.580102</td>\n",
       "      <td>0.637684</td>\n",
       "      <td>21379</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "              system     experiment  fold      class  precision    recall  \\\n",
       "115  covid_bert_base  ACL Anthology     1  micro avg   0.713653  0.565555   \n",
       "243  covid_bert_base  ACL Anthology     2  micro avg   0.712580  0.575703   \n",
       "371  covid_bert_base  ACL Anthology     3  micro avg   0.715581  0.583814   \n",
       "499  covid_bert_base  ACL Anthology     4  micro avg   0.707958  0.580102   \n",
       "\n",
       "     f1-score  support  \n",
       "115  0.631031    21646  \n",
       "243  0.636870    21558  \n",
       "371  0.643016    21500  \n",
       "499  0.637684    21379  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "electra-base-discriminator\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>system</th>\n",
       "      <th>experiment</th>\n",
       "      <th>fold</th>\n",
       "      <th>class</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>electra-base-discriminator</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>1</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.699091</td>\n",
       "      <td>0.543518</td>\n",
       "      <td>0.611566</td>\n",
       "      <td>21646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>147</th>\n",
       "      <td>electra-base-discriminator</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>2</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.702621</td>\n",
       "      <td>0.547221</td>\n",
       "      <td>0.615260</td>\n",
       "      <td>21558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>275</th>\n",
       "      <td>electra-base-discriminator</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>3</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.703474</td>\n",
       "      <td>0.554698</td>\n",
       "      <td>0.620290</td>\n",
       "      <td>21500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>403</th>\n",
       "      <td>electra-base-discriminator</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>4</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.707529</td>\n",
       "      <td>0.552084</td>\n",
       "      <td>0.620215</td>\n",
       "      <td>21379</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         system     experiment  fold      class  precision  \\\n",
       "19   electra-base-discriminator  ACL Anthology     1  micro avg   0.699091   \n",
       "147  electra-base-discriminator  ACL Anthology     2  micro avg   0.702621   \n",
       "275  electra-base-discriminator  ACL Anthology     3  micro avg   0.703474   \n",
       "403  electra-base-discriminator  ACL Anthology     4  micro avg   0.707529   \n",
       "\n",
       "       recall  f1-score  support  \n",
       "19   0.543518  0.611566    21646  \n",
       "147  0.547221  0.615260    21558  \n",
       "275  0.554698  0.620290    21500  \n",
       "403  0.552084  0.620215    21379  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "roberta-base\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>system</th>\n",
       "      <th>experiment</th>\n",
       "      <th>fold</th>\n",
       "      <th>class</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>roberta-base</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>1</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.707644</td>\n",
       "      <td>0.541440</td>\n",
       "      <td>0.613484</td>\n",
       "      <td>21646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>131</th>\n",
       "      <td>roberta-base</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>2</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.702251</td>\n",
       "      <td>0.561462</td>\n",
       "      <td>0.624014</td>\n",
       "      <td>21558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>259</th>\n",
       "      <td>roberta-base</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>3</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.705830</td>\n",
       "      <td>0.556326</td>\n",
       "      <td>0.622223</td>\n",
       "      <td>21500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>387</th>\n",
       "      <td>roberta-base</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>4</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.702513</td>\n",
       "      <td>0.561018</td>\n",
       "      <td>0.623843</td>\n",
       "      <td>21379</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "           system     experiment  fold      class  precision    recall  \\\n",
       "3    roberta-base  ACL Anthology     1  micro avg   0.707644  0.541440   \n",
       "131  roberta-base  ACL Anthology     2  micro avg   0.702251  0.561462   \n",
       "259  roberta-base  ACL Anthology     3  micro avg   0.705830  0.556326   \n",
       "387  roberta-base  ACL Anthology     4  micro avg   0.702513  0.561018   \n",
       "\n",
       "     f1-score  support  \n",
       "3    0.613484    21646  \n",
       "131  0.624014    21558  \n",
       "259  0.622223    21500  \n",
       "387  0.623843    21379  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "scibert-scivocab-uncased\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>system</th>\n",
       "      <th>experiment</th>\n",
       "      <th>fold</th>\n",
       "      <th>class</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>scibert-scivocab-uncased</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>1</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.721928</td>\n",
       "      <td>0.617805</td>\n",
       "      <td>0.665820</td>\n",
       "      <td>21646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>227</th>\n",
       "      <td>scibert-scivocab-uncased</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>2</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.728194</td>\n",
       "      <td>0.620002</td>\n",
       "      <td>0.669757</td>\n",
       "      <td>21558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>355</th>\n",
       "      <td>scibert-scivocab-uncased</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>3</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.727209</td>\n",
       "      <td>0.628512</td>\n",
       "      <td>0.674268</td>\n",
       "      <td>21500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>483</th>\n",
       "      <td>scibert-scivocab-uncased</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>4</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.722643</td>\n",
       "      <td>0.620562</td>\n",
       "      <td>0.667724</td>\n",
       "      <td>21379</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                       system     experiment  fold      class  precision  \\\n",
       "99   scibert-scivocab-uncased  ACL Anthology     1  micro avg   0.721928   \n",
       "227  scibert-scivocab-uncased  ACL Anthology     2  micro avg   0.728194   \n",
       "355  scibert-scivocab-uncased  ACL Anthology     3  micro avg   0.727209   \n",
       "483  scibert-scivocab-uncased  ACL Anthology     4  micro avg   0.722643   \n",
       "\n",
       "       recall  f1-score  support  \n",
       "99   0.617805  0.665820    21646  \n",
       "227  0.620002  0.669757    21558  \n",
       "355  0.628512  0.674268    21500  \n",
       "483  0.620562  0.667724    21379  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "xlnet-base-cased\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>system</th>\n",
       "      <th>experiment</th>\n",
       "      <th>fold</th>\n",
       "      <th>class</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>xlnet-base-cased</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>1</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.704042</td>\n",
       "      <td>0.584219</td>\n",
       "      <td>0.638558</td>\n",
       "      <td>21646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>xlnet-base-cased</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>2</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.702576</td>\n",
       "      <td>0.595974</td>\n",
       "      <td>0.644899</td>\n",
       "      <td>21558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323</th>\n",
       "      <td>xlnet-base-cased</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>3</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.700539</td>\n",
       "      <td>0.598977</td>\n",
       "      <td>0.645789</td>\n",
       "      <td>21500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>451</th>\n",
       "      <td>xlnet-base-cased</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>4</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.700089</td>\n",
       "      <td>0.589176</td>\n",
       "      <td>0.639862</td>\n",
       "      <td>21379</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               system     experiment  fold      class  precision    recall  \\\n",
       "67   xlnet-base-cased  ACL Anthology     1  micro avg   0.704042  0.584219   \n",
       "195  xlnet-base-cased  ACL Anthology     2  micro avg   0.702576  0.595974   \n",
       "323  xlnet-base-cased  ACL Anthology     3  micro avg   0.700539  0.598977   \n",
       "451  xlnet-base-cased  ACL Anthology     4  micro avg   0.700089  0.589176   \n",
       "\n",
       "     f1-score  support  \n",
       "67   0.638558    21646  \n",
       "195  0.644899    21558  \n",
       "323  0.645789    21500  \n",
       "451  0.639862    21379  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "for sys in sorted(systems):\n",
    "    print(sys)\n",
    "    display(df[(df['system'] == sys) & (df['class'] == 'micro avg')])\n",
    "    #display(df[(df['system'] == sys) & (df['class'] == 'macro avg')])\n",
    "    \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 364,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>system</th>\n",
       "      <th>experiment</th>\n",
       "      <th>fold</th>\n",
       "      <th>class</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>support</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>67</th>\n",
       "      <td>xlnet-base-cased</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>1</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.704042</td>\n",
       "      <td>0.584219</td>\n",
       "      <td>0.638558</td>\n",
       "      <td>21646</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>195</th>\n",
       "      <td>xlnet-base-cased</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>2</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.702576</td>\n",
       "      <td>0.595974</td>\n",
       "      <td>0.644899</td>\n",
       "      <td>21558</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>323</th>\n",
       "      <td>xlnet-base-cased</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>3</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.700539</td>\n",
       "      <td>0.598977</td>\n",
       "      <td>0.645789</td>\n",
       "      <td>21500</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>451</th>\n",
       "      <td>xlnet-base-cased</td>\n",
       "      <td>ACL Anthology</td>\n",
       "      <td>4</td>\n",
       "      <td>micro avg</td>\n",
       "      <td>0.700089</td>\n",
       "      <td>0.589176</td>\n",
       "      <td>0.639862</td>\n",
       "      <td>21379</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "               system     experiment  fold      class  precision    recall  \\\n",
       "67   xlnet-base-cased  ACL Anthology     1  micro avg   0.704042  0.584219   \n",
       "195  xlnet-base-cased  ACL Anthology     2  micro avg   0.702576  0.595974   \n",
       "323  xlnet-base-cased  ACL Anthology     3  micro avg   0.700539  0.598977   \n",
       "451  xlnet-base-cased  ACL Anthology     4  micro avg   0.700089  0.589176   \n",
       "\n",
       "     f1-score  support  \n",
       "67   0.638558    21646  \n",
       "195  0.644899    21558  \n",
       "323  0.645789    21500  \n",
       "451  0.639862    21379  "
      ]
     },
     "execution_count": 364,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df[(df['system'] == 'xlnet-base-cased') & (df['class'] == 'micro avg')]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 365,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['system', 'experiment', 'fold', 'class', 'precision', 'recall',\n",
       "       'f1-score', 'support'],\n",
       "      dtype='object')"
      ]
     },
     "execution_count": 365,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 440,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<style  type=\"text/css\" >\n",
       "    #T_72193890_a97a_11ea_b634_ac1f6bf53d0erow0_col8 {\n",
       "            background-color:  green;\n",
       "        }    #T_72193890_a97a_11ea_b634_ac1f6bf53d0erow0_col9 {\n",
       "            background-color:  green;\n",
       "        }    #T_72193890_a97a_11ea_b634_ac1f6bf53d0erow1_col7 {\n",
       "            background-color:  green;\n",
       "        }    #T_72193890_a97a_11ea_b634_ac1f6bf53d0erow2_col1 {\n",
       "            background-color:  green;\n",
       "        }    #T_72193890_a97a_11ea_b634_ac1f6bf53d0erow2_col2 {\n",
       "            background-color:  green;\n",
       "        }    #T_72193890_a97a_11ea_b634_ac1f6bf53d0erow3_col5 {\n",
       "            background-color:  green;\n",
       "        }    #T_72193890_a97a_11ea_b634_ac1f6bf53d0erow5_col11 {\n",
       "            background-color:  green;\n",
       "        }    #T_72193890_a97a_11ea_b634_ac1f6bf53d0erow6_col0 {\n",
       "            background-color:  green;\n",
       "        }    #T_72193890_a97a_11ea_b634_ac1f6bf53d0erow6_col4 {\n",
       "            background-color:  green;\n",
       "        }    #T_72193890_a97a_11ea_b634_ac1f6bf53d0erow6_col6 {\n",
       "            background-color:  green;\n",
       "        }    #T_72193890_a97a_11ea_b634_ac1f6bf53d0erow6_col10 {\n",
       "            background-color:  green;\n",
       "        }    #T_72193890_a97a_11ea_b634_ac1f6bf53d0erow7_col3 {\n",
       "            background-color:  green;\n",
       "        }</style><table id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0e\" ><thead>    <tr>        <th class=\"index_name level0\" >experiment</th>        <th class=\"col_heading level0 col0\" colspan=12>ACL Anthology</th>    </tr>    <tr>        <th class=\"index_name level1\" >class</th>        <th class=\"col_heading level1 col0\" colspan=6>macro avg</th>        <th class=\"col_heading level1 col6\" colspan=6>micro avg</th>    </tr>    <tr>        <th class=\"blank level2\" ></th>        <th class=\"col_heading level2 col0\" colspan=2>f1-score</th>        <th class=\"col_heading level2 col2\" colspan=2>precision</th>        <th class=\"col_heading level2 col4\" colspan=2>recall</th>        <th class=\"col_heading level2 col6\" colspan=2>f1-score</th>        <th class=\"col_heading level2 col8\" colspan=2>precision</th>        <th class=\"col_heading level2 col10\" colspan=2>recall</th>    </tr>    <tr>        <th class=\"blank level3\" ></th>        <th class=\"col_heading level3 col0\" >mean</th>        <th class=\"col_heading level3 col1\" >std</th>        <th class=\"col_heading level3 col2\" >mean</th>        <th class=\"col_heading level3 col3\" >std</th>        <th class=\"col_heading level3 col4\" >mean</th>        <th class=\"col_heading level3 col5\" >std</th>        <th class=\"col_heading level3 col6\" >mean</th>        <th class=\"col_heading level3 col7\" >std</th>        <th class=\"col_heading level3 col8\" >mean</th>        <th class=\"col_heading level3 col9\" >std</th>        <th class=\"col_heading level3 col10\" >mean</th>        <th class=\"col_heading level3 col11\" >std</th>    </tr>    <tr>        <th class=\"index_name level0\" >system</th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>        <th class=\"blank\" ></th>    </tr></thead><tbody>\n",
       "                <tr>\n",
       "                        <th id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0elevel0_row0\" class=\"row_heading level0 row0\" >baseline-rnn</th>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow0_col0\" class=\"data row0 col0\" >0.061683</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow0_col1\" class=\"data row0 col1\" >0.000719</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow0_col2\" class=\"data row0 col2\" >0.064348</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow0_col3\" class=\"data row0 col3\" >0.001687</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow0_col4\" class=\"data row0 col4\" >0.059252</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow0_col5\" class=\"data row0 col5\" >0.000681</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow0_col6\" class=\"data row0 col6\" >0.318319</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow0_col7\" class=\"data row0 col7\" >0.003878</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow0_col8\" class=\"data row0 col8\" >0.772175</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow0_col9\" class=\"data row0 col9\" >0.020243</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow0_col10\" class=\"data row0 col10\" >0.200511</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow0_col11\" class=\"data row0 col11\" >0.002825</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0elevel0_row1\" class=\"row_heading level0 row1\" >baseline-rnn__fasttext__custom</th>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow1_col0\" class=\"data row1 col0\" >0.059225</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow1_col1\" class=\"data row1 col1\" >0.001185</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow1_col2\" class=\"data row1 col2\" >0.099773</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow1_col3\" class=\"data row1 col3\" >0.011536</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow1_col4\" class=\"data row1 col4\" >0.054530</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow1_col5\" class=\"data row1 col5\" >0.001650</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow1_col6\" class=\"data row1 col6\" >0.297258</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow1_col7\" class=\"data row1 col7\" >0.007813</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow1_col8\" class=\"data row1 col8\" >0.766275</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow1_col9\" class=\"data row1 col9\" >0.012788</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow1_col10\" class=\"data row1 col10\" >0.184450</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow1_col11\" class=\"data row1 col11\" >0.006356</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0elevel0_row2\" class=\"row_heading level0 row2\" >bert-base-cased</th>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow2_col0\" class=\"data row2 col0\" >0.257502</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow2_col1\" class=\"data row2 col1\" >0.007443</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow2_col2\" class=\"data row2 col2\" >0.435477</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow2_col3\" class=\"data row2 col3\" >0.087384</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow2_col4\" class=\"data row2 col4\" >0.236415</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow2_col5\" class=\"data row2 col5\" >0.004551</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow2_col6\" class=\"data row2 col6\" >0.647711</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow2_col7\" class=\"data row2 col7\" >0.003220</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow2_col8\" class=\"data row2 col8\" >0.717877</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow2_col9\" class=\"data row2 col9\" >0.002484</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow2_col10\" class=\"data row2 col10\" >0.590047</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow2_col11\" class=\"data row2 col11\" >0.004178</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0elevel0_row3\" class=\"row_heading level0 row3\" >covid_bert_base</th>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow3_col0\" class=\"data row3 col0\" >0.246851</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow3_col1\" class=\"data row3 col1\" >0.007166</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow3_col2\" class=\"data row3 col2\" >0.368444</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow3_col3\" class=\"data row3 col3\" >0.031464</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow3_col4\" class=\"data row3 col4\" >0.226926</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow3_col5\" class=\"data row3 col5\" >0.006183</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow3_col6\" class=\"data row3 col6\" >0.637150</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow3_col7\" class=\"data row3 col7\" >0.004906</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow3_col8\" class=\"data row3 col8\" >0.712443</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow3_col9\" class=\"data row3 col9\" >0.003238</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow3_col10\" class=\"data row3 col10\" >0.576293</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow3_col11\" class=\"data row3 col11\" >0.007889</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0elevel0_row4\" class=\"row_heading level0 row4\" >electra-base-discriminator</th>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow4_col0\" class=\"data row4 col0\" >0.222802</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow4_col1\" class=\"data row4 col1\" >0.002680</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow4_col2\" class=\"data row4 col2\" >0.264237</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow4_col3\" class=\"data row4 col3\" >0.002626</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow4_col4\" class=\"data row4 col4\" >0.205030</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow4_col5\" class=\"data row4 col5\" >0.002318</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow4_col6\" class=\"data row4 col6\" >0.616833</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow4_col7\" class=\"data row4 col7\" >0.004227</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow4_col8\" class=\"data row4 col8\" >0.703179</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow4_col9\" class=\"data row4 col9\" >0.003466</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow4_col10\" class=\"data row4 col10\" >0.549380</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow4_col11\" class=\"data row4 col11\" >0.004987</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0elevel0_row5\" class=\"row_heading level0 row5\" >roberta-base</th>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow5_col0\" class=\"data row5 col0\" >0.228616</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow5_col1\" class=\"data row5 col1\" >0.003366</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow5_col2\" class=\"data row5 col2\" >0.262577</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow5_col3\" class=\"data row5 col3\" >0.003093</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow5_col4\" class=\"data row5 col4\" >0.210920</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow5_col5\" class=\"data row5 col5\" >0.004346</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow5_col6\" class=\"data row5 col6\" >0.620891</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow5_col7\" class=\"data row5 col7\" >0.005003</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow5_col8\" class=\"data row5 col8\" >0.704560</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow5_col9\" class=\"data row5 col9\" >0.002623</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow5_col10\" class=\"data row5 col10\" >0.555061</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow5_col11\" class=\"data row5 col11\" >0.009374</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0elevel0_row6\" class=\"row_heading level0 row6\" >scibert-scivocab-uncased</th>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow6_col0\" class=\"data row6 col0\" >0.304538</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow6_col1\" class=\"data row6 col1\" >0.003965</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow6_col2\" class=\"data row6 col2\" >0.432449</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow6_col3\" class=\"data row6 col3\" >0.033814</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow6_col4\" class=\"data row6 col4\" >0.275802</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow6_col5\" class=\"data row6 col5\" >0.004590</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow6_col6\" class=\"data row6 col6\" >0.669392</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow6_col7\" class=\"data row6 col7\" >0.003626</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow6_col8\" class=\"data row6 col8\" >0.724994</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow6_col9\" class=\"data row6 col9\" >0.003166</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow6_col10\" class=\"data row6 col10\" >0.621720</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow6_col11\" class=\"data row6 col11\" >0.004681</td>\n",
       "            </tr>\n",
       "            <tr>\n",
       "                        <th id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0elevel0_row7\" class=\"row_heading level0 row7\" >xlnet-base-cased</th>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow7_col0\" class=\"data row7 col0\" >0.242897</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow7_col1\" class=\"data row7 col1\" >0.003791</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow7_col2\" class=\"data row7 col2\" >0.327985</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow7_col3\" class=\"data row7 col3\" >0.126468</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow7_col4\" class=\"data row7 col4\" >0.230349</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow7_col5\" class=\"data row7 col5\" >0.003581</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow7_col6\" class=\"data row7 col6\" >0.642277</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow7_col7\" class=\"data row7 col7\" >0.003600</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow7_col8\" class=\"data row7 col8\" >0.701811</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow7_col9\" class=\"data row7 col9\" >0.001839</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow7_col10\" class=\"data row7 col10\" >0.592086</td>\n",
       "                        <td id=\"T_72193890_a97a_11ea_b634_ac1f6bf53d0erow7_col11\" class=\"data row7 col11\" >0.006657</td>\n",
       "            </tr>\n",
       "    </tbody></table>"
      ],
      "text/plain": [
       "<pandas.io.formats.style.Styler at 0x7fc408f58f10>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#agg_cols = {col_name:  ['mean', 'std', 'count'] for col_name, _ in metric_cols.items()}\n",
    "\n",
    "agg_cols = {col_name:  ['mean', 'std',] for col_name in ['precision', 'recall', 'f1-score']}\n",
    "\n",
    "overall_df = df[df['class'].isin(['micro avg', 'macro avg'])]\\\n",
    "    .groupby(['system','class','experiment', ])\\\n",
    "    .agg(agg_cols)\\\n",
    "    .unstack().unstack()\\\n",
    "    .swaplevel(0,2,axis=1).swaplevel(1,3,axis=1).sort_index(axis=1)\n",
    "    \n",
    "\n",
    "with pd.option_context('display.float_format', lambda x: ('%.3f' % x)[1:]):   \n",
    "    display(\n",
    "        overall_df.style.apply(highlight_max)\n",
    "    )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 370,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\\begin{tabular}{l|r|r|r|r|r|r}\n",
      "\\toprule\n",
      "experiment & \\multicolumn{3}{l}{ACL Anthology} \\\\\n",
      "class & \\multicolumn{3}{l}{macro avg} & \\multicolumn{3}{l}{micro avg} \\\\\n",
      "{} & \\multicolumn{1}{l}{f1-score} & \\multicolumn{1}{l}{precision} & \\multicolumn{1}{l}{recall} & \\multicolumn{1}{l}{f1-score} & \\multicolumn{1}{l}{precision} & \\multicolumn{1}{l}{recall} \\\\\n",
      "\n",
      "\n",
      "\n",
      "\\midrule\n",
      "\\rowcolor{Gray} \n",
      "\n",
      "baseline-rnn                   &          .062  $\\pm$.001 &      .064  $\\pm$.002 &   .059  $\\pm$.001 &      .318  $\\pm$.004 &      .772  $\\pm$.020 &   .201  $\\pm$.003\n",
      " \\\\\n",
      "\n",
      "baseline-rnn\\_\\_fasttext\\_\\_custom &          .059  $\\pm$.001 &      .100  $\\pm$.012 &   .055  $\\pm$.002 &      .297  $\\pm$.008 &      .766  $\\pm$.013 &   .184  $\\pm$.006\n",
      " \\\\\n",
      "\\rowcolor{Gray} \n",
      "\n",
      "bert-base-cased                &          .258  $\\pm$.007 &      .435  $\\pm$.087 &   .236  $\\pm$.005 &      .648  $\\pm$.003 &      .718  $\\pm$.002 &   .590  $\\pm$.004\n",
      " \\\\\n",
      "\n",
      "covid\\_bert\\_base                &          .247  $\\pm$.007 &      .368  $\\pm$.031 &   .227  $\\pm$.006 &      .637  $\\pm$.005 &      .712  $\\pm$.003 &   .576  $\\pm$.008\n",
      " \\\\\n",
      "\\rowcolor{Gray} \n",
      "\n",
      "electra-base-discriminator     &          .223  $\\pm$.003 &      .264  $\\pm$.003 &   .205  $\\pm$.002 &      .617  $\\pm$.004 &      .703  $\\pm$.003 &   .549  $\\pm$.005\n",
      " \\\\\n",
      "\n",
      "roberta-base                   &          .229  $\\pm$.003 &      .263  $\\pm$.003 &   .211  $\\pm$.004 &      .621  $\\pm$.005 &      .705  $\\pm$.003 &   .555  $\\pm$.009\n",
      " \\\\\n",
      "\\rowcolor{Gray} \n",
      "\n",
      "scibert-scivocab-uncased       &          .305  $\\pm$.004 &      .432  $\\pm$.034 &   .276  $\\pm$.005 &      .669  $\\pm$.004 &      .725  $\\pm$.003 &   .622  $\\pm$.005\n",
      " \\\\\n",
      "\n",
      "xlnet-base-cased               &          .243  $\\pm$.004 &      .328  $\\pm$.126 &   .230  $\\pm$.004 &      .642  $\\pm$.004 &      .702  $\\pm$.002 &   .592  $\\pm$.007\n",
      " \\\\\n",
      "\\rowcolor{Gray} \n",
      "\n",
      "\n",
      " \\\\\n",
      "\\bottomrule\n",
      "\n",
      "\\end{tabular}\n",
      "\n"
     ]
    }
   ],
   "source": [
    "# pd.set_option('display.float_format', lambda x: '%.3f' % x)\n",
    "\n",
    "with pd.option_context('display.float_format', lambda x: ('%.3f' % x)[1:]):   \n",
    "    overall_tex = overall_df.to_latex().split(r'\\midrule', 2)\n",
    "    body_tex, footer_tex = overall_tex[1].split(r'\\bottomrule', 2)    \n",
    "    header_tex = overall_tex[0]\n",
    "    \n",
    "    \n",
    "    # {} &       mean\n",
    "    \n",
    "    header_tex = re.sub(r'(.*)mean(.*)', '', header_tex)\n",
    "    header_tex = re.sub(r'(.*)system(.*)', '', header_tex)\n",
    "        \n",
    "    #print(re.search(r'(.*)mean(.*)', header_tex))\n",
    "    \n",
    "    print(\n",
    "        header_tex\\\n",
    "            .replace(r'multicolumn{2}{l}', r'multicolumn{1}{l}')\\\n",
    "            .replace(r'multicolumn{12}{l}', r'multicolumn{6}{l}')\\\n",
    "            .replace(r'multicolumn{6}{l}', r'multicolumn{3}{l}')\\\n",
    "            .replace('lrrrrrrrrrrrr', 'l|r|r|r|r|r|r')\n",
    "    )\n",
    "    print(r'\\midrule')\n",
    "    \n",
    "    for i, line in enumerate(body_tex.split(r' \\\\')):\n",
    "        if (i % 2) == 0:\n",
    "            print(r'\\rowcolor{Gray} ')\n",
    "        \n",
    "        print(\n",
    "            line.replace('& .', r' $\\pm$.')\n",
    "        )\n",
    "        print(r' \\\\')\n",
    "        \n",
    "    print(r'\\bottomrule')\n",
    "    print(footer_tex)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 371,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "FrozenList([['acl_docrel'], ['macro avg', 'micro avg'], ['f1-score', 'precision', 'recall'], ['mean', 'std']])"
      ]
     },
     "execution_count": 371,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "odf.columns.levels"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# By label class\n",
    "\n",
    "- label X: sys_1_prec, sys_1_rec, sys_1_f1, sys_2_prec, ..."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 443,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th>experiment</th>\n",
       "      <th colspan=\"6\" halign=\"left\">ACL Anthology</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>system</th>\n",
       "      <th colspan=\"3\" halign=\"left\">bert-base-cased</th>\n",
       "      <th colspan=\"3\" halign=\"left\">scibert-scivocab-uncased</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>f1-score</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "      <th>f1-score</th>\n",
       "      <th>precision</th>\n",
       "      <th>recall</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>class</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>background</th>\n",
       "      <td>.097</td>\n",
       "      <td>.524</td>\n",
       "      <td>.054</td>\n",
       "      <td>.246</td>\n",
       "      <td>.636</td>\n",
       "      <td>.156</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>conclusion</th>\n",
       "      <td>.002</td>\n",
       "      <td>.250</td>\n",
       "      <td>.001</td>\n",
       "      <td>.000</td>\n",
       "      <td>.000</td>\n",
       "      <td>.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>discussion</th>\n",
       "      <td>.002</td>\n",
       "      <td>.250</td>\n",
       "      <td>.001</td>\n",
       "      <td>.000</td>\n",
       "      <td>.000</td>\n",
       "      <td>.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>evaluation</th>\n",
       "      <td>.000</td>\n",
       "      <td>.000</td>\n",
       "      <td>.000</td>\n",
       "      <td>.004</td>\n",
       "      <td>.250</td>\n",
       "      <td>.002</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>experiment</th>\n",
       "      <td>.314</td>\n",
       "      <td>.517</td>\n",
       "      <td>.227</td>\n",
       "      <td>.366</td>\n",
       "      <td>.495</td>\n",
       "      <td>.292</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>introduction</th>\n",
       "      <td>.488</td>\n",
       "      <td>.551</td>\n",
       "      <td>.438</td>\n",
       "      <td>.521</td>\n",
       "      <td>.569</td>\n",
       "      <td>.480</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>method</th>\n",
       "      <td>.000</td>\n",
       "      <td>.000</td>\n",
       "      <td>.000</td>\n",
       "      <td>.000</td>\n",
       "      <td>.000</td>\n",
       "      <td>.000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>previous work</th>\n",
       "      <td>.051</td>\n",
       "      <td>.938</td>\n",
       "      <td>.027</td>\n",
       "      <td>.294</td>\n",
       "      <td>.718</td>\n",
       "      <td>.186</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>related work</th>\n",
       "      <td>.598</td>\n",
       "      <td>.648</td>\n",
       "      <td>.555</td>\n",
       "      <td>.641</td>\n",
       "      <td>.657</td>\n",
       "      <td>.625</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>results</th>\n",
       "      <td>.000</td>\n",
       "      <td>.000</td>\n",
       "      <td>.000</td>\n",
       "      <td>.012</td>\n",
       "      <td>.283</td>\n",
       "      <td>.006</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>.609</td>\n",
       "      <td>.638</td>\n",
       "      <td>.582</td>\n",
       "      <td>.629</td>\n",
       "      <td>.645</td>\n",
       "      <td>.615</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>none</th>\n",
       "      <td>.931</td>\n",
       "      <td>.911</td>\n",
       "      <td>.951</td>\n",
       "      <td>.942</td>\n",
       "      <td>.936</td>\n",
       "      <td>.949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>macro avg</th>\n",
       "      <td>.258</td>\n",
       "      <td>.435</td>\n",
       "      <td>.236</td>\n",
       "      <td>.305</td>\n",
       "      <td>.432</td>\n",
       "      <td>.276</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>micro avg</th>\n",
       "      <td>.648</td>\n",
       "      <td>.718</td>\n",
       "      <td>.590</td>\n",
       "      <td>.669</td>\n",
       "      <td>.725</td>\n",
       "      <td>.622</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>samples avg</th>\n",
       "      <td>.639</td>\n",
       "      <td>.665</td>\n",
       "      <td>.635</td>\n",
       "      <td>.669</td>\n",
       "      <td>.694</td>\n",
       "      <td>.667</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weighted avg</th>\n",
       "      <td>.611</td>\n",
       "      <td>.664</td>\n",
       "      <td>.590</td>\n",
       "      <td>.640</td>\n",
       "      <td>.676</td>\n",
       "      <td>.622</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "experiment      ACL Anthology                                            \\\n",
       "system        bert-base-cased                  scibert-scivocab-uncased   \n",
       "                     f1-score precision recall                 f1-score   \n",
       "class                                                                     \n",
       "background               .097      .524   .054                     .246   \n",
       "conclusion               .002      .250   .001                     .000   \n",
       "discussion               .002      .250   .001                     .000   \n",
       "evaluation               .000      .000   .000                     .004   \n",
       "experiment               .314      .517   .227                     .366   \n",
       "introduction             .488      .551   .438                     .521   \n",
       "method                   .000      .000   .000                     .000   \n",
       "previous work            .051      .938   .027                     .294   \n",
       "related work             .598      .648   .555                     .641   \n",
       "results                  .000      .000   .000                     .012   \n",
       "other                    .609      .638   .582                     .629   \n",
       "none                     .931      .911   .951                     .942   \n",
       "macro avg                .258      .435   .236                     .305   \n",
       "micro avg                .648      .718   .590                     .669   \n",
       "samples avg              .639      .665   .635                     .669   \n",
       "weighted avg             .611      .664   .590                     .640   \n",
       "\n",
       "experiment                      \n",
       "system                          \n",
       "              precision recall  \n",
       "class                           \n",
       "background         .636   .156  \n",
       "conclusion         .000   .000  \n",
       "discussion         .000   .000  \n",
       "evaluation         .250   .002  \n",
       "experiment         .495   .292  \n",
       "introduction       .569   .480  \n",
       "method             .000   .000  \n",
       "previous work      .718   .186  \n",
       "related work       .657   .625  \n",
       "results            .283   .006  \n",
       "other              .645   .615  \n",
       "none               .936   .949  \n",
       "macro avg          .432   .276  \n",
       "micro avg          .725   .622  \n",
       "samples avg        .694   .667  \n",
       "weighted avg       .676   .622  "
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "selected_system = 'scibert-scivocab-uncased'\n",
    "selected_systems = [\n",
    "    'bert-base-cased',\n",
    "    'scibert-scivocab-uncased'\n",
    "]\n",
    "\n",
    "label_df = df[df['system'].isin(selected_systems)]\\\n",
    "    .groupby(['class', 'experiment', 'system' ])\\\n",
    "    .agg({c: 'mean' for c in ['f1-score', 'precision', 'recall']})\\\n",
    "    .unstack()\\\n",
    "    .unstack()\\\n",
    "    .swaplevel(0,2,axis=1).sort_index(axis=1)\\\n",
    "    .reindex(sorted_label_classes)\n",
    "\n",
    "    #.drop(columns=[('ACL Anthology', 'fold'), ('ACL Anthology', 'support')])\\\n",
    "    #.reindex(sorted_label_classes)\n",
    "    \n",
    "with pd.option_context('display.float_format', lambda x: ('%.3f' % x)[1:]):  \n",
    "    display(label_df)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 372,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "set()"
      ]
     },
     "execution_count": 372,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "label_classes = set([c.split('__')[0] for c in df.columns if '__' in c and ' avg' not in c])\n",
    "label_classes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 189,
   "metadata": {},
   "outputs": [],
   "source": [
    "sorted_label_classes = [\n",
    "    'background',\n",
    "    'conclusion',\n",
    "    'discussion',\n",
    "    'evaluation',\n",
    "    'experiment',\n",
    "    'introduction',    \n",
    "    'method',\n",
    "    'previous work',\n",
    "    'related work',\n",
    "    'results',\n",
    "    'other',\n",
    "    'none',\n",
    "    'macro avg',\n",
    "    'micro avg',\n",
    "    'samples avg',\n",
    "    'weighted avg'\n",
    "]\n",
    "\n",
    "#systems = df['system'].unique().tolist()\n",
    "#footer_classes = ['other', 'none', 'macro avg', 'micro avg', ]\n",
    "\n",
    "\n",
    "by_label_class_rows = []\n",
    "\n",
    "for idx, row in df.iterrows():\n",
    "    for col in row.keys():\n",
    "        v = col.split('__')\n",
    "        \n",
    "        if len(v) == 2 and row['system'] in selected_systems:\n",
    "            by_label_class_rows.append({\n",
    "                'label': v[0],\n",
    "                row['system'] + '__' + v[1]: row[col],\n",
    "                #'fold': row['fold'],\n",
    "            })\n",
    "\n",
    "by_label_class_df = pd.DataFrame(by_label_class_rows)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 190,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([  341.25,   289.5 ,   283.  ,   242.75,  1006.25,  4069.75,\n",
       "       21520.75,   179.75, 21520.75,  6068.75,  5394.25,   200.25,\n",
       "        3150.  ,   295.25, 21520.75, 21520.75])"
      ]
     },
     "execution_count": 190,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "by_label_class_df.groupby('label').agg({selected_systems[0] + '__support': 'mean'}).values[:,0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 191,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr th {\n",
       "        text-align: left;\n",
       "    }\n",
       "\n",
       "    .dataframe thead tr:last-of-type th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th colspan=\"3\" halign=\"left\">bert-base-cased__precision</th>\n",
       "      <th colspan=\"3\" halign=\"left\">bert-base-cased__recall</th>\n",
       "      <th colspan=\"3\" halign=\"left\">bert-base-cased__f1-score</th>\n",
       "      <th>bert-base-cased__support</th>\n",
       "      <th>...</th>\n",
       "      <th>scibert-scivocab-uncased__precision</th>\n",
       "      <th colspan=\"3\" halign=\"left\">scibert-scivocab-uncased__recall</th>\n",
       "      <th colspan=\"3\" halign=\"left\">scibert-scivocab-uncased__f1-score</th>\n",
       "      <th colspan=\"3\" halign=\"left\">scibert-scivocab-uncased__support</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th></th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>...</th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "      <th>mean</th>\n",
       "      <th>std</th>\n",
       "      <th>count</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>label</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>background</th>\n",
       "      <td>0.524306</td>\n",
       "      <td>0.102925</td>\n",
       "      <td>4</td>\n",
       "      <td>0.053892</td>\n",
       "      <td>0.032251</td>\n",
       "      <td>4</td>\n",
       "      <td>0.096560</td>\n",
       "      <td>0.054327</td>\n",
       "      <td>4</td>\n",
       "      <td>341.25</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.155641</td>\n",
       "      <td>0.056622</td>\n",
       "      <td>4</td>\n",
       "      <td>0.245910</td>\n",
       "      <td>0.067291</td>\n",
       "      <td>4</td>\n",
       "      <td>341.25</td>\n",
       "      <td>5.057997</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>conclusion</th>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000842</td>\n",
       "      <td>0.001684</td>\n",
       "      <td>4</td>\n",
       "      <td>0.001678</td>\n",
       "      <td>0.003356</td>\n",
       "      <td>4</td>\n",
       "      <td>289.50</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>289.50</td>\n",
       "      <td>8.185353</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>discussion</th>\n",
       "      <td>0.250000</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000845</td>\n",
       "      <td>0.001689</td>\n",
       "      <td>4</td>\n",
       "      <td>0.001684</td>\n",
       "      <td>0.003367</td>\n",
       "      <td>4</td>\n",
       "      <td>283.00</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>283.00</td>\n",
       "      <td>15.556349</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>evaluation</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>242.75</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.001976</td>\n",
       "      <td>0.003953</td>\n",
       "      <td>4</td>\n",
       "      <td>0.003922</td>\n",
       "      <td>0.007843</td>\n",
       "      <td>4</td>\n",
       "      <td>242.75</td>\n",
       "      <td>8.920949</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>experiment</th>\n",
       "      <td>0.516850</td>\n",
       "      <td>0.040845</td>\n",
       "      <td>4</td>\n",
       "      <td>0.227432</td>\n",
       "      <td>0.027428</td>\n",
       "      <td>4</td>\n",
       "      <td>0.314036</td>\n",
       "      <td>0.019344</td>\n",
       "      <td>4</td>\n",
       "      <td>1006.25</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.291549</td>\n",
       "      <td>0.026184</td>\n",
       "      <td>4</td>\n",
       "      <td>0.365973</td>\n",
       "      <td>0.016732</td>\n",
       "      <td>4</td>\n",
       "      <td>1006.25</td>\n",
       "      <td>9.673848</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>introduction</th>\n",
       "      <td>0.550998</td>\n",
       "      <td>0.005105</td>\n",
       "      <td>4</td>\n",
       "      <td>0.437522</td>\n",
       "      <td>0.013988</td>\n",
       "      <td>4</td>\n",
       "      <td>0.487598</td>\n",
       "      <td>0.007017</td>\n",
       "      <td>4</td>\n",
       "      <td>4069.75</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.479862</td>\n",
       "      <td>0.005320</td>\n",
       "      <td>4</td>\n",
       "      <td>0.520798</td>\n",
       "      <td>0.001348</td>\n",
       "      <td>4</td>\n",
       "      <td>4069.75</td>\n",
       "      <td>20.645823</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>method</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>179.75</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>179.75</td>\n",
       "      <td>4.856267</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>previous work</th>\n",
       "      <td>0.937500</td>\n",
       "      <td>0.125000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.027428</td>\n",
       "      <td>0.031565</td>\n",
       "      <td>4</td>\n",
       "      <td>0.051268</td>\n",
       "      <td>0.056530</td>\n",
       "      <td>4</td>\n",
       "      <td>200.25</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.186333</td>\n",
       "      <td>0.048796</td>\n",
       "      <td>4</td>\n",
       "      <td>0.293741</td>\n",
       "      <td>0.065023</td>\n",
       "      <td>4</td>\n",
       "      <td>200.25</td>\n",
       "      <td>3.593976</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>related work</th>\n",
       "      <td>0.647622</td>\n",
       "      <td>0.013010</td>\n",
       "      <td>4</td>\n",
       "      <td>0.555243</td>\n",
       "      <td>0.013937</td>\n",
       "      <td>4</td>\n",
       "      <td>0.597880</td>\n",
       "      <td>0.013545</td>\n",
       "      <td>4</td>\n",
       "      <td>3150.00</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.625103</td>\n",
       "      <td>0.016830</td>\n",
       "      <td>4</td>\n",
       "      <td>0.640680</td>\n",
       "      <td>0.011552</td>\n",
       "      <td>4</td>\n",
       "      <td>3150.00</td>\n",
       "      <td>17.606817</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>results</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>4</td>\n",
       "      <td>295.25</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.006037</td>\n",
       "      <td>0.004381</td>\n",
       "      <td>4</td>\n",
       "      <td>0.011782</td>\n",
       "      <td>0.008494</td>\n",
       "      <td>4</td>\n",
       "      <td>295.25</td>\n",
       "      <td>11.295279</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>other</th>\n",
       "      <td>0.637806</td>\n",
       "      <td>0.007519</td>\n",
       "      <td>4</td>\n",
       "      <td>0.582426</td>\n",
       "      <td>0.009370</td>\n",
       "      <td>4</td>\n",
       "      <td>0.608787</td>\n",
       "      <td>0.003984</td>\n",
       "      <td>4</td>\n",
       "      <td>5394.25</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.614528</td>\n",
       "      <td>0.006158</td>\n",
       "      <td>4</td>\n",
       "      <td>0.629327</td>\n",
       "      <td>0.005177</td>\n",
       "      <td>4</td>\n",
       "      <td>5394.25</td>\n",
       "      <td>20.271080</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>none</th>\n",
       "      <td>0.910647</td>\n",
       "      <td>0.005430</td>\n",
       "      <td>4</td>\n",
       "      <td>0.951349</td>\n",
       "      <td>0.006443</td>\n",
       "      <td>4</td>\n",
       "      <td>0.930533</td>\n",
       "      <td>0.003214</td>\n",
       "      <td>4</td>\n",
       "      <td>6068.75</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.948589</td>\n",
       "      <td>0.004968</td>\n",
       "      <td>4</td>\n",
       "      <td>0.942321</td>\n",
       "      <td>0.003352</td>\n",
       "      <td>4</td>\n",
       "      <td>6068.75</td>\n",
       "      <td>0.500000</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>macro avg</th>\n",
       "      <td>0.435477</td>\n",
       "      <td>0.087384</td>\n",
       "      <td>4</td>\n",
       "      <td>0.236415</td>\n",
       "      <td>0.004551</td>\n",
       "      <td>4</td>\n",
       "      <td>0.257502</td>\n",
       "      <td>0.007443</td>\n",
       "      <td>4</td>\n",
       "      <td>21520.75</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.275802</td>\n",
       "      <td>0.004590</td>\n",
       "      <td>4</td>\n",
       "      <td>0.304538</td>\n",
       "      <td>0.003965</td>\n",
       "      <td>4</td>\n",
       "      <td>21520.75</td>\n",
       "      <td>111.950510</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>micro avg</th>\n",
       "      <td>0.717877</td>\n",
       "      <td>0.002484</td>\n",
       "      <td>4</td>\n",
       "      <td>0.590047</td>\n",
       "      <td>0.004178</td>\n",
       "      <td>4</td>\n",
       "      <td>0.647711</td>\n",
       "      <td>0.003220</td>\n",
       "      <td>4</td>\n",
       "      <td>21520.75</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.621720</td>\n",
       "      <td>0.004681</td>\n",
       "      <td>4</td>\n",
       "      <td>0.669392</td>\n",
       "      <td>0.003626</td>\n",
       "      <td>4</td>\n",
       "      <td>21520.75</td>\n",
       "      <td>111.950510</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>samples avg</th>\n",
       "      <td>0.665303</td>\n",
       "      <td>0.003255</td>\n",
       "      <td>4</td>\n",
       "      <td>0.635215</td>\n",
       "      <td>0.003804</td>\n",
       "      <td>4</td>\n",
       "      <td>0.639053</td>\n",
       "      <td>0.003270</td>\n",
       "      <td>4</td>\n",
       "      <td>21520.75</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.666768</td>\n",
       "      <td>0.004571</td>\n",
       "      <td>4</td>\n",
       "      <td>0.668702</td>\n",
       "      <td>0.003941</td>\n",
       "      <td>4</td>\n",
       "      <td>21520.75</td>\n",
       "      <td>111.950510</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>weighted avg</th>\n",
       "      <td>0.663731</td>\n",
       "      <td>0.012238</td>\n",
       "      <td>4</td>\n",
       "      <td>0.590047</td>\n",
       "      <td>0.004178</td>\n",
       "      <td>4</td>\n",
       "      <td>0.611471</td>\n",
       "      <td>0.003922</td>\n",
       "      <td>4</td>\n",
       "      <td>21520.75</td>\n",
       "      <td>...</td>\n",
       "      <td>4</td>\n",
       "      <td>0.621720</td>\n",
       "      <td>0.004681</td>\n",
       "      <td>4</td>\n",
       "      <td>0.639696</td>\n",
       "      <td>0.003785</td>\n",
       "      <td>4</td>\n",
       "      <td>21520.75</td>\n",
       "      <td>111.950510</td>\n",
       "      <td>4</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>16 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "              bert-base-cased__precision                  \\\n",
       "                                    mean       std count   \n",
       "label                                                      \n",
       "background                      0.524306  0.102925     4   \n",
       "conclusion                      0.250000  0.500000     4   \n",
       "discussion                      0.250000  0.500000     4   \n",
       "evaluation                      0.000000  0.000000     4   \n",
       "experiment                      0.516850  0.040845     4   \n",
       "introduction                    0.550998  0.005105     4   \n",
       "method                          0.000000  0.000000     4   \n",
       "previous work                   0.937500  0.125000     4   \n",
       "related work                    0.647622  0.013010     4   \n",
       "results                         0.000000  0.000000     4   \n",
       "other                           0.637806  0.007519     4   \n",
       "none                            0.910647  0.005430     4   \n",
       "macro avg                       0.435477  0.087384     4   \n",
       "micro avg                       0.717877  0.002484     4   \n",
       "samples avg                     0.665303  0.003255     4   \n",
       "weighted avg                    0.663731  0.012238     4   \n",
       "\n",
       "              bert-base-cased__recall                  \\\n",
       "                                 mean       std count   \n",
       "label                                                   \n",
       "background                   0.053892  0.032251     4   \n",
       "conclusion                   0.000842  0.001684     4   \n",
       "discussion                   0.000845  0.001689     4   \n",
       "evaluation                   0.000000  0.000000     4   \n",
       "experiment                   0.227432  0.027428     4   \n",
       "introduction                 0.437522  0.013988     4   \n",
       "method                       0.000000  0.000000     4   \n",
       "previous work                0.027428  0.031565     4   \n",
       "related work                 0.555243  0.013937     4   \n",
       "results                      0.000000  0.000000     4   \n",
       "other                        0.582426  0.009370     4   \n",
       "none                         0.951349  0.006443     4   \n",
       "macro avg                    0.236415  0.004551     4   \n",
       "micro avg                    0.590047  0.004178     4   \n",
       "samples avg                  0.635215  0.003804     4   \n",
       "weighted avg                 0.590047  0.004178     4   \n",
       "\n",
       "              bert-base-cased__f1-score                  \\\n",
       "                                   mean       std count   \n",
       "label                                                     \n",
       "background                     0.096560  0.054327     4   \n",
       "conclusion                     0.001678  0.003356     4   \n",
       "discussion                     0.001684  0.003367     4   \n",
       "evaluation                     0.000000  0.000000     4   \n",
       "experiment                     0.314036  0.019344     4   \n",
       "introduction                   0.487598  0.007017     4   \n",
       "method                         0.000000  0.000000     4   \n",
       "previous work                  0.051268  0.056530     4   \n",
       "related work                   0.597880  0.013545     4   \n",
       "results                        0.000000  0.000000     4   \n",
       "other                          0.608787  0.003984     4   \n",
       "none                           0.930533  0.003214     4   \n",
       "macro avg                      0.257502  0.007443     4   \n",
       "micro avg                      0.647711  0.003220     4   \n",
       "samples avg                    0.639053  0.003270     4   \n",
       "weighted avg                   0.611471  0.003922     4   \n",
       "\n",
       "              bert-base-cased__support  ...  \\\n",
       "                                  mean  ...   \n",
       "label                                   ...   \n",
       "background                      341.25  ...   \n",
       "conclusion                      289.50  ...   \n",
       "discussion                      283.00  ...   \n",
       "evaluation                      242.75  ...   \n",
       "experiment                     1006.25  ...   \n",
       "introduction                   4069.75  ...   \n",
       "method                          179.75  ...   \n",
       "previous work                   200.25  ...   \n",
       "related work                   3150.00  ...   \n",
       "results                         295.25  ...   \n",
       "other                          5394.25  ...   \n",
       "none                           6068.75  ...   \n",
       "macro avg                     21520.75  ...   \n",
       "micro avg                     21520.75  ...   \n",
       "samples avg                   21520.75  ...   \n",
       "weighted avg                  21520.75  ...   \n",
       "\n",
       "              scibert-scivocab-uncased__precision  \\\n",
       "                                            count   \n",
       "label                                               \n",
       "background                                      4   \n",
       "conclusion                                      4   \n",
       "discussion                                      4   \n",
       "evaluation                                      4   \n",
       "experiment                                      4   \n",
       "introduction                                    4   \n",
       "method                                          4   \n",
       "previous work                                   4   \n",
       "related work                                    4   \n",
       "results                                         4   \n",
       "other                                           4   \n",
       "none                                            4   \n",
       "macro avg                                       4   \n",
       "micro avg                                       4   \n",
       "samples avg                                     4   \n",
       "weighted avg                                    4   \n",
       "\n",
       "              scibert-scivocab-uncased__recall                  \\\n",
       "                                          mean       std count   \n",
       "label                                                            \n",
       "background                            0.155641  0.056622     4   \n",
       "conclusion                            0.000000  0.000000     4   \n",
       "discussion                            0.000000  0.000000     4   \n",
       "evaluation                            0.001976  0.003953     4   \n",
       "experiment                            0.291549  0.026184     4   \n",
       "introduction                          0.479862  0.005320     4   \n",
       "method                                0.000000  0.000000     4   \n",
       "previous work                         0.186333  0.048796     4   \n",
       "related work                          0.625103  0.016830     4   \n",
       "results                               0.006037  0.004381     4   \n",
       "other                                 0.614528  0.006158     4   \n",
       "none                                  0.948589  0.004968     4   \n",
       "macro avg                             0.275802  0.004590     4   \n",
       "micro avg                             0.621720  0.004681     4   \n",
       "samples avg                           0.666768  0.004571     4   \n",
       "weighted avg                          0.621720  0.004681     4   \n",
       "\n",
       "              scibert-scivocab-uncased__f1-score                  \\\n",
       "                                            mean       std count   \n",
       "label                                                              \n",
       "background                              0.245910  0.067291     4   \n",
       "conclusion                              0.000000  0.000000     4   \n",
       "discussion                              0.000000  0.000000     4   \n",
       "evaluation                              0.003922  0.007843     4   \n",
       "experiment                              0.365973  0.016732     4   \n",
       "introduction                            0.520798  0.001348     4   \n",
       "method                                  0.000000  0.000000     4   \n",
       "previous work                           0.293741  0.065023     4   \n",
       "related work                            0.640680  0.011552     4   \n",
       "results                                 0.011782  0.008494     4   \n",
       "other                                   0.629327  0.005177     4   \n",
       "none                                    0.942321  0.003352     4   \n",
       "macro avg                               0.304538  0.003965     4   \n",
       "micro avg                               0.669392  0.003626     4   \n",
       "samples avg                             0.668702  0.003941     4   \n",
       "weighted avg                            0.639696  0.003785     4   \n",
       "\n",
       "              scibert-scivocab-uncased__support                    \n",
       "                                           mean         std count  \n",
       "label                                                              \n",
       "background                               341.25    5.057997     4  \n",
       "conclusion                               289.50    8.185353     4  \n",
       "discussion                               283.00   15.556349     4  \n",
       "evaluation                               242.75    8.920949     4  \n",
       "experiment                              1006.25    9.673848     4  \n",
       "introduction                            4069.75   20.645823     4  \n",
       "method                                   179.75    4.856267     4  \n",
       "previous work                            200.25    3.593976     4  \n",
       "related work                            3150.00   17.606817     4  \n",
       "results                                  295.25   11.295279     4  \n",
       "other                                   5394.25   20.271080     4  \n",
       "none                                    6068.75    0.500000     4  \n",
       "macro avg                              21520.75  111.950510     4  \n",
       "micro avg                              21520.75  111.950510     4  \n",
       "samples avg                            21520.75  111.950510     4  \n",
       "weighted avg                           21520.75  111.950510     4  \n",
       "\n",
       "[16 rows x 24 columns]"
      ]
     },
     "execution_count": 191,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "by_label_class_df.groupby('label').agg(['mean', 'std', 'count']).reindex(sorted_label_classes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Samples"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 192,
   "metadata": {},
   "outputs": [
    {
     "ename": "FileNotFoundError",
     "evalue": "[Errno 2] File ./output/.ipynb_checkpoints/folds/1/bert-base-cased/results.csv does not exist: './output/.ipynb_checkpoints/folds/1/bert-base-cased/results.csv'",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mFileNotFoundError\u001b[0m                         Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-192-09952eedc1f5>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      3\u001b[0m \u001b[0mresults_cols\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m[\u001b[0m\u001b[0;34m'from_title'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'to_title'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'true'\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m'predicted'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      4\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 5\u001b[0;31m \u001b[0mresults_df\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msys_run_results_fp\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      6\u001b[0m \u001b[0mresults_df\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mhead\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda2/envs/acl-anthology/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36mparser_f\u001b[0;34m(filepath_or_buffer, sep, delimiter, header, names, index_col, usecols, squeeze, prefix, mangle_dupe_cols, dtype, engine, converters, true_values, false_values, skipinitialspace, skiprows, skipfooter, nrows, na_values, keep_default_na, na_filter, verbose, skip_blank_lines, parse_dates, infer_datetime_format, keep_date_col, date_parser, dayfirst, cache_dates, iterator, chunksize, compression, thousands, decimal, lineterminator, quotechar, quoting, doublequote, escapechar, comment, encoding, dialect, error_bad_lines, warn_bad_lines, delim_whitespace, low_memory, memory_map, float_precision)\u001b[0m\n\u001b[1;32m    674\u001b[0m         )\n\u001b[1;32m    675\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 676\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0m_read\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfilepath_or_buffer\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    677\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    678\u001b[0m     \u001b[0mparser_f\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m__name__\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda2/envs/acl-anthology/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_read\u001b[0;34m(filepath_or_buffer, kwds)\u001b[0m\n\u001b[1;32m    446\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    447\u001b[0m     \u001b[0;31m# Create the parser.\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 448\u001b[0;31m     \u001b[0mparser\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mTextFileReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mfp_or_buf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    449\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    450\u001b[0m     \u001b[0;32mif\u001b[0m \u001b[0mchunksize\u001b[0m \u001b[0;32mor\u001b[0m \u001b[0miterator\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda2/envs/acl-anthology/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, f, engine, **kwds)\u001b[0m\n\u001b[1;32m    878\u001b[0m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"has_index_names\"\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    879\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 880\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mengine\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    881\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    882\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclose\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda2/envs/acl-anthology/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m_make_engine\u001b[0;34m(self, engine)\u001b[0m\n\u001b[1;32m   1112\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_make_engine\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mengine\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m\"c\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1113\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"c\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1114\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_engine\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mCParserWrapper\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mf\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0moptions\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1115\u001b[0m         \u001b[0;32melse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1116\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0mengine\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m\"python\"\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/miniconda2/envs/acl-anthology/lib/python3.7/site-packages/pandas/io/parsers.py\u001b[0m in \u001b[0;36m__init__\u001b[0;34m(self, src, **kwds)\u001b[0m\n\u001b[1;32m   1889\u001b[0m         \u001b[0mkwds\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m\"usecols\"\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0musecols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1890\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m-> 1891\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mparsers\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mTextReader\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0msrc\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m**\u001b[0m\u001b[0mkwds\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m   1892\u001b[0m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_reader\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0munnamed_cols\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m   1893\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader.__cinit__\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;32mpandas/_libs/parsers.pyx\u001b[0m in \u001b[0;36mpandas._libs.parsers.TextReader._setup_parser_source\u001b[0;34m()\u001b[0m\n",
      "\u001b[0;31mFileNotFoundError\u001b[0m: [Errno 2] File ./output/.ipynb_checkpoints/folds/1/bert-base-cased/results.csv does not exist: './output/.ipynb_checkpoints/folds/1/bert-base-cased/results.csv'"
     ]
    }
   ],
   "source": [
    "sys_run_results_fp = os.path.join(exp_dir, 'folds', '1', 'bert-base-cased', 'results.csv')\n",
    "\n",
    "results_cols = ['from_title', 'to_title', 'true', 'predicted']\n",
    "\n",
    "results_df = pd.read_csv(sys_run_results_fp)\n",
    "results_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df[results_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "without_predictions = pd.isnull(results_df['predicted']).sum()\n",
    "\n",
    "print(f'Samples for that no probabilty was above classification threshold: {without_predictions:,} / {len(results_df):,}')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df[results_df['predicted_background'] > 0][results_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df[results_df['true_background'] > 0][results_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df[results_df['predicted_related work'] > 0][results_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df[results_df['true_related work'] > 0][results_cols]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df['true']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df['true'].value_counts()[:20]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df['predicted'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "results_df[results_df['predicted_related work'] > 0][results_cols]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Multi-label confusion\n",
    "\n",
    "```\n",
    "\n",
    "missed classification for... | predicted labels                                | true labels\n",
    "                                introduction   | background | experiment | ... |\n",
    "introduction                 |        -        | 12         | 3          | \n",
    "(true_introduction=1,predicted_introductioin=0)\n",
    "\n",
    "\n",
    "```\n",
    "\n",
    "(color background min/max value)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}