adasyn_other_categories_classification.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "HRLNxJQK7KV5"
      },
      "outputs": [],
      "source": [
        "import pandas as pd\n",
        "from sklearn.model_selection import train_test_split\n",
        "from sklearn.feature_extraction.text import CountVectorizer\n",
        "from sklearn.model_selection import GridSearchCV\n",
        "import matplotlib.pyplot as plt\n",
        "import seaborn as sns\n",
        "import numpy as np\n",
        "from sklearn.metrics import classification_report\n",
        "from sklearn.linear_model import LogisticRegression\n",
        "from sklearn.model_selection import cross_validate\n",
        "from sklearn.metrics import accuracy_score, f1_score, precision_score, recall_score,roc_auc_score, confusion_matrix, matthews_corrcoef, make_scorer\n",
        "from imblearn.over_sampling import *\n",
        "import pickle"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df = pd.read_csv(\"/content/Latest_Merged_Coded.csv\",  encoding='ISO-8859–1')\n",
        "df.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 206
        },
        "id": "TXk7IyUN_CSb",
        "outputId": "899a7514-0155-46f7-efbb-5fa1e816b309"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "   Unnamed: 0   index                            command category coder  \\\n",
              "0           0  522355  play hymns the sun by the beatles    music     G   \n",
              "1           1  314268              alexa what time is it   search     G   \n",
              "2           2  187019                         alexa stop    music     G   \n",
              "3           3  141151                               play    music     G   \n",
              "4           4  402967        can you say that joke again     joke     G   \n",
              "\n",
              "   make_code  alarm  joke  misc  music  polite  search  volume  weather  \n",
              "0          4    0.0   0.0     0    1.0     0.0     0.0       0      0.0  \n",
              "1          7    0.0   0.0     0    0.0     0.0     1.0       0      0.0  \n",
              "2          4    0.0   0.0     0    1.0     0.0     0.0       0      0.0  \n",
              "3          4    0.0   0.0     0    1.0     0.0     0.0       0      0.0  \n",
              "4          2    0.0   1.0     0    0.0     0.0     0.0       0      0.0  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e0fc0632-2c8e-4062-97d0-eab7ffedba42\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>Unnamed: 0</th>\n",
              "      <th>index</th>\n",
              "      <th>command</th>\n",
              "      <th>category</th>\n",
              "      <th>coder</th>\n",
              "      <th>make_code</th>\n",
              "      <th>alarm</th>\n",
              "      <th>joke</th>\n",
              "      <th>misc</th>\n",
              "      <th>music</th>\n",
              "      <th>polite</th>\n",
              "      <th>search</th>\n",
              "      <th>volume</th>\n",
              "      <th>weather</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>0</td>\n",
              "      <td>522355</td>\n",
              "      <td>play hymns the sun by the beatles</td>\n",
              "      <td>music</td>\n",
              "      <td>G</td>\n",
              "      <td>4</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>1</td>\n",
              "      <td>314268</td>\n",
              "      <td>alexa what time is it</td>\n",
              "      <td>search</td>\n",
              "      <td>G</td>\n",
              "      <td>7</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>2</td>\n",
              "      <td>187019</td>\n",
              "      <td>alexa stop</td>\n",
              "      <td>music</td>\n",
              "      <td>G</td>\n",
              "      <td>4</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>3</td>\n",
              "      <td>141151</td>\n",
              "      <td>play</td>\n",
              "      <td>music</td>\n",
              "      <td>G</td>\n",
              "      <td>4</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>4</td>\n",
              "      <td>402967</td>\n",
              "      <td>can you say that joke again</td>\n",
              "      <td>joke</td>\n",
              "      <td>G</td>\n",
              "      <td>2</td>\n",
              "      <td>0.0</td>\n",
              "      <td>1.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0.0</td>\n",
              "      <td>0</td>\n",
              "      <td>0.0</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e0fc0632-2c8e-4062-97d0-eab7ffedba42')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e0fc0632-2c8e-4062-97d0-eab7ffedba42 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e0fc0632-2c8e-4062-97d0-eab7ffedba42');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 3
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "df.at[1130,'category']='volume'\n",
        "df.at[298,'category']='misc'\n",
        "df.at[92,'category']='misc'\n",
        "df.at[1816,'category']='misc'\n",
        "df = df[df['category']!='random']\n",
        "df['command']  = df['command'].astype('str')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "4E4pRn4R_GxO",
        "outputId": "0270c728-e794-4c61-dd3e-78feec6c7607"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-4-f0461d694083>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  df['command']  = df['command'].astype('str')\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "X = df['command']\n",
        "y = df['category']\n",
        "vectorizer = CountVectorizer()\n",
        "vectors = vectorizer.fit_transform(X)"
      ],
      "metadata": {
        "id": "XjAuiKzWh6sf"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "adasyn = ADASYN(sampling_strategy='minority')\n",
        "for _ in range(6):\n",
        "  vectors, y = adasyn.fit_resample(vectors, y)"
      ],
      "metadata": {
        "id": "-0mp6RtskjNV"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "y.value_counts()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sMh5puB4iXze",
        "outputId": "33a8795d-5faa-4f9d-b64f-1ba592b4198e"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "search     487\n",
              "weather    478\n",
              "joke       476\n",
              "music      474\n",
              "polite     473\n",
              "volume     473\n",
              "alarm      472\n",
              "misc       462\n",
              "Name: category, dtype: int64"
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "train_vectors, test_vectors, y_train, y_test = train_test_split(vectors, y, test_size=0.2, random_state=40)"
      ],
      "metadata": {
        "id": "qPh-bzK0k2Ah"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "parameters = {\n",
        "    'penalty' : ['l2'], \n",
        "    'C'       : np.logspace(-3,3,7),\n",
        "    'solver'  : ['lbfgs', 'liblinear'],\n",
        "}\n",
        "logreg = LogisticRegression(max_iter = 10000)\n",
        "clf = GridSearchCV(logreg,                    \n",
        "                   param_grid = parameters,   \n",
        "                   scoring='roc_auc_ovr_weighted',        \n",
        "                   cv=10)"
      ],
      "metadata": {
        "id": "uV2lFluUlI1q"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "clf.fit(train_vectors,y_train)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "7152e53a-ddd2-486c-c019-6297345ab575",
        "id": "ZZGmw43nlI1s"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "GridSearchCV(cv=10, estimator=LogisticRegression(max_iter=10000),\n",
              "             param_grid={'C': array([1.e-03, 1.e-02, 1.e-01, 1.e+00, 1.e+01, 1.e+02, 1.e+03]),\n",
              "                         'penalty': ['l2'], 'solver': ['lbfgs', 'liblinear']},\n",
              "             scoring='roc_auc_ovr_weighted')"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"Tuned Hyperparameters :\", clf.best_params_)\n",
        "print(\"roc_auc_ovr_weighted :\",clf.best_score_)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "outputId": "37083ec7-2b5f-410b-8443-1434d6d34419",
        "id": "tcHRaNp3lI1t"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Tuned Hyperparameters : {'C': 1.0, 'penalty': 'l2', 'solver': 'lbfgs'}\n",
            "roc_auc_ovr_weighted : 0.9793634871162362\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "best_logreg = LogisticRegression(class_weight = 'balanced',max_iter = 10000,C=1,penalty='l2',solver='lbfgs')\n",
        "best_logreg.fit(train_vectors,y_train)\n",
        "pred = best_logreg.predict(test_vectors)"
      ],
      "metadata": {
        "id": "MtTE7e1BlI1u"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "my_tags = y.value_counts().index\n",
        "print(classification_report(y_test, pred))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "VlutXqcrlYmg",
        "outputId": "70f5bf94-cf61-4d72-c151-03a80e367032"
      },
      "execution_count": 31,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "              precision    recall  f1-score   support\n",
            "\n",
            "       alarm       0.99      0.89      0.94        92\n",
            "        joke       0.99      0.86      0.92        85\n",
            "        misc       0.86      0.70      0.77       100\n",
            "       music       0.90      0.80      0.85        91\n",
            "      polite       0.50      1.00      0.67        95\n",
            "      search       0.87      0.82      0.84        88\n",
            "      volume       0.97      0.79      0.87       106\n",
            "     weather       0.88      0.69      0.77       102\n",
            "\n",
            "    accuracy                           0.82       759\n",
            "   macro avg       0.87      0.82      0.83       759\n",
            "weighted avg       0.87      0.82      0.83       759\n",
            "\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "x_axis_labels = y.value_counts().index\n",
        "y_axis_labels = y.value_counts().index\n",
        "cf_matrix = confusion_matrix(y_test, pred)\n",
        "plt.figure(figsize=(8, 8))\n",
        "sns.heatmap(cf_matrix, annot=True, fmt='g', vmin=9999999, vmax=9999999, linewidths=.5,cbar=False, xticklabels=x_axis_labels, yticklabels=y_axis_labels).set(title=\"Confusion Matrix\")\n",
        "plt.show()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 499
        },
        "id": "pD0hsz4I_oct",
        "outputId": "a080a8ef-200b-49fa-cb33-622bd2491125"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "display_data",
          "data": {
            "text/plain": [
              "<Figure size 576x576 with 1 Axes>"
            ],
            "image/png": "iVBORw0KGgoAAAANSUhEUgAAAd0AAAHiCAYAAACtERYWAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADh0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uMy4yLjIsIGh0dHA6Ly9tYXRwbG90bGliLm9yZy+WH4yJAAAgAElEQVR4nOzdeXwVhbn/8c8TEhCosogKARQEcW0VBUGKFsQCLqjdcKlarZZWUPT259pqq3a11lZvr/dqsO6CgFUriICCimgFogmyVstiTAJFRFFwCQnP748zSSMGMpAzM8nwfb9e55UzM+ec+T6ZyXkyc2bmmLsjIiIi0ctJOoCIiMjuQk1XREQkJmq6IiIiMVHTFRERiYmaroiISEzUdEVERGKipisSMzNraWZTzGyjmU1uwOt838xmZjNbEszsWTP7QdI5ROKgpiuyHWZ2rpkVmtkmM1sTNIeBWXjp7wL7AXu7+/d29UXc/VF3H5qFPF9gZoPMzM3syW3GHxmMfzHk69xkZo/U9zh3P9ndH9zFuCJNipquSB3M7KfAHcBvyTTI/YH/Bc7IwssfALzl7pVZeK2ovAccZ2Z71xr3A+CtbM3AMvQeJLsVrfAi2zCzNsAtwBh3f8LdN7v7Fnef4u5XB49pYWZ3mFl5cLvDzFoE0waZWamZ/T8zWxdsJV8UTLsZ+AVwVrAFffG2W4Rm1i3YoswNhi80s5Vm9rGZrTKz79caP7fW8waY2YJgt/UCMxtQa9qLZvYrM3sleJ2ZZtZhB7+GCuAp4Ozg+c2As4BHt/ld3Wlm75rZR2b2upkdH4wfDvysVp0La+X4jZm9AnwCHBiMuySY/n9m9rdar3+rmc0yMwu9AEUaMTVdkS87DtgDeHIHj/k50B84CjgSOBa4odb0jkAboDNwMXCXmbVz91+S2Xqe6O5fcfe/7iiImbUG/hs42d33BAYAxXU8rj3wTPDYvYE/Ac9ss6V6LnARsC/QHLhqR/MGHgIuCO4PAxYD5ds8ZgGZ30F7YDww2cz2cPfp29R5ZK3nnA+MAvYE3tnm9f4f8NXgH4rjyfzufuC6Xq2khJquyJftDayvZ/fv94Fb3H2du78H3EymmVTbEkzf4u7TgE3AwbuYZytwhJm1dPc17r6kjsecCrzt7g+7e6W7TwCWAyNqPeZ+d3/L3T8FJpFpltvl7q8C7c3sYDLN96E6HvOIu78fzPN2oAX11/mAuy8JnrNlm9f7hMzv8U/AI8Dl7l5az+uJNBlquiJf9j7QoXr37nbk88WttHeCcTWvsU3T/gT4ys4GcffNZHbr/gRYY2bPmNkhIfJUZ+pca3jtLuR5GLgMGEwdW/5mdpWZLQt2aX9IZut+R7utAd7d0UR3nwesBIzMPwciqaGmK/Jl/wA+B87cwWPKyRwQVW1/vrzrNazNQKtawx1rT3T3Ge7+TaATma3XcSHyVGcq28VM1R4GRgPTgq3QGsHu32uAkUA7d28LbCTTLAG2t0t4h7uKzWwMmS3m8uD1RVJDTVdkG+6+kczBTneZ2Zlm1srM8szsZDP7Q/CwCcANZrZPcEDSL8jsDt0VxcAJZrZ/cBDX9dUTzGw/Mzsj+Gz3czK7qbfW8RrTgF7BaU65ZnYWcBgwdRczAeDuq4BvkPkMe1t7ApVkjnTONbNfAHvVmv5voNvOHKFsZr2AXwPnkdnNfI2Z7XA3uEhToqYrUofg88mfkjk46j0yu0QvI3NEL2QaQyHwJrAIeCMYtyvzeg6YGLzW63yxUeYEOcqBDWQa4KV1vMb7wGlkDkR6n8wW4mnuvn5XMm3z2nPdva6t+BnAdDKnEb0DfMYXdx1XX/jjfTN7o775BLvzHwFudfeF7v42mSOgH64+MlykqTMdFCgiIhIPbemKiIjERE1XREQkJmq6IiIiMVHTFRERiYmaroiISEx2dMWdbNHh0SIisrup80s64mi65DbvXP+DmqDKiszFflRf01RZUZba2iDd9e0O6yaovqaqur66aPeyiIhITNR0RUREYqKmKyIiEhM1XRERkZio6YqIiMRETVdERCQmaroiIiIxUdMVERGJiZquiIhITNR0RUREYqKmKyIiEpPQ1142swFAt9rPcfeHIsgkIiKSSqGarpk9DPQAioGqYLQDaroiIiIhhd3S7QMc5u76mj4REZFdFPYz3cVAxyiDiIiIpN0Ot3TNbAqZ3ch7AkvNbD7wefV0dz892ngiIiLpUd/u5T/GkkJERGQ3sMOm6+4vAZhZd2CNu38WDLcE9os+noiISHqE/Ux3MrC11nBVME5ERERCCtt0c929onoguN88mkgiIiLpFLbpvmdmNQdNmdkZwPpoIoV3xdgfsbB4NsVFs3jk4bto0aIFDz34F5YsnkNx0SzGFdxObm7o6380asOGDmLJ4jksXzqXa64ek3ScrFN9Tcu4gtspL11IcdGsmnHt2rVl+rQJLFsyl+nTJtC2bZsEE2ZX2pZfbWmuDRpffWGb7k+An5lZiZmVANcCo6KLVb/8/I5cNuaH9Ot/Ckf1HkKzZs04a+QZTJjwJIcfcQJH9R5Cy5Z7cPEPz00yZlbk5OTw33f+htNGnMdXjxzMWWedyaGHHpR0rKxRfU3PQw9N4tTTvv+FcddeM4bZL8zl0MMHMvuFuVx7TfJvcNmQxuVXLc21QeOsr96ma2bNgEvdvT9wGJmLZAxw9xWRp6tHbm4uLVvuQbNmzWjVsiVr1qzl2emza6YvWFBMly6dEkyYHcf27c2KFatZtaqELVu2MGnS3zl9xLCkY2WN6mt6Xp47jw0ffPiFcSNGDOOhhzOHejz08GROP314EtGyLo3Lr1qaa4PGWV+9Tdfdq4CBwf1N7r4p8lQhlJev5U9/vptVK+ZTWlLExo8+4rnn59RMz83N5fvf/w4zZryQYMrsyO/ckXdLy2uGS8vWkJ+fnmuVqL502G/fDqxduw6AtWvXsd++HRJOlB1pXn5prg0aZ31hdy8XmdnTZna+mX27+hZpsnq0bduG00cMo2ev/nQ94Ghat27Fuef+J9L//OW3vPzyPOa+Mj/BlCK7L101VuTLwjbdPYD3gROBEcHttO092MxGmVmhmRUWFBQ0PGUdhgw5nlWrS1i/fgOVlZU8+dSzHNe/DwA33vBf7LPP3lx19U2RzDtu5WVr6dolv2a4S+dOlJevTTBRdqm+dPj3uvV07LgvAB077su6995POFF2pHn5pbk2aJz1hWq67n5RHbcf7uDxBe7ex937jBoVzfFW75aU0a/f0bRsuQcAJw4eyPLlb/PDi85h6DcH8f3zxqTmP+0FhcX07Nmdbt26kpeXx8iRZzBl6sykY2WN6kuHqVNmcsH53wPggvO/x5QpMxJOlB1pXn5prg0aZ31hv9pvD+Bi4HAyW70A7KjxRm3+giKeeOIZFsyfQWVlJcXFSxh376N89OHbvPNOKXNffhqAp56axq9/c0dSMbOiqqqKK668gWnPjKdZTg4PPDiRpUvfSjpW1qi+pueRh+/iGyccR4cO7Vm9spCbb/kjt952F4+Nv5uLLjyHkpJSzj73J0nHzIo0Lr9qaa4NGmd9FmZr0MwmA8uBc4FbgO8Dy9z9ihDz8NzmnRsUsrGqrCgDQPU1TZUVZamtDdJd3+6wboLqa6qC+qyuaWE/0+3p7jcCm939QeBUoF924omIiOwewjbdLcHPD83sCKANsG80kURERNIp7DUSC8ysHXAj8DTwFeAXkaUSERFJoVBN193vDe6+BBwYXRwREZH0CrV72cz2M7O/mtmzwfBhZnZxtNFERETSJexnug8AM4Dqs4zfAq6MIpCIiEhahW26Hdx9EsEX2bt7JZkvshcREZGQwjbdzWa2N+AAZtYf2BhZKhERkRQKe/TyT8kctdzDzF4B9gG+G1kqERGRFAq7pdsDOBkYQOaz3bcJ37BFRESE8E33Rnf/CGgHDAb+F/i/yFKJiIikUNimW33Q1KnAOHd/BmgeTSQREZF0Ctt0y8zsHuAsYJqZtdiJ54qIiAjhG+dIMp/lDnP3D4H2wNWRpRIREUmhsJeB/AR4otbwGmBNVKFERETSSLuIRUREYqKmKyIiEhM1XRERkZio6YqIiMRETVdERCQmaroiIiIxUdMVERGJiZquiIhITMzdo55H5DMQERFpZKyukbF8PV9u885xzCZ2lRVlgOprqiorylJbG6S7vt1h3QTV11RV11cX7V4WERGJiZquiIhITNR0RUREYqKmKyIiEhM1XRERkZio6YqIiMRETVdERCQm9TZdM2tmZi/EEUZERCTN6m267l4FbDWzNjHkERERSa2wV6TaBCwys+eAzdUj3X1sJKlERERSKGzTfSK4iYiIyC4K1XTd/UEzawns7+7/jDiTiIhIKoU6etnMRgDFwPRg+CgzezrKYCIiImkT9pShm4BjgQ8B3L0YODCiTCIiIqkUtulucfeN24zbmu0wIiIiaRb2QKolZnYu0MzMDgLGAq9GF0tERCR9wm7pXg4cDnwOTAA+Aq6MKpSIiEgahT16+RPg58FNREREdkGopmtmvYCrgG61n+PuJ0YTS0REJH3CfqY7GbgbuBeoii6OiIhIeoVtupXu/n+RJhEREUm5HR5IZWbtzaw9MMXMRptZp+pxwfhGZdjQQSxZPIflS+dyzdVjko7TIL169aBwwcya24b1yxl7+SXcfNPVvPH6cxQumMmzz4ynU6f9ko6aFWladnVJW33jCm6nvHQhxUWzasbd+rsbWLzoJd54/Tken3wvbdrslWDC7Erb8qstzbVB46vP3H37E81WAQ5YHZPd3cNcIMNzm3fexXjh5eTksGzJyww/5RxKS9fw2j+mcd75o1m27O3I5llZUQZA1PXl5ORQsvp1Bgw8jQ8+2MjHH28C4LIxP+TQQ3sx5rLrIplvnPXFvewgU19a102Itr7jB/Zj06bN3H//nRzVewgA3zzpBGa/8ApVVVX87rc/A+D6n/02kvnHtW5C+t9bklo3IfX11dU3d7yl6+7dg8Z6aHC/5gYclv2ou+7Yvr1ZsWI1q1aVsGXLFiZN+junjxiWdKysGHLiQFaufIeSkrKahgvQunUrdvRPU1OR5mUH6azv5bnz2PDBh18Y99zzc6iqyhzy8dq8N+jcuVMS0bIujcuvWpprg8ZZX9jzdOu6EEajujhGfueOvFtaXjNcWraG/PyOCSbKnpEjz+CxiU/VDP/qlmtZtWIB55zzLW66+bYEk2VHmpcdpL++ulx04dlMn/FC0jGyIs3LL821QeOsr77PdDua2TFASzPrbWZHB7dBQKsdPG+UmRWaWWFBQUGWI+9e8vLyGHHaUB7/29SacTf+4la69+jLhAlPMmb0RQmmE/my668bS2VlJePH69tARbZV39HLw4ALgS7An2qN/xj42fae5O4FQHW39dGX3dyAiOGUl62la5f8muEunTtRXr428vlGbfjwwRQVLWLduvVfmjZ+whNMefphbr7l9gSSZU9al121tNdX2wXnj+TUU07im8NGJh0la9K8/NJcGzTO+ur7TPdBdx8MXOjug2vdTnf3RvVv7ILCYnr27E63bl3Jy8tj5MgzmDJ1ZtKxGuzss878wq7lnj2719w/fcQw/vnPFUnEyqq0Lrtqaa+v2rChg7jqqks589sX8umnnyUdJ2vSvPzSXBs0zvrCXgbyb2Z2KpnrL+9Ra/wtUQXbWVVVVVxx5Q1Me2Y8zXJyeODBiSxd+lbSsRqkVauWnDTkBC4dfW3NuN/+5np69erB1q1bKSkpY/SYaI5cjlMal11taazvkYfv4hsnHEeHDu1ZvbKQm2/5I9decxktWrRg+rOPATBv3huRHVkfpzQuv2pprg0aZ307PGWo5kFmd5P5DHcwmatSfReY7+4Xh5hHLKcMJSHO0xaSsDvUl9baIN317Q7rJqi+pmqXTxmqZYC7XwB84O43A8cBvbITT0REZPcQtul+Gvz8xMzygS1AOk7CExERiUnYay9PNbO2wG3AG2SuUnVvZKlERERSKOyBVL8K7v7NzKYCe7j7xuhiiYiIpE+o3ctm1srMbjSzce7+ObCvmZ0WcTYREZFUCfuZ7v3A52QOoAIoA34dSSIREZGUCtt0e7j7H8gcQIW7f8J2DocWERGRuoVtuhVm1pLMAVSYWQ8yW74iIiISUtijl38JTAe6mtmjwNfJXJNZREREQgrbdH8APAM8DqwErnD3L1+BX0RERLYrbNP9K3A88E2gB1BkZnPc/c7IkomIiKRM2PN0XzCzOUBfMtdf/gmZLz9Q0xUREQkpVNM1s1lAa+AfwMtAX3dfF2UwERGRtAl79PKbQAVwBPA14IjgaGYREREJKezu5f8CMLM9yRy1fD/QEWgRWTIREZGUCbt7+TIyB1IdA6wG7iOzm1lERERCCnv08h7An4DX3b0ywjwiIiKpFXb38h+jDiIiIpJ2YQ+kEhERkQZS0xUREYmJmq6IiEhM1HRFRERiYu4e9Twin4GIiEgjU+d3zoc9ZahBcpt3jmM2sausKANUX1NVWVGW2tog3fXtDusmqL6mqrq+umj3soiISEzUdEVERGKipisiIhITNV0REZGYqOmKiIjERE1XREQkJmq6IiIiMVHTFRERiYmaroiISEzUdEVERGKipisiIhITNV0REZGYqOmKiIjERE1XREQkJmq6IiIiMVHTFRERiYmaroiISEzqbbpmtp+Z/dXMng2GDzOzi6OPJiIiki5htnQfAGYA+cHwW8CVUQUSERFJqzBNt4O7TwK2Arh7JVAVaSoREZEUCtN0N5vZ3oADmFl/YGOkqURERFIoTNP9KfA00MPMXgEeAi6PNNUuGFdwO+WlCykumpV0lEgMGzqIJYvnsHzpXK65ekzScRqsV68eFC6YWXPbsH45Yy+/hHbt2jJ92gSWLZnL9GkTaNu2TdJRG2x7tTZl2/t7GzP6IhYveomFxbP5/e9+nlC67GvTZi8mPlbA4kUvsejNF+nf75ikI2VN2t5bttXY6jN33/EDzLoD7wIHAwb8EzjK3ReEnIfnNu/coJBhHD+wH5s2beb+++/kqN5DIp8fQGVFGQBR15eTk8OyJS8z/JRzKC1dw2v/mMZ5549m2bK3I51vnPWVrH6dAQNPY/SlF7Jhw4f84ba7uObqMbRr14brf/bbSOZbWVEWeW3bql1rSUlZpPOKsr66/t4GfWMA1183lhFnXEBFRQX77LM37733fiTzj2vdrHbfX+9g7tx53Hf/BPLy8mjVqiUbN34U2fz03pIdCddndWYK8fzHgf3cfYm7LwaOA+7LXrzseHnuPDZ88GHSMSJxbN/erFixmlWrStiyZQuTJv2d00cMSzpW1gw5cSArV75DSUkZI0YM46GHJwPw0MOTOf304Qmny67atTZldf29/fjHF/CH2+6ioqICILKGG7e99tqT4wf24777JwCwZcuWSBtunNL+3tIY6wvTdH8CPGVmHc3sFOAvwCnRxpLa8jt35N3S8prh0rI15Od3TDBRdo0ceQaPTXwKgP327cDatesAWLt2Hfvt2yHJaFlXu9a0OeigAxk48FhenTuF2c8/Tp9jjkw6UlZ0774/69e/z1/v/TML5s/gnrtvo1WrlknHyoq0v7c0xvrqbbrBbuSxwEzgJuAkd3834lyym8jLy2PEaUN5/G9T65xe38cfTUl9tTZ1ubnNaNeuLQMGjuDa637NhPF3Jx0pK3KbNaN3769yzz0P0ffYYWze/AnXXnNZ0rGkidpu0zWzKWb2tJk9DVwPtAI+B/4ajNsuMxtlZoVmVlhQUJDdxLuh8rK1dO2SXzPcpXMnysvXJpgoe4YPH0xR0SLWrVsPwL/Xradjx30B6NhxX9alZBclfLnWtCkrXcNTTz0LwILCYrZu3UqHDu0TTtVwpWVrKC1dw/wFRQA88cQz9D7qqwmnyo40v7dA46xvR1u6fwRur3W7GLih1vB2uXuBu/dx9z6jRo3KVtbd1oLCYnr27E63bl3Jy8tj5MgzmDJ1ZtKxsuLss878wu7WqVNmcsH53wPggvO/x5QpM5KKlnXb1po2f396BoMGDQAyu5qbN2/O+vUbEk7VcP/+93uUlpbTq1cPAE48cSDLlr2VcKrsSPN7CzTO+nK3N8HdX6q+b2b7AX2Dwfnuvi7qYDvrkYfv4hsnHEeHDu1ZvbKQm2/5I/c/8FjSsbKiqqqKK668gWnPjKdZTg4PPDiRpUub/h99q1YtOWnICVw6+tqacbfedhePjb+biy48h5KSUs4+9ycJJsyeumptyrb393bvuNspLppFRcUWfnhxei5cd8V/3chDD/6F5s3zWLWqhIsv+WnSkbIire8t1RpjfWFOGRoJ3Aa8SOYQ6OOBq9398ZDziOWUoSTEfdpC3HaH+tJaG6S7vt1h3QTV11Tt6JSh7W7p1vJzoG/11q2Z7QM8T+ZUIhEREQkpzClDOdvsTn4/5PNERESkljBbutPNbAYwIRg+G3g2ukgiIiLpVG/TdferzezbwNeDUXe7e3oPwRQREYnIdpuumc1194Fm9jGZbxiq/lB4lJltBTYAt7n7/8aQU0REpMnb0SlDA4Ofe9Y1Pfi6v1cBNV0REZEQdvmAKHd/HxiUvSgiIiLp1qCjkN19TbaCiIiIpJ1O/REREYmJmq6IiEhM1HRFRERioqYrIiISEzVdERGRmKjpioiIxERNV0REJCZquiIiIjFR0xUREYmJmq6IiEhM1HRFRERioqYrIiISEzVdERGRmKjpioiIxMTcPep5RD4DERGRRsbqGqktXRERkZjkxjKT5p3jmE3sKivKANXXVFVWlKW2Nkh3fbvDugmqr6mqrq8u2tIVERGJiZquiIhITNR0RUREYqKmKyIiEhM1XRERkZio6YqIiMRETVdERCQmaroiIiIxUdMVERGJiZquiIhITNR0RUREYqKmKyIiEhM1XRERkZiEarpm9pyZta013M7MZkQXS0REJH3Cbul2cPcPqwfc/QNg32giiYiIpFPYprvVzPavHjCzAwCPJpKIiEg6hf0S+58Dc83sJcCA44FRkaUSERFJoVBN192nm9nRQP9g1JXuvj66WCIiIumzw93LZnZI8PNoYH+gPLjtH4wTERGRkOrb0v0pmd3It9cxzYETs55IREQkpXa4pevuo4Kfg+u4NbqGO2zoIJYsnsPypXO55uoxScfJqjTXBnDF2B+xsHg2xUWzeOThu2jRokXSkRqkV68eFC6YWXPbsH45Yy+/hJtvupo3Xn+OwgUzefaZ8XTqtF/SUbOiTZu9mPhYAYsXvcSiN1+kf79jko6UNeMKbqe8dCHFRbOSjpJ1Xbrk8/zMyby58AUWFs/m8ssuTjpS1jW25Wfu9R+EbGbfA6a7+8dmdgNwNPArdy8KMQ/Pbd65gTHrl5OTw7IlLzP8lHMoLV3Da/+Yxnnnj2bZsrcjm2dlRRkAUdeXRG0QX335+R156YUn+eqRg/nss8+YMP5unn12Ng89PCnS+VZWlEVeG2SWX8nq1xkw8DQ++GAjH3+8CYDLxvyQQw/txZjLrotkvnHVB3DfX+9g7tx53Hf/BPLy8mjVqiUbN34U2fziWjcBjh/Yj02bNnP//XdyVO8hkc8P4quvY8d96dRxX4qKF/OVr7Rm/rzpfOe7P0zNewskuvysrmlhTxm6MWi4A4GTgL8Cd2cnXnYc27c3K1asZtWqErZs2cKkSX/n9BHDko6VFWmurVpubi4tW+5Bs2bNaNWyJWvWrE06UtYMOXEgK1e+Q0lJWU3DBWjduhVh/ult7Pbaa0+OH9iP++6fAMCWLVsibbhxe3nuPDZ88GH9D2yC1q5dR1HxYgA2bdrM8uVv0zm/Y8KpsquxLb+wTbcq+HkqUODuzwDNo4m0a/I7d+Td0vKa4dKyNeSnZOVJc20A5eVr+dOf72bVivmUlhSx8aOPeO75OUnHypqRI8/gsYlP1Qz/6pZrWbViAeec8y1uuvm2BJNlR/fu+7N+/fv89d4/s2D+DO65+zZatWqZdCzZSQcc0IWjjjyCefPD7MCUXRW26ZaZ2T3AWcA0M2uxE88V2aG2bdtw+ohh9OzVn64HHE3r1q0499xvJx0rK/Ly8hhx2lAe/9vUmnE3/uJWuvfoy4QJTzJm9EUJpsuO3GbN6N37q9xzz0P0PXYYmzd/wrXXXJZ0LNkJrVu3YtLEcfz0ql9+YW+MZF/YxjkSmAEMCy4H2R64ensPNrNRZlZoZoUFBQVZiFm/8rK1dO2SXzPcpXMnysvTsYsyzbUBDBlyPKtWl7B+/QYqKyt58qlnOa5/n6RjZcXw4YMpKlrEunVfPq19/IQn+Na3TkkgVXaVlq2htHQN8xdktpCeeOIZeh/11YRTSVi5ublMnjiOCROe5Kmnnk06TuqFvvYyUAh8HlwOMg9Yvr0Hu3uBu/dx9z6jRsVz4aoFhcX07Nmdbt26kpeXx8iRZzBl6sxY5h21NNcG8G5JGf36HU3LlnsAcOLggSxfHu2BHHE5+6wzv7BruWfP7jX3Tx8xjH/+c0USsbLq3/9+j9LScnr16gHAiScOZNmytxJOJWGNK7idZcv/xR13xrOBtLsLexnIZ8icl2vAHkB34J/A4RHl2mlVVVVcceUNTHtmPM1ycnjgwYksXZqOP/w01wYwf0ERTzzxDAvmz6CyspLi4iWMu/fRpGM1WKtWLTlpyAlcOvramnG//c319OrVg61bt1JSUsboMdEcuRy3K/7rRh568C80b57HqlUlXHzJT5OOlDWPPHwX3zjhODp0aM/qlYXcfMsfuf+Bx5KOlRVfH9CX88/7Lm8uWkrhgsw/8jfe+HuenT474WTZ09iWX6hThr70pMzVqEa7+yUhHh7LKUNJiPOw9yTsDvWltTZId327w7oJqq+pysYpQ1/g7m8A/RqQSUREZLcTaveymdXeV5QDHEPmGswiIiISUtjPdPfkP9+fWwlMAf4WSSIREZGUCtt0pwE/A7rVes51wNciyCQiIpJKYZvuI8BVwGJga3RxRERE0its033P3adEmkRERCTlwjbdX5rZvcAs4PPqke7+RCSpREREUihs070IOITMlaiqdy87oKYrIiISUtim29fdD2GspbUAACAASURBVI40iYiISMqFvTjGq2Z2WKRJREREUi7slm5/oNjMVpH5TNcAd3edMiQiIhJS2KY7PNIUIiIiu4FQTdfd34k6iIiISNrt0hceiIiIyM5T0xUREYmJmq6IiEhM1HRFRERioqYrIiISEzVdERGRmKjpioiIxERNV0REJCZquiIiIjFR0xUREYmJuXvU84h8BiIiIo2M1TUy7BceNEhu885xzCZ2lRVlgOprqiorylJbG6S7vt1h3QTV11RV11cX7V4WERGJiZquiIhITNR0RUREYqKmKyIiEhM1XRERkZio6YqIiMRETVdERCQmaroiIiIxUdMVERGJiZquiIhITNR0RUREYqKmKyIiEhM1XRERkZio6YqIiMQkdNM1s4FmdlFwfx8z6x5dLBERkfQJ1XTN7JfAtcD1wag84JGoQomIiKRR2C3dbwGnA5sB3L0c2DOqUCIiImkUtulWuLsDDmBmraOLJCIikk5hm+4kM7sHaGtmPwKeB+6NLpaIiEj65IZ5kLv/0cy+CXwEHAz8wt2fizSZiIhIyoRqumZ2q7tfCzxXxzgREREJIezu5W/WMe7kbAbJhmFDB7Fk8RyWL53LNVePSTpOVqW5NlB9TdHll11McdEsFhbPZuzllwDwixt/yjurCilcMJPCBTM5efiJCafMjjQuv2pprg0aX307bLpmdqmZLQIONrM3a91WAW/GEzGcnJwc/vvO33DaiPP46pGDOeusMzn00IOSjpUVaa4NVF9TdPjhB3Pxxedy3IBTOfqYb3LqKSfRo0c3AO7873H06TuUPn2H8uz02ckGzYI0Lr9qaa4NGmd99W3pjgdGAE8HP6tvx7j7eRFn2ynH9u3NihWrWbWqhC1btjBp0t85fcSwpGNlRZprA9XXFB1yyEHMn1/Ep59+RlVVFXNefo1vndnodn5lRRqXX7U01waNs776mq67+2pgDPBxrRtm1j7aaDsnv3NH3i0trxkuLVtDfn7HBBNlT5prA9XXFC1ZspyBA/vRvn07Wrbcg5OHn0iXLvkAjL70It54/TnGFdxO27ZtEk7acGlcftXSXBs0zvrCbOkCvA4UBj9frzUsIruh5cv/xW233cWz08YzbeqjFC9cQlXVVu6+5yF6HTKAY/oMZe3addz2h18kHVWkUdlh03X304Kf3d39wOBn9e3A7T3PzEaZWaGZFRYUFGQ7c53Ky9bSNfhPG6BL506Ul6+NZd5RS3NtoPqaqvsfeIx+/U9m8JDv8OGHG3n77ZWsW7eerVu34u7c+9dH6dv3qKRjNlhalx+kuzZonPXVdyDV0Tu6be957l7g7n3cvc+oUaOyn7oOCwqL6dmzO926dSUvL4+RI89gytSZscw7ammuDVRfU7XPPnsD0LVrPmeeeTITHnuSjh33rZl+5hkns2TJP5OKlzVpXX6Q7tqgcdZX33m6t+9gmgON5nyAqqoqrrjyBqY9M55mOTk88OBEli59K+lYWZHm2kD1NVWTJ46j/d7t2LKlkrFjf87GjR9x5x2/5sgjD8PdeeedUi4d3fRP5U/r8oN01waNsz7LXFI5Up7bvHPU80hEZUUZAKqvaaqsKEttbZDu+naHdRNUX1MV1Gd1TQt7Rao84FLghGDUi8A97r4lC/lERER2C6GaLvB/ZL5D93+D4fODcZdEEUpERCSNwjbdvu5+ZK3h2Wa2MIpAIiIiaRX22stVZtajesDMDgSqookkIiKSTmG3dK8GXjCzlcFwN+CiSBKJiIikVNgt3VeAe4CtwIbg/j+iCiUiIpJGYZvuQ0B34FfAX4ADgYejCiUiIpJGYXcvH+Huh9UafsHMlkYRSEREJK3Cbum+YWb9qwfMrB/6wgMREZGdEnZL9xjgVTMrCYb3B/4ZfMG9u/vXIkknIiKSImGb7vBIU4iIiOwGQjVdd38n6iAiIiJpF/YzXREREWkgNV0REZGYqOmKiIjERE1XREQkJmq6IiIiMVHTFRERiYmaroiISEzUdEVERGKipisiIhITNV0REZGYmLtHPY/IZyAiItLIWF0jw37hQYPkNu8cx2xiV1lRBqi+pqqyoiy1tUG669sd1k1QfU1VdX110e5lERGRmKjpioiIxERNV0REJCZquiIiIjFR0xUREYmJmq6IiEhM1HRFRERioqYrIiISEzVdERGRmKjpioiIxCRU0zWzA8zspOB+SzPbM9pYIiIi6VNv0zWzHwGPA/cEo7oAT0UZSkREJI3CbOmOAb4OfATg7m8D+0YZSkREJI3CNN3P3b2iesDMctHX9YmIiOy0ME33JTP7GdDSzL4JTAamRBtLREQkfcI03euA94BFwI+BacANUYYSERFJo3q/xN7dtwLjgpuIiIjsojBHL59mZkVmtsHMPjKzj83sozjCiYiIpEm9W7rAHcC3gUXurgOoREREdlGYz3TfBRar4YqIiDRMmC3da4BpZvYS8Hn1SHf/U2SpREREUijMlu5vgE+APYA9a90anWFDB7Fk8RyWL53LNVePSTpOVqW5thYtWvCPV6byeuFzLCyezS9/8f+SjpRV4wpup7x0IcVFs5KOEok0Lr9evXpQuGBmzW3D+uWMvfwSbv3dDSxe9BJvvP4cj0++lzZt9ko6aoOl+b0F4PLLLqa4aBYLi2cz9vJLko6D1bfX2MwWu/sRDZiH5zbv3ICnh5OTk8OyJS8z/JRzKC1dw2v/mMZ5549m2bK3I5tnZUUZAFHXl0RtEF99AK1bt2Lz5k/Izc1lzotP8l8//SXz5r8R6TwrK8piqe34gf3YtGkz999/J0f1HhL5/KrFVR/Ev/ziXDdzcnIoWf06AwaexsG9ejD7hVeoqqrid7/9GQDX/+y3WZ+n3luy4/DDD+bRR/6X4wacSkXFFqZNfZTRl13HihWrI51vUJ/VNS3Mlu40Mxua1UQROLZvb1asWM2qVSVs2bKFSZP+zukjhiUdKyvSXFu1zZs/ASAvL5fcvDzSdAjBy3PnseGDD5OOEak0L78hJw5k5cp3KCkp47nn51BVVQXAa/PeoHPnTgmna5i0v7cccshBzJ9fxKeffkZVVRVzXn6Nb515cqKZwjTdS4HpZvZpYz5lKL9zR94tLa8ZLi1bQ35+xwQTZU+aa6uWk5ND4YKZrCl7k1mz5jB/QVHSkWQnpHn5jRx5Bo9N/PJ3vFx04dlMn/FCAomyJ+3vLUuWLGfgwH60b9+Oli334OThJ9KlS36imeptuu6+p7vnuHtLd98rGG76H2RIo7J161b69B3KAd370LdPbw4//OCkI8lOSOvyy8vLY8RpQ3n8b1O/MP7668ZSWVnJ+PFPJJRMwli+/F/cdttdPDttPNOmPkrxwiVUVW1NNFOYi2OcUNetnueMMrNCMyssKCjIXtodKC9bS9da/8F06dyJ8vK1scw7ammubVsbN37Eiy+9wrChg5KOIrsgbctv+PDBFBUtYt269TXjLjh/JKeechLnX3BZgsmyY3d4b7n/gcfo1/9kBg/5Dh9+uJG3316ZaJ4wu5evrnW7kcyXHdy0oye4e4G793H3PqNGjWpwyDAWFBbTs2d3unXrSl5eHiNHnsGUqTNjmXfU0lwbQIcO7WuOAt1jjz04acgJ/POfKxJOJWGlefmdfdaZX9i1PGzoIK666lLO/PaFfPrpZwkmy460v7cA7LPP3gB07ZrPmWeezITHnkw0T5hrL4+oPWxmXclcpapRqaqq4oorb2DaM+NplpPDAw9OZOnSt5KOlRVprg2gU6f9uO+vd9CsWQ45OTk8/vgUnpn2fNKxsuaRh+/iGyccR4cO7Vm9spCbb/kj9z/wWNKxsiaty69Vq5acNOQELh19bc24O+/4NS1atGD6s5nlN2/eG4y57LqkIjZY2t9bACZPHEf7vduxZUslY8f+nI0bkz0kqd5Thr70BDMDlrj7YSGfEsspQ0mI87SFJOwO9aW1Nkh3fbvDugmqr6na0SlD9W7pmtlf+M+X1ucARwHRnkApIiKSQmEuA1lY634lMMHdX4koj4iISGqF+Uz3wTiCiIiIpN12m66ZLeI/u5W/MAlwd/9aZKlERERSaEdbuqfFlkJERGQ3sN2m6+7vVN83s/2AvsHgfHdfF3UwERGRtAlzRaqRwHzge8BIYJ6ZfTfqYCIiImkT5ujlnwN9q7duzWwf4Hng8SiDiYiIpE2Yy0DmbLM7+f2QzxMREZFawmzpPmtmM4AJwfBZwLToIomIiKRTmKa7DniEzJWoAArcPdkrRouIiDRBYXYTtwauA44FVgGvRppIREQkpcJ8if3N7n44MAboBLxkZk3/K0RERERitjMHRK0D1pI5kGrfaOKIiIikV5jzdEeb2YvALGBv4Ee6BKSIiMjOC3MgVVfgSncvjjqMiIhImoX5lqHr4wgiIiKSdrrIhYiISEzUdEVERGKipisiIhITNV0REZGYqOmKiIjExNw96nlEPgMREZFGxuoaqS1dERGRmIS5OEbDZ9K8cxyziV1lRRmg+pqqyoqy1NYG6a5vd1g3QfU1VdX11UVbuiIiIjFR0xUREYmJmq6IiEhM1HRFRERioqYrIiISEzVdERGRmKjpioiIxERNV0REJCZquiIiIjFR0xUREYmJmq6IiEhM1HRFRERioqYrIiISEzVdERGRmIRqumZ2mpkVmdkGM/vIzD42s4+iDiciIpImYb9P9w7g28Aid/cI84iIiKRW2N3L7wKL1XBFRER2Xdgt3WuAaWb2EvB59Uh3/1MkqURERFIobNP9DbAJ2ANoHl0cERGR9ArbdPPd/YhIk4iIiKRc2M90p5nZ0EiTiIiIpFzYpnspMN3MPm3MpwwNGzqIJYvnsHzpXK65ekzScbIqzbWB6mtqunTJ5/mZk3lz4QssLJ7N5ZddDMCtv7uBxYte4o3Xn+PxyffSps1eCSfNjrQtvyvG/oiFxbMpLprFIw/fRYsWLWqm/flPt/DhhrcSTJddjW3Z1dt0zSwHGO7uOe7e0t33cvc93b1R/TXl5OTw33f+htNGnMdXjxzMWWedyaGHHpR0rKxIc22g+pqiyspKrr7mZr525GC+PnAEl156IYceehDPz5rDkUedyNHHfJO3317JdddelnTUBkvb8svP78hlY35Iv/6ncFTvITRr1oyzRp4BwDFHf4127domnDB7GuOyq7fpuvtW4H9iyNIgx/btzYoVq1m1qoQtW7YwadLfOX3EsKRjZUWaawPV1xStXbuOouLFAGzatJnly9+mc35Hnnt+DlVVVQC8Nu8NOnfulGTMrEjj8svNzaVlyz1o1qwZrVq2ZM2ateTk5HDr72/kuut/nXS8rGmMyy7s7uVZZvYdM7NI0zRAfueOvFtaXjNcWraG/PyOCSbKnjTXBqqvqTvggC4cdeQRzJtf9IXxF114NtNnvJBQquxJ2/IrL1/Ln/58N6tWzKe0pIiNH33Ec8/PYczoi5gydSZr165LOmLWNMZlF7bp/hiYDHwe5jNdMxtlZoVmVlhQUJCVoCLS+LRu3YpJE8fx06t+yccfb6oZf/11Y6msrGT8+CcSTCd1adu2DaePGEbPXv3pesDRtG7divPO+y7f/c5p/M9d9yUdL/VCnTLk7nvuzIu6ewFQ3W199GU372yunVZetpauXfJrhrt07kR5+drI5xuHNNcGqq+pys3NZfLEcUyY8CRPPfVszfgLzh/JqaecxDeHjUwwXfakbfkNGXI8q1aXsH79BgCefOpZfnnj/6Nlyz3457JXAGjVqiXLl87lkMMGJhm1wRrjsgv9LUNm1s7MjjWzE6pvUQbbWQsKi+nZszvdunUlLy+PkSPPYMrUmUnHyoo01waqr6kaV3A7y5b/izvu/M/erGFDB3HVVZdy5rcv5NNPP0swXfakbfm9W1JGv35H07LlHgCcOHggd9xZQJf9e9OzV3969urPJ5982uQbLjTOZRdqS9fMLgGuALoAxUB/4B/AidFF2zlVVVVcceUNTHtmPM1ycnjgwYksXZqOw97TXBuovqbo6wP6cv553+XNRUspXJB5E7vxxt/z5z/dQosWLZj+7GMAzJv3BmMuuy7JqA2WtuU3f0ERTzzxDAvmz6CyspLi4iWMu/fRpGNFojEuOwvzHQZmtgjoC7zm7keZ2SHAb9392yHm4bnNOzcwZuNUWVEGgOprmiorylJbG6S7vt1h3QTV11QF9dV54HHY3cufuftnAGbWwt2XAwdnJ56IiMjuIey1l0vNrC3wFPCcmX0AvBNdLBERkfQJe/Tyt4K7N5nZC0AbYHpkqURERFJoh03XzNrXMXpR8PMrwIasJxIREUmp+rZ0XwecL34gXD3swIER5RIREUmdHTZdd+9efT/Y6j2IzBfZi4iIyE5qyHm6rwJDoosmIiKSLmFPGbqCzHm677j7YKA3sDGyVCIiIimk83RFRERiovN0RUREYqLzdEVERGISdku3hru/FEUQERGRtAv91X4iIiLSMGq6IiIiMVHTFRERiYmaroiISEzUdEVERGKipisiIhITNV0REZGYqOmKiIjERE1XREQkJmq6IiIiMTF3j3oekc9ARESkkbG6Ru70tZd3RW7zznHMJnaVFWWA6muqKivKUlsbpLu+3WHdBNXXVFXXVxftXhYREYmJmq6IiEhM1HRFRERioqYrIiISEzVdERGRmKjpioiIxERNV0REJCZquiIiIjFR0xUREYmJmq6IiEhM1HRFRERioqYrIiISEzVdERGRmKjpioiIxERNV0REJCZquiIiIjFR0xUREYlJqKZrZmPMrG2t4XZmNjq6WCIiIukTdkv3R+7+YfWAu38A/CiaSCIiIukUtuk2MzOrHjCzZkDzaCKJiIikU27Ix00HJprZPcHwj4NxIiIiElLYLd1rgdnApcFtFnBNVKF21bChg1iyeA7Ll87lmqvHJB0nq8YV3E556UKKi2YlHSUSaV52kL76unTJ5/mZk3lz4QssLJ7N5ZddXDNtzOiLWLzoJRYWz+b3v/t5gimzJycnhwXzZ/D3Jx9MOkrWpe29pVevHhQumFlz27B+OWMvv4R27doyfdoEli2Zy/RpE2jbtk0i+czdd+4JZu2BLu7+ZsineG7zzjsdbGfl5OSwbMnLDD/lHEpL1/DaP6Zx3vmjWbbs7cjmWVlRBkAc9R0/sB+bNm3m/vvv5KjeQyKfH8RXXxLLDjL1pXXdhGjr69hxXzp13Jei4sV85SutmT9vOt/57g/Zb999uP66sYw44wIqKirYZ5+9ee+997M+/zj/9gCuvGIUxxzzNfbac0/O+NYPIp+f3luyIycnh5LVrzNg4GmMvvRCNmz4kD/cdhfXXD2Gdu3acP3PfhvJfIP6rK5pYY9eftHM9goa7uvAODP7c/YiNtyxfXuzYsVqVq0qYcuWLUya9HdOHzEs6VhZ8/LceWz44MP6H9gEpX3ZpbG+tWvXUVS8GIBNmzazfPnbdM7vyI9/fAF/uO0uKioqACJpuHHr3LkTp5w8hPvum5B0lEik+b1lyIkDWbnyHUpKyhgxYhgPPTwZgIcenszppw9PJFPY3ctt3P0j4NvAQ+7eD4jnX6KQ8jt35N3S8prh0rI15Od3TDCRhJX2ZZf2+g44oAtHHXkE8+YXcdBBBzJw4LG8OncKs59/nD7HHJl0vAb70+03c931v2br1q1JR5GdNHLkGTw28SkA9tu3A2vXrgMy/zTut2+HRDKFbbq5ZtYJGAlMjTCPiDQhrVu3YtLEcfz0ql/y8cebyM1tRrt2bRkwcATXXvdrJoy/O+mIDXLqKSexbt163ihalHQU2Ul5eXmMOG0oj/+t7pa1sx+tZkvYpnsLMAP4l7svMLMDge1+IGVmo8ys0MwKCwoKspGzXuVla+naJb9muEvnTpSXr41l3tIwaV92aa0vNzeXyRPHMWHCkzz11LMAlJWuqbm/oLCYrVu30qFD+yRjNsiAAX0YcdpQ/vXWazz6yP8yePDXefCB/046loQwfPhgiooWsW7degD+vW49HTvuC2SOSViX0EcfoZquu09296+5++hgeKW7f2cHjy9w9z7u3mfUqFHZyrpDCwqL6dmzO926dSUvL4+RI89gytSZscxbGibtyy6t9Y0ruJ1ly//FHXf+5x/rvz89g0GDBgBw0EEH0rx5c9av35BUxAb7+Q2/p9uBfejZqz/fP280L7zwCj+4cGzSsSSEs886s2bXMsDUKTO54PzvAXDB+d9jypQZieTa4Xm6ZnaNu//BzP4CfGlb3N0bzdpXVVXFFVfewLRnxtMsJ4cHHpzI0qVvJR0rax55+C6+ccJxdOjQntUrC7n5lj9y/wOPJR0rK9K+7NJY39cH9OX8877Lm4uWUrgg8w/EjTf+nvsfeIx7x91OcdEsKiq28MOLr0w4qdQnje8trVq15KQhJ3Dp6Gtrxt162108Nv5uLrrwHEpKSjn73J8kkm2HpwyZ2Qh3n2JmP6DupvtQiHnEcspQEuI+bSFuu0N9aa0N0l3f7rBuguprqnZ0ytAOt3TdfUpwdynwM6Bbrec4EKbpioiICOEvA/kIcDWwCNBx8yIiIrsgbNN9z92fjjSJiIhIyoVtur80s3vJXHP58+qR7v5EJKlERERSKGzTvQg4BMjjP7uXHVDTFRERCSls0+3r7gdHmkRERCTlwl6R6lUzOyzSJCIiIikXdku3P1BsZqvIfKZrgLv71yJLJiIikjJhm24y34EkIiKSIqGarru/E3UQERGRtAv7ma6IiIg0kJquiIhITNR0RUREYqKmKyIiEhM1XRERkZio6YqIiMRETVdERCQmaroiIiIxUdMVERGJiZquiIhITNR0RUREYqKmKyIiEhNz96jnEfkMREREGhmra2QcW7oW583Mfhz3PFWf6lN96a5N9TX9WwL11SmNu5dHJR0gYqqvaUtzfWmuDVRfU9co6ktj0xUREWmU1HRFRERiksamW5B0gIipvqYtzfWluTZQfU1do6gvjqOXRUREhHRu6YqIiDRKu23TNbNBZjY16Rw7y8wuNLP8WsOrzaxDkpnCMrNXdzCtSS6PbDKze83ssKRz7Aoze9HM+gT3p5lZ2+A2OulsO8PMupnZ4qRzRKEpvVdkk5mdbmbXJZ2jWuqbrpnlJp0hyy4E8ut7UBhx/27cfUCc82tq3P0Sd1+adI6GcvdT3P1DoC3QpJqu/IeZNUs6Qza4+9Pu/vukc1RrdE3XzFqb2TNmttDMFpvZWWZ2jJm9ZGavm9kMM+sUPPZHZrYgeOzfzKxVMP4BM7vbzOYBfzCznmb2fPC4N8ysRzC7r5jZ42a23MweNbPtntDcgHquNrOxwf0/m9ns4P6JwTyHmtk/glyTzewrwfRfBLUtNrMCy/gu0Ad41MyKzaxlMJvLg+cvMrNDav0e7zOz+WZWZGZnBOMvNLOngxyzsl1vPb+LTUEdtwV1LTKzs+p4XN8gc4/tLfukBVtEy4N17a1gWZ5kZq+Y2dtmdqyZ3WRmV9V6zuLgeV9ax4PptbcWhwfLdKGZxbqctqnvUTNbFvydtDKzIcGyWRSsXy3qeG71FtXvgR7BunpbMO3qYL1+08xujqmW35vZmFrDNwU56lsPLzSz/6k1PNXMBgX3NwXPXxK8txwbLL+VZnZ68JhmwWOq6/1xDOVWZ30q+JtZYmZfOj91e9ODum43s4XAcWHqTFLIv8Oa5Whm3wuW+UIzmxOMa2ZmfwzGv2lml0ca2t0b1Q34DjCu1nAb4FVgn2D4LOC+4P7etR73a+Dy4P4DwFSgWTA8D/hWcH8PoBUwCNgIdCHzz8c/gIER1NMfmBzcfxmYD+QBvwSuBeYArYPp1wK/CO63r/UaDwMjgvsvAn1qTVtdq+7RwL3B/d8C5wX32wJvAa3JbCmX1n79GJftpmD5Pgc0A/YDSoBOwfKYCgwAXgf2D35PdS77pG9AN6AS+Gqw/rwO3EfmSjRnAE8BNwFX1XrO4uB5X1rHay9bYB/gXaD7tutCzPU58PVg+D7ghiBXr2DcQ8CV266XwTrZIXiNxbVecyiZI0gt+J1NBU6IoZbewEu1hpcCP9jOeliTOfhb+Z9az5sKDAruO3BycP9JYGawvh4JFAfjRwE3BPdbAIXVyzSGmtsHP1sG693e1ctle9Nr1TWy1uvUW2eSt5B/hzXLEVgEdA7utw1+Xgo8DuTW/t1EdWt0W7pkfinfNLNbzex4oCtwBPCcmRWT+cPvEjz2CDN72cwWAd8HDq/1OpPdvcrM9iTzS34SwN0/c/dPgsfMd/dSd98KFJNZgNn2OnCMme0FfE6mufcBjgc+BQ4DXglq+wFwQPC8wWY2L6jtxG1q29YTtebVLbg/FLgueN0XyfyzsX8w7Tl339Dw0nbJQGCCu1e5+7+Bl4C+wbRDybwpj3D3EuBgtr/sG4NV7r4oWH+WALM881e7iB2vS19Yx9194zbT+wNz3H0VQILL6l13fyW4/wgwhEzNbwXjHgRO2InXGxrcioA3gEOAg7KUdbvcvQjY18zyzexI4APgKLa/HoZRAUwP7i8i09S38MVlPxS4IFh355FpfJHXGxgbbK2+RuY9dNv5bm96FfC3Wo8LU2fSdubv8BXgATP7EZl/uABOAu5x90qI/u+t0X3e6e5vmdnRwClktl5nA0vc/bg6Hv4AcKa7LzSzC8lsLVXbHGJ2n9e6X0UEvw9332Jmq8j8t/Uq/7+dewmN6orjOP79CaIgIiiKILVKbWihgg9QglAo1FVdCFXaSt24sJFCV9WND0QXLuJCRIpiKS6EIqGuChpEEAZJVBQqPshKodWoqSj4qBDNv4v/mcx1kmtmMsmdGP6fTTLcuWfOmfP+n5vAdeALYAlwB58Av8veI2k68Au+c/hb0l580sxTLke2DAK+NrOeqrRXU9t30wy9eDmXA/fxMuTV/USQbT8DmdcDeD285u0jnOkwtI1LOm9m+wrIb72q8kVKgAAAAt9JREFU/57wKT5xjJaAA2Z2rIE0RqsD2ADMB04Bi2u4Z9j6S/rTwA6ZujezAVWelRAehepsJOP1SiHwL4FWM3sp6QKZvI9w/ZWZvckkV0s5m22kfjjIzNrSGPgVcFXSymKyWDHhdrryJ3NfmtlJoB1YDcyV1JquT5VU3vXNBHolTcV3ukOY2TPgH0nr0/3TlM5+C1QCfsZDySWgDV/tdwNrJC1JeZshqYVKB/hXfsa7IZPWM7zcI+nEz3qV0l4+FgUZAyXgm3SOMhffKV1O157ineFAGhh6yK/798FdYAVAmmQXp9+r2/iKqvu6gc8lld8/u6gMV1lY/u6BTXh4dFG5vQKb8R1inuq22glsUeW5hQWS5o1xnvOcAr7F+1IH726HZXeBZZKmSPoAWFXnZ3YC29L4hKQWSTMaKEOtZgFP0oT6CR45qef6pCXpIzO7ZGZ7gD58l38O+KG8iBjv/jZRVipZS4F2SQNAPx5vfw0cljQLz/MhPIywGw/b9KWfeZPRZuCYpH0pzY3jWoKhSsBOoMvMXkh6BZTMrC/t0H9X5YGUXWkndBw/a3kAXMmkdQI4Kuk/4F07wP3493Rd0hR8V71uLAs1CoafC7UCf6XXO8zsQer8mNlDSeuAM8AWfJAcru7fB3/g4cWbePssh2WHa+ODUrvYCpxOdfcIWFtctgf1AD9K+g0/B/0JXxB0pAHqCnA072Yze5weaLkBnDGz7ZI+BbrSWvA58D1evnFlZjfTUdM9M+uVlNcOF2Vuu4j3m1vAbTwkXo9f8fDmtbT47QPWN1KOGp0F2iTdxuuwu87rk1m7pI/xKMR5vP5vAC34WNkPHAeO5CfRmPiPVKEQkuYA18zswxHfHJouTT5/mtlnTc5KCJPKhAsvh8knhVO7gIPNzksIITRT7HRDCCGEgsRON4QQQihITLohhBBCQWLSDSGEEAoSk24IIYRQkJh0QwghhILEpBtCCCEU5H9dML8bhsJmugAAAABJRU5ErkJggg==\n"
          },
          "metadata": {
            "needs_background": "light"
          }
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "filename = '/content/adasyn_other_categories_model.sav'\n",
        "pickle.dump(best_logreg, open(filename, 'wb'))"
      ],
      "metadata": {
        "id": "DmZVUVLB_pa_"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "cnf_matrix = cf_matrix\n",
        "FP = cnf_matrix.sum(axis=0) - np.diag(cnf_matrix) \n",
        "FN = cnf_matrix.sum(axis=1) - np.diag(cnf_matrix)\n",
        "TP = np.diag(cnf_matrix)\n",
        "TN = cnf_matrix.sum() - (FP + FN + TP)\n",
        "FP = FP.astype(float)\n",
        "FN = FN.astype(float)\n",
        "TP = TP.astype(float)\n",
        "TN = TN.astype(float)"
      ],
      "metadata": {
        "id": "oDHiAcI-6RPj"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def mathew(fp, fn, tp, tn):\n",
        "  num = (tp*tn)-(fp*fn)\n",
        "  deno = (tp+fp)*(tp+fn)*(tn+fp)*(tn+fn)\n",
        "  deno = np.sqrt(deno)\n",
        "\n",
        "  num1 = tp+tn\n",
        "  deno1 = tp+tn+fp+fn\n",
        "  return num/deno, num1/deno1"
      ],
      "metadata": {
        "id": "_xl6FXdy60yR"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "mc = []\n",
        "acc = []\n",
        "for fp,fn,tp,tn in zip(FP,FN,TP,TN):\n",
        "  mcor, accu = mathew(fp, fn, tp, tn)\n",
        "  mc.append(mcor)\n",
        "  acc.append(accu)\n",
        "mc,acc"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "RtLJFxCD7yZR",
        "outputId": "335ee06a-e7c4-4d5e-e383-1a505455a2cd"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "([0.9305491694740868,\n",
              "  0.9114466983749739,\n",
              "  0.7485173705410237,\n",
              "  0.8313818753541556,\n",
              "  0.6545715052014461,\n",
              "  0.8225266110508788,\n",
              "  0.8572619893275792,\n",
              "  0.7453499664814034],\n",
              " [0.9855072463768116,\n",
              "  0.9828722002635046,\n",
              "  0.9459815546772069,\n",
              "  0.9657444005270093,\n",
              "  0.8748353096179183,\n",
              "  0.9644268774703557,\n",
              "  0.9670619235836627,\n",
              "  0.9446640316205533])"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from  sklearn.metrics import precision_recall_fscore_support\n",
        "from sklearn.metrics import roc_curve, auc\n",
        "from sklearn.preprocessing import LabelBinarizer\n",
        "import numpy as np\n",
        "from scipy import interp"
      ],
      "metadata": {
        "id": "LTRNkKvlA23w"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "def class_report(y_true, y_pred, y_score=None, average='micro'):\n",
        "    if y_true.shape != y_pred.shape:\n",
        "        print(\"Error! y_true %s is not the same shape as y_pred %s\" % (\n",
        "              y_true.shape,\n",
        "              y_pred.shape)\n",
        "        )\n",
        "        return\n",
        "\n",
        "    lb = LabelBinarizer()\n",
        "\n",
        "    if len(y_true.shape) == 1:\n",
        "        lb.fit(y_true)\n",
        "\n",
        "    #Value counts of predictions\n",
        "    labels, cnt = np.unique(\n",
        "        y_pred,\n",
        "        return_counts=True)\n",
        "    print(labels)\n",
        "    n_classes = len(labels)\n",
        "    pred_cnt = pd.Series(cnt, index=labels)\n",
        "\n",
        "    metrics_summary = precision_recall_fscore_support(\n",
        "            y_true=y_true,\n",
        "            y_pred=y_pred,\n",
        "            labels=labels)\n",
        "\n",
        "    avg = list(precision_recall_fscore_support(\n",
        "            y_true=y_true, \n",
        "            y_pred=y_pred,\n",
        "            average='weighted'))\n",
        "\n",
        "    metrics_sum_index = ['precision', 'recall', 'f1-score', 'support']\n",
        "    class_report_df = pd.DataFrame(\n",
        "        list(metrics_summary),\n",
        "        index=metrics_sum_index,\n",
        "        columns=labels)\n",
        "\n",
        "    support = class_report_df.loc['support']\n",
        "    total = support.sum() \n",
        "    class_report_df['avg / total'] = avg[:-1] + [total]\n",
        "\n",
        "    class_report_df = class_report_df.T\n",
        "    class_report_df['pred'] = pred_cnt\n",
        "    class_report_df['pred'].iloc[-1] = total\n",
        "\n",
        "    if not (y_score is None):\n",
        "        fpr = dict()\n",
        "        tpr = dict()\n",
        "        roc_auc = dict()\n",
        "        for label_it, label in enumerate(labels):\n",
        "            fpr[label], tpr[label], _ = roc_curve(\n",
        "                (y_true == label).astype(int), \n",
        "                y_score[:, label_it])\n",
        "\n",
        "            roc_auc[label] = auc(fpr[label], tpr[label])\n",
        "\n",
        "        if average == 'micro':\n",
        "            if n_classes <= 2:\n",
        "                fpr[\"avg / total\"], tpr[\"avg / total\"], _ = roc_curve(\n",
        "                    lb.transform(y_true).ravel(), \n",
        "                    y_score[:, 1].ravel())\n",
        "            else:\n",
        "                fpr[\"avg / total\"], tpr[\"avg / total\"], _ = roc_curve(\n",
        "                        lb.transform(y_true).ravel(), \n",
        "                        y_score.ravel())\n",
        "\n",
        "            roc_auc[\"avg / total\"] = auc(\n",
        "                fpr[\"avg / total\"], \n",
        "                tpr[\"avg / total\"])\n",
        "\n",
        "        elif average == 'macro':\n",
        "            # First aggregate all false positive rates\n",
        "            all_fpr = np.unique(np.concatenate([\n",
        "                fpr[i] for i in labels]\n",
        "            ))\n",
        "\n",
        "            # Then interpolate all ROC curves at this points\n",
        "            mean_tpr = np.zeros_like(all_fpr)\n",
        "            for i in labels:\n",
        "                mean_tpr += interp(all_fpr, fpr[i], tpr[i])\n",
        "\n",
        "            # Finally average it and compute AUC\n",
        "            mean_tpr /= n_classes\n",
        "\n",
        "            fpr[\"macro\"] = all_fpr\n",
        "            tpr[\"macro\"] = mean_tpr\n",
        "\n",
        "            roc_auc[\"avg / total\"] = auc(fpr[\"macro\"], tpr[\"macro\"])\n",
        "\n",
        "        class_report_df['AUC'] = pd.Series(roc_auc)\n",
        "\n",
        "    return class_report_df"
      ],
      "metadata": {
        "id": "D-LZAc7YAzpD"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "report_with_auc = class_report(\n",
        "    y_true=y_test, \n",
        "    y_pred=best_logreg.predict(test_vectors), \n",
        "    y_score=best_logreg.predict_proba(test_vectors))\n",
        "\n",
        "print(report_with_auc)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "2BpEKUtyA9Sh",
        "outputId": "4b69c2f8-1d08-4f64-98f6-4c647b50efa1"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "['alarm' 'joke' 'misc' 'music' 'polite' 'search' 'volume' 'weather']\n",
            "             precision    recall  f1-score  support   pred       AUC\n",
            "alarm         0.987952  0.891304  0.937143     92.0   83.0  0.988756\n",
            "joke          0.986486  0.858824  0.918239     85.0   74.0  0.987013\n",
            "misc          0.864198  0.700000  0.773481    100.0   81.0  0.943498\n",
            "music         0.901235  0.802198  0.848837     91.0   81.0  0.973202\n",
            "polite        0.500000  1.000000  0.666667     95.0  190.0  0.972892\n",
            "search        0.867470  0.818182  0.842105     88.0   83.0  0.978882\n",
            "volume        0.965517  0.792453  0.870466    106.0   87.0  0.980684\n",
            "weather       0.875000  0.686275  0.769231    102.0   80.0  0.966015\n",
            "avg / total   0.867730  0.815547  0.826126    759.0  759.0  0.977710\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "scoring = ['accuracy','precision_macro', 'recall_macro', 'f1_macro']\n",
        "cv_logreg = LogisticRegression(class_weight = 'balanced',max_iter = 10000,C=1,penalty='l2',solver='lbfgs')\n",
        "scores = cross_validate(cv_logreg, train_vectors, y_train, cv=10, scoring=scoring)\n",
        "\n",
        "scores"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "UZBuvFmQxemx",
        "outputId": "9907e56e-dd0f-41a4-e6e9-94de3762ab2c"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "{'fit_time': array([0.23814583, 0.23223686, 0.2346909 , 0.21950126, 0.22419262,\n",
              "        0.21332502, 0.22402716, 0.31011057, 0.21442938, 0.2156074 ]),\n",
              " 'score_time': array([0.0072372 , 0.00773478, 0.00684214, 0.00718331, 0.00712729,\n",
              "        0.00699091, 0.00769711, 0.00689459, 0.00705814, 0.00720692]),\n",
              " 'test_accuracy': array([0.81907895, 0.85526316, 0.81578947, 0.82236842, 0.82565789,\n",
              "        0.86842105, 0.82838284, 0.88118812, 0.84488449, 0.81188119]),\n",
              " 'test_precision_macro': array([0.87039766, 0.89517225, 0.85739361, 0.87291852, 0.86364087,\n",
              "        0.90733316, 0.87474806, 0.90678186, 0.8695477 , 0.86606923]),\n",
              " 'test_recall_macro': array([0.81809211, 0.85489819, 0.81516691, 0.81980907, 0.82229692,\n",
              "        0.86807394, 0.82729256, 0.88061885, 0.84435448, 0.81036044]),\n",
              " 'test_f1_macro': array([0.82836162, 0.86259369, 0.82195311, 0.8295467 , 0.82720861,\n",
              "        0.87685099, 0.83660031, 0.88563432, 0.84788203, 0.82081663])}"
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "print(\"test acc: \",scores['test_accuracy'].mean())\n",
        "print(\"test_precision_macro: \",scores['test_precision_macro'].mean())\n",
        "print(\"test_recall_macro: \",scores['test_recall_macro'].mean())\n",
        "print(\"test_f1_macro: \",scores['test_f1_macro'].mean())\n",
        "myscore = make_scorer(roc_auc_score, multi_class='ovr',needs_proba=True)\n",
        "suc = cross_validate(cv_logreg, train_vectors, y_train, cv=10, scoring = myscore)\n",
        "print(\"ROC AUC score: \",suc['test_score'].mean())"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "NteEugcVxxV6",
        "outputId": "500938c2-8d35-425d-fa2b-c47db174f491"
      },
      "execution_count": null,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "test acc:  0.8372915581031789\n",
            "test_precision_macro:  0.8784002919294183\n",
            "test_recall_macro:  0.8360963468529258\n",
            "test_f1_macro:  0.8437448020815219\n",
            "ROC AUC score:  0.9791173547242631\n"
          ]
        }
      ]
    }
  ]
}