Untitled1.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "a83ca249-1cb6-43b1-9272-d34b75b5a139",
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sb\n",
    "import numpy as np\n",
    "from scipy import stats\n",
    "from sklearn.preprocessing import PowerTransformer\n",
    "from sklearn.feature_selection import VarianceThreshold\n",
    "from sklearn.impute import KNNImputer"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "0c74fb23-0187-4001-82ba-87e532b78deb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>account_number</th>\n",
       "      <th>bad_flag</th>\n",
       "      <th>onus_attribute_1</th>\n",
       "      <th>transaction_attribute_1</th>\n",
       "      <th>transaction_attribute_2</th>\n",
       "      <th>transaction_attribute_3</th>\n",
       "      <th>transaction_attribute_4</th>\n",
       "      <th>transaction_attribute_5</th>\n",
       "      <th>transaction_attribute_6</th>\n",
       "      <th>transaction_attribute_7</th>\n",
       "      <th>...</th>\n",
       "      <th>bureau_enquiry_47</th>\n",
       "      <th>bureau_enquiry_48</th>\n",
       "      <th>bureau_enquiry_49</th>\n",
       "      <th>bureau_enquiry_50</th>\n",
       "      <th>onus_attribute_43</th>\n",
       "      <th>onus_attribute_44</th>\n",
       "      <th>onus_attribute_45</th>\n",
       "      <th>onus_attribute_46</th>\n",
       "      <th>onus_attribute_47</th>\n",
       "      <th>onus_attribute_48</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>221000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>25000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>8.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>0</td>\n",
       "      <td>86000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>30.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>215000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>70000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>25.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>7</td>\n",
       "      <td>0</td>\n",
       "      <td>90000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>8</td>\n",
       "      <td>0</td>\n",
       "      <td>47000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>105000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>15.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>10</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>45.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>6.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>11</th>\n",
       "      <td>12</td>\n",
       "      <td>0</td>\n",
       "      <td>80000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>17.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>12</th>\n",
       "      <td>13</td>\n",
       "      <td>0</td>\n",
       "      <td>300000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>13</th>\n",
       "      <td>14</td>\n",
       "      <td>0</td>\n",
       "      <td>61000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>14</th>\n",
       "      <td>15</td>\n",
       "      <td>0</td>\n",
       "      <td>55000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>31.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>15</th>\n",
       "      <td>16</td>\n",
       "      <td>0</td>\n",
       "      <td>500000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>16</th>\n",
       "      <td>17</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>2.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>17</th>\n",
       "      <td>18</td>\n",
       "      <td>0</td>\n",
       "      <td>95000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>13.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>18</th>\n",
       "      <td>19</td>\n",
       "      <td>0</td>\n",
       "      <td>100000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>19</th>\n",
       "      <td>20</td>\n",
       "      <td>0</td>\n",
       "      <td>32000.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>12.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>20 rows × 1216 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "    account_number  bad_flag  onus_attribute_1  transaction_attribute_1  \\\n",
       "0                1         0               NaN                      NaN   \n",
       "1                2         0          221000.0                      0.0   \n",
       "2                3         0           25000.0                      0.0   \n",
       "3                4         0           86000.0                      0.0   \n",
       "4                5         0          215000.0                      0.0   \n",
       "5                6         0           70000.0                      0.0   \n",
       "6                7         0           90000.0                      0.0   \n",
       "7                8         0           47000.0                      0.0   \n",
       "8                9         0          105000.0                      0.0   \n",
       "9               10         0               NaN                      NaN   \n",
       "10              11         0               NaN                      NaN   \n",
       "11              12         0           80000.0                      0.0   \n",
       "12              13         0          300000.0                      0.0   \n",
       "13              14         0           61000.0                      0.0   \n",
       "14              15         0           55000.0                      0.0   \n",
       "15              16         0          500000.0                      0.0   \n",
       "16              17         0               NaN                      NaN   \n",
       "17              18         0           95000.0                      0.0   \n",
       "18              19         0          100000.0                      0.0   \n",
       "19              20         0           32000.0                      0.0   \n",
       "\n",
       "    transaction_attribute_2  transaction_attribute_3  transaction_attribute_4  \\\n",
       "0                       NaN                      NaN                      NaN   \n",
       "1                       0.0                      0.0                      0.0   \n",
       "2                       0.0                      0.0                      0.0   \n",
       "3                       0.0                      0.0                      0.0   \n",
       "4                       0.0                      0.0                      0.0   \n",
       "5                       0.0                      0.0                      0.0   \n",
       "6                       0.0                      0.0                      0.0   \n",
       "7                       0.0                      0.0                      0.0   \n",
       "8                       0.0                      0.0                      0.0   \n",
       "9                       NaN                      NaN                      NaN   \n",
       "10                      NaN                      NaN                      NaN   \n",
       "11                      0.0                      0.0                      0.0   \n",
       "12                      0.0                      0.0                      0.0   \n",
       "13                      0.0                      0.0                      0.0   \n",
       "14                      0.0                      0.0                      0.0   \n",
       "15                      0.0                      0.0                      0.0   \n",
       "16                      NaN                      NaN                      NaN   \n",
       "17                      0.0                      0.0                      0.0   \n",
       "18                      0.0                      0.0                      0.0   \n",
       "19                      0.0                      0.0                      0.0   \n",
       "\n",
       "    transaction_attribute_5  transaction_attribute_6  transaction_attribute_7  \\\n",
       "0                       NaN                      NaN                      NaN   \n",
       "1                       0.0                      0.0                      0.0   \n",
       "2                       0.0                      0.0                      0.0   \n",
       "3                       0.0                      0.0                      0.0   \n",
       "4                       0.0                      0.0                      0.0   \n",
       "5                       0.0                      0.0                      0.0   \n",
       "6                       0.0                      0.0                      0.0   \n",
       "7                       0.0                      0.0                      0.0   \n",
       "8                       0.0                      0.0                      0.0   \n",
       "9                       NaN                      NaN                      NaN   \n",
       "10                      NaN                      NaN                      NaN   \n",
       "11                      0.0                      0.0                      0.0   \n",
       "12                      0.0                      0.0                      0.0   \n",
       "13                      0.0                      0.0                      0.0   \n",
       "14                      0.0                      0.0                      0.0   \n",
       "15                      0.0                      0.0                      0.0   \n",
       "16                      NaN                      NaN                      NaN   \n",
       "17                      0.0                      0.0                      0.0   \n",
       "18                      0.0                      0.0                      0.0   \n",
       "19                      0.0                      0.0                      0.0   \n",
       "\n",
       "    ...  bureau_enquiry_47  bureau_enquiry_48  bureau_enquiry_49  \\\n",
       "0   ...                0.0                0.0                0.0   \n",
       "1   ...                0.0                0.0                2.0   \n",
       "2   ...                0.0                0.0                0.0   \n",
       "3   ...                0.0                0.0                0.0   \n",
       "4   ...                0.0                0.0                0.0   \n",
       "5   ...                0.0                0.0                0.0   \n",
       "6   ...                0.0                0.0                0.0   \n",
       "7   ...                0.0                0.0                0.0   \n",
       "8   ...                0.0                1.0                0.0   \n",
       "9   ...                0.0                0.0                0.0   \n",
       "10  ...                0.0                0.0                0.0   \n",
       "11  ...                0.0                0.0                0.0   \n",
       "12  ...                0.0                0.0                0.0   \n",
       "13  ...                0.0                2.0                0.0   \n",
       "14  ...                0.0                0.0                0.0   \n",
       "15  ...                0.0                0.0                0.0   \n",
       "16  ...                0.0                2.0                0.0   \n",
       "17  ...                0.0                0.0                0.0   \n",
       "18  ...                0.0                0.0                0.0   \n",
       "19  ...                0.0                0.0                0.0   \n",
       "\n",
       "    bureau_enquiry_50  onus_attribute_43  onus_attribute_44  \\\n",
       "0                 1.0                NaN                NaN   \n",
       "1                 3.0                0.0                0.0   \n",
       "2                 8.0                NaN                NaN   \n",
       "3                30.0                NaN                NaN   \n",
       "4                 1.0                NaN                NaN   \n",
       "5                25.0                NaN                NaN   \n",
       "6                 6.0                NaN                NaN   \n",
       "7                 5.0                NaN                NaN   \n",
       "8                15.0                NaN                NaN   \n",
       "9                45.0                NaN                NaN   \n",
       "10                6.0                NaN                NaN   \n",
       "11               17.0                NaN                NaN   \n",
       "12                2.0                NaN                NaN   \n",
       "13                1.0                NaN                NaN   \n",
       "14               31.0                NaN                NaN   \n",
       "15                3.0                NaN                NaN   \n",
       "16                4.0                NaN                NaN   \n",
       "17               13.0                0.0                1.0   \n",
       "18                0.0                NaN                NaN   \n",
       "19               12.0                NaN                NaN   \n",
       "\n",
       "    onus_attribute_45  onus_attribute_46  onus_attribute_47  onus_attribute_48  \n",
       "0                 NaN                NaN                NaN                NaN  \n",
       "1                 0.0                0.0                0.0                0.0  \n",
       "2                 NaN                NaN                NaN                NaN  \n",
       "3                 NaN                NaN                NaN                NaN  \n",
       "4                 NaN                NaN                NaN                NaN  \n",
       "5                 NaN                NaN                NaN                NaN  \n",
       "6                 NaN                NaN                NaN                NaN  \n",
       "7                 NaN                NaN                NaN                NaN  \n",
       "8                 NaN                NaN                NaN                NaN  \n",
       "9                 NaN                NaN                NaN                NaN  \n",
       "10                NaN                NaN                NaN                NaN  \n",
       "11                NaN                NaN                NaN                NaN  \n",
       "12                NaN                NaN                NaN                NaN  \n",
       "13                NaN                NaN                NaN                NaN  \n",
       "14                NaN                NaN                NaN                NaN  \n",
       "15                NaN                NaN                NaN                NaN  \n",
       "16                NaN                NaN                NaN                NaN  \n",
       "17                0.0                1.0                0.0                0.0  \n",
       "18                NaN                NaN                NaN                NaN  \n",
       "19                NaN                NaN                NaN                NaN  \n",
       "\n",
       "[20 rows x 1216 columns]"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Load data\n",
    "dev_dataset = pd.read_csv(\"train.csv\")\n",
    "dev_dataset.head(20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "240f05e8-e3ce-4750-bf5c-32c32bd92644",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 96806 entries, 0 to 96805\n",
      "Data columns (total 1216 columns):\n",
      " #     Column                     Non-Null Count  Dtype  \n",
      "---    ------                     --------------  -----  \n",
      " 0     account_number             96806 non-null  int64  \n",
      " 1     bad_flag                   96806 non-null  int64  \n",
      " 2     onus_attribute_1           71575 non-null  float64\n",
      " 3     transaction_attribute_1    71575 non-null  float64\n",
      " 4     transaction_attribute_2    71575 non-null  float64\n",
      " 5     transaction_attribute_3    71575 non-null  float64\n",
      " 6     transaction_attribute_4    71575 non-null  float64\n",
      " 7     transaction_attribute_5    71575 non-null  float64\n",
      " 8     transaction_attribute_6    71575 non-null  float64\n",
      " 9     transaction_attribute_7    71575 non-null  float64\n",
      " 10    transaction_attribute_8    71575 non-null  float64\n",
      " 11    transaction_attribute_9    71575 non-null  float64\n",
      " 12    transaction_attribute_10   71575 non-null  float64\n",
      " 13    transaction_attribute_11   71575 non-null  float64\n",
      " 14    transaction_attribute_12   71575 non-null  float64\n",
      " 15    transaction_attribute_13   71575 non-null  float64\n",
      " 16    transaction_attribute_14   71575 non-null  float64\n",
      " 17    transaction_attribute_15   71575 non-null  float64\n",
      " 18    transaction_attribute_16   71575 non-null  float64\n",
      " 19    transaction_attribute_17   71575 non-null  float64\n",
      " 20    transaction_attribute_18   71575 non-null  float64\n",
      " 21    transaction_attribute_19   71575 non-null  float64\n",
      " 22    transaction_attribute_20   71575 non-null  float64\n",
      " 23    transaction_attribute_21   71575 non-null  float64\n",
      " 24    transaction_attribute_22   71575 non-null  float64\n",
      " 25    transaction_attribute_23   71575 non-null  float64\n",
      " 26    transaction_attribute_24   71575 non-null  float64\n",
      " 27    transaction_attribute_25   71575 non-null  float64\n",
      " 28    transaction_attribute_26   71575 non-null  float64\n",
      " 29    transaction_attribute_27   71575 non-null  float64\n",
      " 30    transaction_attribute_28   71575 non-null  float64\n",
      " 31    transaction_attribute_29   71575 non-null  float64\n",
      " 32    transaction_attribute_30   71575 non-null  float64\n",
      " 33    transaction_attribute_31   71575 non-null  float64\n",
      " 34    transaction_attribute_32   71575 non-null  float64\n",
      " 35    transaction_attribute_33   71575 non-null  float64\n",
      " 36    transaction_attribute_34   71575 non-null  float64\n",
      " 37    transaction_attribute_35   71575 non-null  float64\n",
      " 38    transaction_attribute_36   71575 non-null  float64\n",
      " 39    transaction_attribute_37   71575 non-null  float64\n",
      " 40    transaction_attribute_38   71575 non-null  float64\n",
      " 41    transaction_attribute_39   71575 non-null  float64\n",
      " 42    transaction_attribute_40   71575 non-null  float64\n",
      " 43    transaction_attribute_41   71575 non-null  float64\n",
      " 44    transaction_attribute_42   71575 non-null  float64\n",
      " 45    transaction_attribute_43   71575 non-null  float64\n",
      " 46    transaction_attribute_44   71575 non-null  float64\n",
      " 47    transaction_attribute_45   71575 non-null  float64\n",
      " 48    transaction_attribute_46   71575 non-null  float64\n",
      " 49    transaction_attribute_47   71575 non-null  float64\n",
      " 50    transaction_attribute_48   71575 non-null  float64\n",
      " 51    transaction_attribute_49   71575 non-null  float64\n",
      " 52    transaction_attribute_50   71575 non-null  float64\n",
      " 53    transaction_attribute_51   71575 non-null  float64\n",
      " 54    transaction_attribute_52   71575 non-null  float64\n",
      " 55    transaction_attribute_53   71575 non-null  float64\n",
      " 56    transaction_attribute_54   71575 non-null  float64\n",
      " 57    transaction_attribute_55   71575 non-null  float64\n",
      " 58    transaction_attribute_56   71575 non-null  float64\n",
      " 59    transaction_attribute_57   71575 non-null  float64\n",
      " 60    transaction_attribute_58   71575 non-null  float64\n",
      " 61    transaction_attribute_59   71575 non-null  float64\n",
      " 62    transaction_attribute_60   71575 non-null  float64\n",
      " 63    transaction_attribute_61   71575 non-null  float64\n",
      " 64    transaction_attribute_62   71575 non-null  float64\n",
      " 65    transaction_attribute_63   71575 non-null  float64\n",
      " 66    transaction_attribute_64   71575 non-null  float64\n",
      " 67    transaction_attribute_65   71575 non-null  float64\n",
      " 68    transaction_attribute_66   71575 non-null  float64\n",
      " 69    transaction_attribute_67   71575 non-null  float64\n",
      " 70    transaction_attribute_68   71575 non-null  float64\n",
      " 71    transaction_attribute_69   71575 non-null  float64\n",
      " 72    transaction_attribute_70   71575 non-null  float64\n",
      " 73    transaction_attribute_71   71575 non-null  float64\n",
      " 74    transaction_attribute_72   71575 non-null  float64\n",
      " 75    transaction_attribute_73   71575 non-null  float64\n",
      " 76    transaction_attribute_74   71575 non-null  float64\n",
      " 77    transaction_attribute_75   71575 non-null  float64\n",
      " 78    transaction_attribute_76   71575 non-null  float64\n",
      " 79    transaction_attribute_77   71575 non-null  float64\n",
      " 80    transaction_attribute_78   71575 non-null  float64\n",
      " 81    transaction_attribute_79   71575 non-null  float64\n",
      " 82    transaction_attribute_80   71575 non-null  float64\n",
      " 83    transaction_attribute_81   71575 non-null  float64\n",
      " 84    transaction_attribute_82   71575 non-null  float64\n",
      " 85    transaction_attribute_83   71575 non-null  float64\n",
      " 86    transaction_attribute_84   71575 non-null  float64\n",
      " 87    transaction_attribute_85   71575 non-null  float64\n",
      " 88    transaction_attribute_86   71575 non-null  float64\n",
      " 89    transaction_attribute_87   71575 non-null  float64\n",
      " 90    transaction_attribute_88   71575 non-null  float64\n",
      " 91    transaction_attribute_89   71575 non-null  float64\n",
      " 92    transaction_attribute_90   71575 non-null  float64\n",
      " 93    transaction_attribute_91   71575 non-null  float64\n",
      " 94    transaction_attribute_92   71575 non-null  float64\n",
      " 95    transaction_attribute_93   71575 non-null  float64\n",
      " 96    transaction_attribute_94   71575 non-null  float64\n",
      " 97    transaction_attribute_95   71575 non-null  float64\n",
      " 98    transaction_attribute_96   71575 non-null  float64\n",
      " 99    transaction_attribute_97   71575 non-null  float64\n",
      " 100   transaction_attribute_98   71575 non-null  float64\n",
      " 101   transaction_attribute_99   71575 non-null  float64\n",
      " 102   transaction_attribute_100  71575 non-null  float64\n",
      " 103   transaction_attribute_101  71575 non-null  float64\n",
      " 104   transaction_attribute_102  71575 non-null  float64\n",
      " 105   transaction_attribute_103  71575 non-null  float64\n",
      " 106   transaction_attribute_104  71575 non-null  float64\n",
      " 107   transaction_attribute_105  71575 non-null  float64\n",
      " 108   transaction_attribute_106  71575 non-null  float64\n",
      " 109   transaction_attribute_107  71575 non-null  float64\n",
      " 110   transaction_attribute_108  71575 non-null  float64\n",
      " 111   transaction_attribute_109  71575 non-null  float64\n",
      " 112   transaction_attribute_110  71575 non-null  float64\n",
      " 113   transaction_attribute_111  71575 non-null  float64\n",
      " 114   transaction_attribute_112  71575 non-null  float64\n",
      " 115   transaction_attribute_113  71575 non-null  float64\n",
      " 116   transaction_attribute_114  71575 non-null  float64\n",
      " 117   transaction_attribute_115  71575 non-null  float64\n",
      " 118   transaction_attribute_116  71575 non-null  float64\n",
      " 119   transaction_attribute_117  71575 non-null  float64\n",
      " 120   transaction_attribute_118  71366 non-null  float64\n",
      " 121   transaction_attribute_119  71366 non-null  float64\n",
      " 122   transaction_attribute_120  71366 non-null  float64\n",
      " 123   transaction_attribute_121  71366 non-null  float64\n",
      " 124   transaction_attribute_122  71366 non-null  float64\n",
      " 125   transaction_attribute_123  71366 non-null  float64\n",
      " 126   transaction_attribute_124  71366 non-null  float64\n",
      " 127   transaction_attribute_125  71366 non-null  float64\n",
      " 128   transaction_attribute_126  71366 non-null  float64\n",
      " 129   transaction_attribute_127  71366 non-null  float64\n",
      " 130   transaction_attribute_128  71366 non-null  float64\n",
      " 131   transaction_attribute_129  71366 non-null  float64\n",
      " 132   transaction_attribute_130  71366 non-null  float64\n",
      " 133   transaction_attribute_131  71366 non-null  float64\n",
      " 134   transaction_attribute_132  71366 non-null  float64\n",
      " 135   transaction_attribute_133  71366 non-null  float64\n",
      " 136   transaction_attribute_134  71366 non-null  float64\n",
      " 137   transaction_attribute_135  71366 non-null  float64\n",
      " 138   transaction_attribute_136  71366 non-null  float64\n",
      " 139   transaction_attribute_137  71366 non-null  float64\n",
      " 140   transaction_attribute_138  71366 non-null  float64\n",
      " 141   transaction_attribute_139  71366 non-null  float64\n",
      " 142   transaction_attribute_140  71366 non-null  float64\n",
      " 143   transaction_attribute_141  71366 non-null  float64\n",
      " 144   transaction_attribute_142  71366 non-null  float64\n",
      " 145   transaction_attribute_143  71366 non-null  float64\n",
      " 146   transaction_attribute_144  71366 non-null  float64\n",
      " 147   transaction_attribute_145  71366 non-null  float64\n",
      " 148   transaction_attribute_146  71366 non-null  float64\n",
      " 149   transaction_attribute_147  71366 non-null  float64\n",
      " 150   transaction_attribute_148  71366 non-null  float64\n",
      " 151   transaction_attribute_149  71366 non-null  float64\n",
      " 152   transaction_attribute_150  71366 non-null  float64\n",
      " 153   transaction_attribute_151  71366 non-null  float64\n",
      " 154   transaction_attribute_152  71366 non-null  float64\n",
      " 155   transaction_attribute_153  71366 non-null  float64\n",
      " 156   transaction_attribute_154  71366 non-null  float64\n",
      " 157   transaction_attribute_155  71366 non-null  float64\n",
      " 158   transaction_attribute_156  71366 non-null  float64\n",
      " 159   transaction_attribute_157  71575 non-null  float64\n",
      " 160   transaction_attribute_158  71575 non-null  float64\n",
      " 161   transaction_attribute_159  71575 non-null  float64\n",
      " 162   transaction_attribute_160  71575 non-null  float64\n",
      " 163   transaction_attribute_161  71575 non-null  float64\n",
      " 164   transaction_attribute_162  71575 non-null  float64\n",
      " 165   transaction_attribute_163  71575 non-null  float64\n",
      " 166   transaction_attribute_164  71575 non-null  float64\n",
      " 167   transaction_attribute_165  71575 non-null  float64\n",
      " 168   transaction_attribute_166  71575 non-null  float64\n",
      " 169   transaction_attribute_167  71575 non-null  float64\n",
      " 170   transaction_attribute_168  71575 non-null  float64\n",
      " 171   transaction_attribute_169  71575 non-null  float64\n",
      " 172   transaction_attribute_170  71575 non-null  float64\n",
      " 173   transaction_attribute_171  71575 non-null  float64\n",
      " 174   transaction_attribute_172  71575 non-null  float64\n",
      " 175   transaction_attribute_173  71575 non-null  float64\n",
      " 176   transaction_attribute_174  71575 non-null  float64\n",
      " 177   transaction_attribute_175  71575 non-null  float64\n",
      " 178   transaction_attribute_176  71575 non-null  float64\n",
      " 179   transaction_attribute_177  71575 non-null  float64\n",
      " 180   transaction_attribute_178  71575 non-null  float64\n",
      " 181   transaction_attribute_179  71575 non-null  float64\n",
      " 182   transaction_attribute_180  71575 non-null  float64\n",
      " 183   transaction_attribute_181  71575 non-null  float64\n",
      " 184   transaction_attribute_182  71575 non-null  float64\n",
      " 185   transaction_attribute_183  71575 non-null  float64\n",
      " 186   transaction_attribute_184  71575 non-null  float64\n",
      " 187   transaction_attribute_185  71575 non-null  float64\n",
      " 188   transaction_attribute_186  71575 non-null  float64\n",
      " 189   transaction_attribute_187  71575 non-null  float64\n",
      " 190   transaction_attribute_188  71575 non-null  float64\n",
      " 191   transaction_attribute_189  71575 non-null  float64\n",
      " 192   transaction_attribute_190  71575 non-null  float64\n",
      " 193   transaction_attribute_191  71575 non-null  float64\n",
      " 194   transaction_attribute_192  71575 non-null  float64\n",
      " 195   transaction_attribute_193  71575 non-null  float64\n",
      " 196   transaction_attribute_194  71575 non-null  float64\n",
      " 197   transaction_attribute_195  71575 non-null  float64\n",
      " 198   transaction_attribute_196  71575 non-null  float64\n",
      " 199   transaction_attribute_197  71575 non-null  float64\n",
      " 200   transaction_attribute_198  71575 non-null  float64\n",
      " 201   transaction_attribute_199  71575 non-null  float64\n",
      " 202   transaction_attribute_200  71575 non-null  float64\n",
      " 203   transaction_attribute_201  71575 non-null  float64\n",
      " 204   transaction_attribute_202  71575 non-null  float64\n",
      " 205   transaction_attribute_203  71575 non-null  float64\n",
      " 206   transaction_attribute_204  71575 non-null  float64\n",
      " 207   transaction_attribute_205  71575 non-null  float64\n",
      " 208   transaction_attribute_206  71575 non-null  float64\n",
      " 209   transaction_attribute_207  71575 non-null  float64\n",
      " 210   transaction_attribute_208  71575 non-null  float64\n",
      " 211   transaction_attribute_209  71575 non-null  float64\n",
      " 212   transaction_attribute_210  71575 non-null  float64\n",
      " 213   transaction_attribute_211  71575 non-null  float64\n",
      " 214   transaction_attribute_212  71575 non-null  float64\n",
      " 215   transaction_attribute_213  71575 non-null  float64\n",
      " 216   transaction_attribute_214  71575 non-null  float64\n",
      " 217   transaction_attribute_215  71575 non-null  float64\n",
      " 218   transaction_attribute_216  71575 non-null  float64\n",
      " 219   transaction_attribute_217  71575 non-null  float64\n",
      " 220   transaction_attribute_218  71575 non-null  float64\n",
      " 221   transaction_attribute_219  71575 non-null  float64\n",
      " 222   transaction_attribute_220  71575 non-null  float64\n",
      " 223   transaction_attribute_221  71575 non-null  float64\n",
      " 224   transaction_attribute_222  71575 non-null  float64\n",
      " 225   transaction_attribute_223  71575 non-null  float64\n",
      " 226   transaction_attribute_224  71575 non-null  float64\n",
      " 227   transaction_attribute_225  71575 non-null  float64\n",
      " 228   transaction_attribute_226  71575 non-null  float64\n",
      " 229   transaction_attribute_227  71575 non-null  float64\n",
      " 230   transaction_attribute_228  71575 non-null  float64\n",
      " 231   transaction_attribute_229  71575 non-null  float64\n",
      " 232   transaction_attribute_230  71575 non-null  float64\n",
      " 233   transaction_attribute_231  71575 non-null  float64\n",
      " 234   transaction_attribute_232  71575 non-null  float64\n",
      " 235   transaction_attribute_233  71575 non-null  float64\n",
      " 236   transaction_attribute_234  71575 non-null  float64\n",
      " 237   transaction_attribute_235  71575 non-null  float64\n",
      " 238   transaction_attribute_236  71575 non-null  float64\n",
      " 239   transaction_attribute_237  71575 non-null  float64\n",
      " 240   transaction_attribute_238  71575 non-null  float64\n",
      " 241   transaction_attribute_239  71575 non-null  float64\n",
      " 242   transaction_attribute_240  71575 non-null  float64\n",
      " 243   transaction_attribute_241  71575 non-null  float64\n",
      " 244   transaction_attribute_242  71575 non-null  float64\n",
      " 245   transaction_attribute_243  71575 non-null  float64\n",
      " 246   transaction_attribute_244  71575 non-null  float64\n",
      " 247   transaction_attribute_245  71575 non-null  float64\n",
      " 248   transaction_attribute_246  71575 non-null  float64\n",
      " 249   transaction_attribute_247  71575 non-null  float64\n",
      " 250   transaction_attribute_248  71575 non-null  float64\n",
      " 251   transaction_attribute_249  71575 non-null  float64\n",
      " 252   transaction_attribute_250  71575 non-null  float64\n",
      " 253   transaction_attribute_251  71575 non-null  float64\n",
      " 254   transaction_attribute_252  71575 non-null  float64\n",
      " 255   transaction_attribute_253  71575 non-null  float64\n",
      " 256   transaction_attribute_254  71575 non-null  float64\n",
      " 257   transaction_attribute_255  71575 non-null  float64\n",
      " 258   transaction_attribute_256  71575 non-null  float64\n",
      " 259   transaction_attribute_257  71575 non-null  float64\n",
      " 260   transaction_attribute_258  71575 non-null  float64\n",
      " 261   transaction_attribute_259  71575 non-null  float64\n",
      " 262   transaction_attribute_260  71575 non-null  float64\n",
      " 263   transaction_attribute_261  71575 non-null  float64\n",
      " 264   transaction_attribute_262  71575 non-null  float64\n",
      " 265   transaction_attribute_263  71575 non-null  float64\n",
      " 266   transaction_attribute_264  71575 non-null  float64\n",
      " 267   transaction_attribute_265  71575 non-null  float64\n",
      " 268   transaction_attribute_266  71575 non-null  float64\n",
      " 269   transaction_attribute_267  71575 non-null  float64\n",
      " 270   transaction_attribute_268  71575 non-null  float64\n",
      " 271   transaction_attribute_269  71575 non-null  float64\n",
      " 272   transaction_attribute_270  71575 non-null  float64\n",
      " 273   transaction_attribute_271  71575 non-null  float64\n",
      " 274   transaction_attribute_272  71575 non-null  float64\n",
      " 275   transaction_attribute_273  71575 non-null  float64\n",
      " 276   transaction_attribute_274  71575 non-null  float64\n",
      " 277   transaction_attribute_275  71575 non-null  float64\n",
      " 278   transaction_attribute_276  71575 non-null  float64\n",
      " 279   transaction_attribute_277  71575 non-null  float64\n",
      " 280   transaction_attribute_278  71575 non-null  float64\n",
      " 281   transaction_attribute_279  71575 non-null  float64\n",
      " 282   transaction_attribute_280  71575 non-null  float64\n",
      " 283   transaction_attribute_281  71575 non-null  float64\n",
      " 284   transaction_attribute_282  71575 non-null  float64\n",
      " 285   transaction_attribute_283  71575 non-null  float64\n",
      " 286   transaction_attribute_284  71575 non-null  float64\n",
      " 287   transaction_attribute_285  71575 non-null  float64\n",
      " 288   transaction_attribute_286  71575 non-null  float64\n",
      " 289   transaction_attribute_287  71575 non-null  float64\n",
      " 290   transaction_attribute_288  71575 non-null  float64\n",
      " 291   transaction_attribute_289  71575 non-null  float64\n",
      " 292   transaction_attribute_290  71575 non-null  float64\n",
      " 293   transaction_attribute_291  71575 non-null  float64\n",
      " 294   transaction_attribute_292  71575 non-null  float64\n",
      " 295   transaction_attribute_293  71575 non-null  float64\n",
      " 296   transaction_attribute_294  71575 non-null  float64\n",
      " 297   transaction_attribute_295  71575 non-null  float64\n",
      " 298   transaction_attribute_296  71575 non-null  float64\n",
      " 299   transaction_attribute_297  71575 non-null  float64\n",
      " 300   transaction_attribute_298  71575 non-null  float64\n",
      " 301   transaction_attribute_299  71575 non-null  float64\n",
      " 302   transaction_attribute_300  71575 non-null  float64\n",
      " 303   transaction_attribute_301  71575 non-null  float64\n",
      " 304   transaction_attribute_302  71575 non-null  float64\n",
      " 305   transaction_attribute_303  71575 non-null  float64\n",
      " 306   transaction_attribute_304  71575 non-null  float64\n",
      " 307   transaction_attribute_305  71575 non-null  float64\n",
      " 308   transaction_attribute_306  71575 non-null  float64\n",
      " 309   transaction_attribute_307  71575 non-null  float64\n",
      " 310   transaction_attribute_308  71575 non-null  float64\n",
      " 311   transaction_attribute_309  71575 non-null  float64\n",
      " 312   transaction_attribute_310  71575 non-null  float64\n",
      " 313   transaction_attribute_311  71575 non-null  float64\n",
      " 314   transaction_attribute_312  71575 non-null  float64\n",
      " 315   transaction_attribute_313  71575 non-null  float64\n",
      " 316   transaction_attribute_314  71575 non-null  float64\n",
      " 317   transaction_attribute_315  71575 non-null  float64\n",
      " 318   transaction_attribute_316  71575 non-null  float64\n",
      " 319   transaction_attribute_317  71575 non-null  float64\n",
      " 320   transaction_attribute_318  71575 non-null  float64\n",
      " 321   transaction_attribute_319  71575 non-null  float64\n",
      " 322   transaction_attribute_320  71575 non-null  float64\n",
      " 323   transaction_attribute_321  71575 non-null  float64\n",
      " 324   transaction_attribute_322  71575 non-null  float64\n",
      " 325   transaction_attribute_323  71575 non-null  float64\n",
      " 326   transaction_attribute_324  71575 non-null  float64\n",
      " 327   transaction_attribute_325  71575 non-null  float64\n",
      " 328   transaction_attribute_326  71575 non-null  float64\n",
      " 329   transaction_attribute_327  71575 non-null  float64\n",
      " 330   transaction_attribute_328  71575 non-null  float64\n",
      " 331   transaction_attribute_329  71575 non-null  float64\n",
      " 332   transaction_attribute_330  71575 non-null  float64\n",
      " 333   transaction_attribute_331  71575 non-null  float64\n",
      " 334   transaction_attribute_332  71575 non-null  float64\n",
      " 335   transaction_attribute_333  71575 non-null  float64\n",
      " 336   transaction_attribute_334  71575 non-null  float64\n",
      " 337   transaction_attribute_335  71575 non-null  float64\n",
      " 338   transaction_attribute_336  71575 non-null  float64\n",
      " 339   transaction_attribute_337  71575 non-null  float64\n",
      " 340   transaction_attribute_338  71575 non-null  float64\n",
      " 341   transaction_attribute_339  71575 non-null  float64\n",
      " 342   transaction_attribute_340  71575 non-null  float64\n",
      " 343   transaction_attribute_341  71575 non-null  float64\n",
      " 344   transaction_attribute_342  71575 non-null  float64\n",
      " 345   transaction_attribute_343  71575 non-null  float64\n",
      " 346   transaction_attribute_344  71575 non-null  float64\n",
      " 347   transaction_attribute_345  71575 non-null  float64\n",
      " 348   transaction_attribute_346  71575 non-null  float64\n",
      " 349   transaction_attribute_347  71575 non-null  float64\n",
      " 350   transaction_attribute_348  71575 non-null  float64\n",
      " 351   transaction_attribute_349  71575 non-null  float64\n",
      " 352   transaction_attribute_350  71575 non-null  float64\n",
      " 353   transaction_attribute_351  71575 non-null  float64\n",
      " 354   transaction_attribute_352  71575 non-null  float64\n",
      " 355   transaction_attribute_353  71575 non-null  float64\n",
      " 356   transaction_attribute_354  71496 non-null  float64\n",
      " 357   transaction_attribute_355  71575 non-null  float64\n",
      " 358   transaction_attribute_356  71575 non-null  float64\n",
      " 359   transaction_attribute_357  71575 non-null  float64\n",
      " 360   transaction_attribute_358  71575 non-null  float64\n",
      " 361   transaction_attribute_359  71496 non-null  float64\n",
      " 362   transaction_attribute_360  71575 non-null  float64\n",
      " 363   transaction_attribute_361  71575 non-null  float64\n",
      " 364   transaction_attribute_362  71575 non-null  float64\n",
      " 365   transaction_attribute_363  71575 non-null  float64\n",
      " 366   transaction_attribute_364  71496 non-null  float64\n",
      " 367   transaction_attribute_365  71575 non-null  float64\n",
      " 368   transaction_attribute_366  71575 non-null  float64\n",
      " 369   transaction_attribute_367  71575 non-null  float64\n",
      " 370   transaction_attribute_368  71575 non-null  float64\n",
      " 371   transaction_attribute_369  71496 non-null  float64\n",
      " 372   transaction_attribute_370  71575 non-null  float64\n",
      " 373   transaction_attribute_371  71575 non-null  float64\n",
      " 374   transaction_attribute_372  71575 non-null  float64\n",
      " 375   transaction_attribute_373  71575 non-null  float64\n",
      " 376   transaction_attribute_374  71496 non-null  float64\n",
      " 377   transaction_attribute_375  71575 non-null  float64\n",
      " 378   transaction_attribute_376  71575 non-null  float64\n",
      " 379   transaction_attribute_377  71575 non-null  float64\n",
      " 380   transaction_attribute_378  71575 non-null  float64\n",
      " 381   transaction_attribute_379  71496 non-null  float64\n",
      " 382   transaction_attribute_380  71575 non-null  float64\n",
      " 383   transaction_attribute_381  71575 non-null  float64\n",
      " 384   transaction_attribute_382  71575 non-null  float64\n",
      " 385   transaction_attribute_383  71575 non-null  float64\n",
      " 386   transaction_attribute_384  71496 non-null  float64\n",
      " 387   transaction_attribute_385  71575 non-null  float64\n",
      " 388   transaction_attribute_386  71575 non-null  float64\n",
      " 389   transaction_attribute_387  71575 non-null  float64\n",
      " 390   transaction_attribute_388  71575 non-null  float64\n",
      " 391   transaction_attribute_389  71496 non-null  float64\n",
      " 392   transaction_attribute_390  71575 non-null  float64\n",
      " 393   transaction_attribute_391  71575 non-null  float64\n",
      " 394   transaction_attribute_392  71575 non-null  float64\n",
      " 395   transaction_attribute_393  71575 non-null  float64\n",
      " 396   transaction_attribute_394  71496 non-null  float64\n",
      " 397   transaction_attribute_395  71575 non-null  float64\n",
      " 398   transaction_attribute_396  71575 non-null  float64\n",
      " 399   transaction_attribute_397  71575 non-null  float64\n",
      " 400   transaction_attribute_398  71575 non-null  float64\n",
      " 401   transaction_attribute_399  71496 non-null  float64\n",
      " 402   transaction_attribute_400  71575 non-null  float64\n",
      " 403   transaction_attribute_401  71575 non-null  float64\n",
      " 404   transaction_attribute_402  71575 non-null  float64\n",
      " 405   transaction_attribute_403  71575 non-null  float64\n",
      " 406   transaction_attribute_404  71496 non-null  float64\n",
      " 407   transaction_attribute_405  71575 non-null  float64\n",
      " 408   transaction_attribute_406  71575 non-null  float64\n",
      " 409   transaction_attribute_407  71575 non-null  float64\n",
      " 410   transaction_attribute_408  71575 non-null  float64\n",
      " 411   transaction_attribute_409  71496 non-null  float64\n",
      " 412   transaction_attribute_410  71575 non-null  float64\n",
      " 413   transaction_attribute_411  71575 non-null  float64\n",
      " 414   transaction_attribute_412  71575 non-null  float64\n",
      " 415   transaction_attribute_413  71575 non-null  float64\n",
      " 416   transaction_attribute_414  71496 non-null  float64\n",
      " 417   transaction_attribute_415  71575 non-null  float64\n",
      " 418   transaction_attribute_416  71575 non-null  float64\n",
      " 419   transaction_attribute_417  71575 non-null  float64\n",
      " 420   transaction_attribute_418  71575 non-null  float64\n",
      " 421   transaction_attribute_419  71496 non-null  float64\n",
      " 422   transaction_attribute_420  71575 non-null  float64\n",
      " 423   transaction_attribute_421  71575 non-null  float64\n",
      " 424   transaction_attribute_422  71575 non-null  float64\n",
      " 425   transaction_attribute_423  71575 non-null  float64\n",
      " 426   transaction_attribute_424  71496 non-null  float64\n",
      " 427   transaction_attribute_425  71575 non-null  float64\n",
      " 428   transaction_attribute_426  71575 non-null  float64\n",
      " 429   transaction_attribute_427  71575 non-null  float64\n",
      " 430   transaction_attribute_428  71575 non-null  float64\n",
      " 431   transaction_attribute_429  71496 non-null  float64\n",
      " 432   transaction_attribute_430  71575 non-null  float64\n",
      " 433   transaction_attribute_431  71575 non-null  float64\n",
      " 434   transaction_attribute_432  71575 non-null  float64\n",
      " 435   transaction_attribute_433  71575 non-null  float64\n",
      " 436   transaction_attribute_434  71496 non-null  float64\n",
      " 437   transaction_attribute_435  71575 non-null  float64\n",
      " 438   transaction_attribute_436  71575 non-null  float64\n",
      " 439   transaction_attribute_437  71575 non-null  float64\n",
      " 440   transaction_attribute_438  71575 non-null  float64\n",
      " 441   transaction_attribute_439  71496 non-null  float64\n",
      " 442   transaction_attribute_440  71575 non-null  float64\n",
      " 443   transaction_attribute_441  71575 non-null  float64\n",
      " 444   transaction_attribute_442  71575 non-null  float64\n",
      " 445   transaction_attribute_443  71575 non-null  float64\n",
      " 446   transaction_attribute_444  71496 non-null  float64\n",
      " 447   transaction_attribute_445  71575 non-null  float64\n",
      " 448   transaction_attribute_446  71575 non-null  float64\n",
      " 449   transaction_attribute_447  71575 non-null  float64\n",
      " 450   transaction_attribute_448  71575 non-null  float64\n",
      " 451   transaction_attribute_449  71496 non-null  float64\n",
      " 452   transaction_attribute_450  71575 non-null  float64\n",
      " 453   transaction_attribute_451  71575 non-null  float64\n",
      " 454   transaction_attribute_452  71575 non-null  float64\n",
      " 455   transaction_attribute_453  71575 non-null  float64\n",
      " 456   transaction_attribute_454  71496 non-null  float64\n",
      " 457   transaction_attribute_455  71575 non-null  float64\n",
      " 458   transaction_attribute_456  71575 non-null  float64\n",
      " 459   transaction_attribute_457  71575 non-null  float64\n",
      " 460   transaction_attribute_458  71575 non-null  float64\n",
      " 461   transaction_attribute_459  71496 non-null  float64\n",
      " 462   transaction_attribute_460  71575 non-null  float64\n",
      " 463   transaction_attribute_461  71575 non-null  float64\n",
      " 464   transaction_attribute_462  71575 non-null  float64\n",
      " 465   transaction_attribute_463  71575 non-null  float64\n",
      " 466   transaction_attribute_464  71496 non-null  float64\n",
      " 467   transaction_attribute_465  71575 non-null  float64\n",
      " 468   transaction_attribute_466  71575 non-null  float64\n",
      " 469   transaction_attribute_467  71575 non-null  float64\n",
      " 470   transaction_attribute_468  71575 non-null  float64\n",
      " 471   transaction_attribute_469  71496 non-null  float64\n",
      " 472   transaction_attribute_470  71575 non-null  float64\n",
      " 473   transaction_attribute_471  71575 non-null  float64\n",
      " 474   transaction_attribute_472  71575 non-null  float64\n",
      " 475   transaction_attribute_473  71575 non-null  float64\n",
      " 476   transaction_attribute_474  71496 non-null  float64\n",
      " 477   transaction_attribute_475  71575 non-null  float64\n",
      " 478   transaction_attribute_476  71575 non-null  float64\n",
      " 479   transaction_attribute_477  71575 non-null  float64\n",
      " 480   transaction_attribute_478  71575 non-null  float64\n",
      " 481   transaction_attribute_479  71496 non-null  float64\n",
      " 482   transaction_attribute_480  71575 non-null  float64\n",
      " 483   transaction_attribute_481  71575 non-null  float64\n",
      " 484   transaction_attribute_482  71575 non-null  float64\n",
      " 485   transaction_attribute_483  71575 non-null  float64\n",
      " 486   transaction_attribute_484  71496 non-null  float64\n",
      " 487   transaction_attribute_485  71575 non-null  float64\n",
      " 488   transaction_attribute_486  71575 non-null  float64\n",
      " 489   transaction_attribute_487  71575 non-null  float64\n",
      " 490   transaction_attribute_488  71575 non-null  float64\n",
      " 491   transaction_attribute_489  71496 non-null  float64\n",
      " 492   transaction_attribute_490  71575 non-null  float64\n",
      " 493   transaction_attribute_491  71575 non-null  float64\n",
      " 494   transaction_attribute_492  71575 non-null  float64\n",
      " 495   transaction_attribute_493  71575 non-null  float64\n",
      " 496   transaction_attribute_494  71496 non-null  float64\n",
      " 497   transaction_attribute_495  71575 non-null  float64\n",
      " 498   transaction_attribute_496  71575 non-null  float64\n",
      " 499   transaction_attribute_497  71575 non-null  float64\n",
      " 500   transaction_attribute_498  71575 non-null  float64\n",
      " 501   transaction_attribute_499  71496 non-null  float64\n",
      " 502   transaction_attribute_500  71575 non-null  float64\n",
      " 503   transaction_attribute_501  71575 non-null  float64\n",
      " 504   transaction_attribute_502  71575 non-null  float64\n",
      " 505   transaction_attribute_503  71575 non-null  float64\n",
      " 506   transaction_attribute_504  71496 non-null  float64\n",
      " 507   transaction_attribute_505  71575 non-null  float64\n",
      " 508   transaction_attribute_506  71575 non-null  float64\n",
      " 509   transaction_attribute_507  71575 non-null  float64\n",
      " 510   transaction_attribute_508  71575 non-null  float64\n",
      " 511   transaction_attribute_509  71496 non-null  float64\n",
      " 512   transaction_attribute_510  71575 non-null  float64\n",
      " 513   transaction_attribute_511  71575 non-null  float64\n",
      " 514   transaction_attribute_512  71575 non-null  float64\n",
      " 515   transaction_attribute_513  71575 non-null  float64\n",
      " 516   transaction_attribute_514  71496 non-null  float64\n",
      " 517   transaction_attribute_515  71575 non-null  float64\n",
      " 518   transaction_attribute_516  71575 non-null  float64\n",
      " 519   transaction_attribute_517  71575 non-null  float64\n",
      " 520   transaction_attribute_518  71575 non-null  float64\n",
      " 521   transaction_attribute_519  71496 non-null  float64\n",
      " 522   transaction_attribute_520  71575 non-null  float64\n",
      " 523   transaction_attribute_521  71575 non-null  float64\n",
      " 524   transaction_attribute_522  71575 non-null  float64\n",
      " 525   transaction_attribute_523  71575 non-null  float64\n",
      " 526   transaction_attribute_524  71496 non-null  float64\n",
      " 527   transaction_attribute_525  71575 non-null  float64\n",
      " 528   transaction_attribute_526  71575 non-null  float64\n",
      " 529   transaction_attribute_527  71575 non-null  float64\n",
      " 530   transaction_attribute_528  71575 non-null  float64\n",
      " 531   transaction_attribute_529  71496 non-null  float64\n",
      " 532   transaction_attribute_530  71575 non-null  float64\n",
      " 533   transaction_attribute_531  71575 non-null  float64\n",
      " 534   transaction_attribute_532  71575 non-null  float64\n",
      " 535   transaction_attribute_533  71575 non-null  float64\n",
      " 536   transaction_attribute_534  71496 non-null  float64\n",
      " 537   transaction_attribute_535  71575 non-null  float64\n",
      " 538   transaction_attribute_536  71575 non-null  float64\n",
      " 539   transaction_attribute_537  71575 non-null  float64\n",
      " 540   transaction_attribute_538  71575 non-null  float64\n",
      " 541   transaction_attribute_539  71496 non-null  float64\n",
      " 542   transaction_attribute_540  71575 non-null  float64\n",
      " 543   transaction_attribute_541  71575 non-null  float64\n",
      " 544   transaction_attribute_542  71575 non-null  float64\n",
      " 545   transaction_attribute_543  71575 non-null  float64\n",
      " 546   transaction_attribute_544  71496 non-null  float64\n",
      " 547   transaction_attribute_545  71575 non-null  float64\n",
      " 548   transaction_attribute_546  71575 non-null  float64\n",
      " 549   transaction_attribute_547  71575 non-null  float64\n",
      " 550   transaction_attribute_548  71471 non-null  float64\n",
      " 551   transaction_attribute_549  71471 non-null  float64\n",
      " 552   transaction_attribute_550  71471 non-null  float64\n",
      " 553   transaction_attribute_551  71471 non-null  float64\n",
      " 554   transaction_attribute_552  71471 non-null  float64\n",
      " 555   transaction_attribute_553  71471 non-null  float64\n",
      " 556   transaction_attribute_554  71471 non-null  float64\n",
      " 557   transaction_attribute_555  71471 non-null  float64\n",
      " 558   transaction_attribute_556  71471 non-null  float64\n",
      " 559   transaction_attribute_557  71471 non-null  float64\n",
      " 560   transaction_attribute_558  71471 non-null  float64\n",
      " 561   transaction_attribute_559  71471 non-null  float64\n",
      " 562   transaction_attribute_560  71471 non-null  float64\n",
      " 563   transaction_attribute_561  71471 non-null  float64\n",
      " 564   transaction_attribute_562  71471 non-null  float64\n",
      " 565   transaction_attribute_563  71471 non-null  float64\n",
      " 566   transaction_attribute_564  71471 non-null  float64\n",
      " 567   transaction_attribute_565  71471 non-null  float64\n",
      " 568   transaction_attribute_566  71471 non-null  float64\n",
      " 569   transaction_attribute_567  71471 non-null  float64\n",
      " 570   transaction_attribute_568  71471 non-null  float64\n",
      " 571   transaction_attribute_569  71471 non-null  float64\n",
      " 572   transaction_attribute_570  71471 non-null  float64\n",
      " 573   transaction_attribute_571  71471 non-null  float64\n",
      " 574   transaction_attribute_572  71471 non-null  float64\n",
      " 575   transaction_attribute_573  71471 non-null  float64\n",
      " 576   transaction_attribute_574  71471 non-null  float64\n",
      " 577   transaction_attribute_575  71471 non-null  float64\n",
      " 578   transaction_attribute_576  71471 non-null  float64\n",
      " 579   transaction_attribute_577  71471 non-null  float64\n",
      " 580   transaction_attribute_578  71471 non-null  float64\n",
      " 581   transaction_attribute_579  71471 non-null  float64\n",
      " 582   transaction_attribute_580  71471 non-null  float64\n",
      " 583   transaction_attribute_581  71471 non-null  float64\n",
      " 584   transaction_attribute_582  71471 non-null  float64\n",
      " 585   transaction_attribute_583  71471 non-null  float64\n",
      " 586   transaction_attribute_584  71471 non-null  float64\n",
      " 587   transaction_attribute_585  71471 non-null  float64\n",
      " 588   transaction_attribute_586  71471 non-null  float64\n",
      " 589   transaction_attribute_587  71575 non-null  float64\n",
      " 590   transaction_attribute_588  71575 non-null  float64\n",
      " 591   transaction_attribute_589  71575 non-null  float64\n",
      " 592   transaction_attribute_590  71575 non-null  float64\n",
      " 593   transaction_attribute_591  71575 non-null  float64\n",
      " 594   transaction_attribute_592  71575 non-null  float64\n",
      " 595   transaction_attribute_593  71575 non-null  float64\n",
      " 596   transaction_attribute_594  71575 non-null  float64\n",
      " 597   transaction_attribute_595  71575 non-null  float64\n",
      " 598   transaction_attribute_596  71575 non-null  float64\n",
      " 599   transaction_attribute_597  71575 non-null  float64\n",
      " 600   transaction_attribute_598  71575 non-null  float64\n",
      " 601   transaction_attribute_599  71575 non-null  float64\n",
      " 602   transaction_attribute_600  71575 non-null  float64\n",
      " 603   transaction_attribute_601  71575 non-null  float64\n",
      " 604   transaction_attribute_602  71575 non-null  float64\n",
      " 605   transaction_attribute_603  71575 non-null  float64\n",
      " 606   transaction_attribute_604  71575 non-null  float64\n",
      " 607   transaction_attribute_605  71575 non-null  float64\n",
      " 608   transaction_attribute_606  71575 non-null  float64\n",
      " 609   transaction_attribute_607  71575 non-null  float64\n",
      " 610   transaction_attribute_608  71575 non-null  float64\n",
      " 611   transaction_attribute_609  71575 non-null  float64\n",
      " 612   transaction_attribute_610  71575 non-null  float64\n",
      " 613   transaction_attribute_611  71575 non-null  float64\n",
      " 614   transaction_attribute_612  71575 non-null  float64\n",
      " 615   transaction_attribute_613  71575 non-null  float64\n",
      " 616   transaction_attribute_614  71575 non-null  float64\n",
      " 617   transaction_attribute_615  71575 non-null  float64\n",
      " 618   transaction_attribute_616  71575 non-null  float64\n",
      " 619   transaction_attribute_617  71575 non-null  float64\n",
      " 620   transaction_attribute_618  71575 non-null  float64\n",
      " 621   transaction_attribute_619  71575 non-null  float64\n",
      " 622   transaction_attribute_620  71575 non-null  float64\n",
      " 623   transaction_attribute_621  71575 non-null  float64\n",
      " 624   transaction_attribute_622  71575 non-null  float64\n",
      " 625   transaction_attribute_623  71575 non-null  float64\n",
      " 626   transaction_attribute_624  71575 non-null  float64\n",
      " 627   transaction_attribute_625  71575 non-null  float64\n",
      " 628   transaction_attribute_626  71575 non-null  float64\n",
      " 629   transaction_attribute_627  71575 non-null  float64\n",
      " 630   transaction_attribute_628  71575 non-null  float64\n",
      " 631   transaction_attribute_629  71575 non-null  float64\n",
      " 632   transaction_attribute_630  71575 non-null  float64\n",
      " 633   transaction_attribute_631  71575 non-null  float64\n",
      " 634   transaction_attribute_632  71575 non-null  float64\n",
      " 635   transaction_attribute_633  71575 non-null  float64\n",
      " 636   transaction_attribute_634  71575 non-null  float64\n",
      " 637   transaction_attribute_635  71575 non-null  float64\n",
      " 638   transaction_attribute_636  71575 non-null  float64\n",
      " 639   transaction_attribute_637  71575 non-null  float64\n",
      " 640   transaction_attribute_638  71575 non-null  float64\n",
      " 641   transaction_attribute_639  71575 non-null  float64\n",
      " 642   transaction_attribute_640  71575 non-null  float64\n",
      " 643   transaction_attribute_641  71575 non-null  float64\n",
      " 644   transaction_attribute_642  71575 non-null  float64\n",
      " 645   transaction_attribute_643  71575 non-null  float64\n",
      " 646   transaction_attribute_644  71575 non-null  float64\n",
      " 647   transaction_attribute_645  71575 non-null  float64\n",
      " 648   transaction_attribute_646  71575 non-null  float64\n",
      " 649   transaction_attribute_647  71575 non-null  float64\n",
      " 650   transaction_attribute_648  71575 non-null  float64\n",
      " 651   transaction_attribute_649  71575 non-null  float64\n",
      " 652   transaction_attribute_650  71575 non-null  float64\n",
      " 653   transaction_attribute_651  71575 non-null  float64\n",
      " 654   transaction_attribute_652  71575 non-null  float64\n",
      " 655   transaction_attribute_653  71575 non-null  float64\n",
      " 656   transaction_attribute_654  71575 non-null  float64\n",
      " 657   transaction_attribute_655  71575 non-null  float64\n",
      " 658   transaction_attribute_656  71575 non-null  float64\n",
      " 659   transaction_attribute_657  71575 non-null  float64\n",
      " 660   transaction_attribute_658  71575 non-null  float64\n",
      " 661   transaction_attribute_659  71575 non-null  float64\n",
      " 662   transaction_attribute_660  71575 non-null  float64\n",
      " 663   transaction_attribute_661  71575 non-null  float64\n",
      " 664   transaction_attribute_662  71575 non-null  float64\n",
      " 665   transaction_attribute_663  71575 non-null  float64\n",
      " 666   transaction_attribute_664  71575 non-null  float64\n",
      " 667   bureau_1                   95613 non-null  float64\n",
      " 668   bureau_2                   95613 non-null  float64\n",
      " 669   bureau_3                   95613 non-null  float64\n",
      " 670   bureau_4                   95613 non-null  float64\n",
      " 671   bureau_5                   95613 non-null  float64\n",
      " 672   bureau_6                   95613 non-null  float64\n",
      " 673   bureau_7                   95613 non-null  float64\n",
      " 674   bureau_8                   95613 non-null  float64\n",
      " 675   bureau_9                   95613 non-null  float64\n",
      " 676   bureau_10                  95613 non-null  float64\n",
      " 677   bureau_11                  95613 non-null  float64\n",
      " 678   bureau_12                  95613 non-null  float64\n",
      " 679   bureau_13                  95613 non-null  float64\n",
      " 680   bureau_14                  95613 non-null  float64\n",
      " 681   bureau_15                  95613 non-null  float64\n",
      " 682   bureau_16                  95613 non-null  float64\n",
      " 683   bureau_17                  95613 non-null  float64\n",
      " 684   bureau_18                  95613 non-null  float64\n",
      " 685   bureau_19                  95613 non-null  float64\n",
      " 686   bureau_20                  95613 non-null  float64\n",
      " 687   bureau_21                  95613 non-null  float64\n",
      " 688   bureau_22                  95613 non-null  float64\n",
      " 689   bureau_23                  95613 non-null  float64\n",
      " 690   bureau_24                  95613 non-null  float64\n",
      " 691   bureau_25                  95613 non-null  float64\n",
      " 692   bureau_26                  95613 non-null  float64\n",
      " 693   bureau_27                  95613 non-null  float64\n",
      " 694   bureau_28                  95613 non-null  float64\n",
      " 695   bureau_29                  95613 non-null  float64\n",
      " 696   bureau_30                  95613 non-null  float64\n",
      " 697   bureau_31                  95613 non-null  float64\n",
      " 698   bureau_32                  95613 non-null  float64\n",
      " 699   bureau_33                  95613 non-null  float64\n",
      " 700   bureau_34                  95613 non-null  float64\n",
      " 701   bureau_35                  95613 non-null  float64\n",
      " 702   bureau_36                  95612 non-null  float64\n",
      " 703   bureau_37                  95612 non-null  float64\n",
      " 704   bureau_38                  95613 non-null  float64\n",
      " 705   bureau_39                  95613 non-null  float64\n",
      " 706   bureau_40                  95612 non-null  float64\n",
      " 707   bureau_41                  95604 non-null  float64\n",
      " 708   bureau_42                  95579 non-null  float64\n",
      " 709   bureau_43                  95570 non-null  float64\n",
      " 710   bureau_44                  95613 non-null  float64\n",
      " 711   bureau_45                  95613 non-null  float64\n",
      " 712   bureau_46                  95613 non-null  float64\n",
      " 713   bureau_47                  95613 non-null  float64\n",
      " 714   bureau_48                  95613 non-null  float64\n",
      " 715   bureau_49                  95613 non-null  float64\n",
      " 716   bureau_50                  95612 non-null  float64\n",
      " 717   bureau_51                  95597 non-null  float64\n",
      " 718   bureau_52                  95596 non-null  float64\n",
      " 719   bureau_53                  95613 non-null  float64\n",
      " 720   bureau_54                  95613 non-null  float64\n",
      " 721   bureau_55                  95613 non-null  float64\n",
      " 722   bureau_56                  95613 non-null  float64\n",
      " 723   bureau_57                  95613 non-null  float64\n",
      " 724   bureau_58                  95613 non-null  float64\n",
      " 725   bureau_59                  95613 non-null  float64\n",
      " 726   bureau_60                  95613 non-null  float64\n",
      " 727   bureau_61                  95613 non-null  float64\n",
      " 728   bureau_62                  95613 non-null  float64\n",
      " 729   bureau_63                  95613 non-null  float64\n",
      " 730   bureau_64                  95613 non-null  float64\n",
      " 731   bureau_65                  95613 non-null  float64\n",
      " 732   bureau_66                  95613 non-null  float64\n",
      " 733   bureau_67                  95613 non-null  float64\n",
      " 734   bureau_68                  95613 non-null  float64\n",
      " 735   bureau_69                  95613 non-null  float64\n",
      " 736   bureau_70                  95613 non-null  float64\n",
      " 737   bureau_71                  95613 non-null  float64\n",
      " 738   bureau_72                  95613 non-null  float64\n",
      " 739   bureau_73                  95613 non-null  float64\n",
      " 740   bureau_74                  95613 non-null  float64\n",
      " 741   bureau_75                  95613 non-null  float64\n",
      " 742   bureau_76                  95613 non-null  float64\n",
      " 743   bureau_77                  95613 non-null  float64\n",
      " 744   bureau_78                  95613 non-null  float64\n",
      " 745   bureau_79                  95613 non-null  float64\n",
      " 746   bureau_80                  95613 non-null  float64\n",
      " 747   bureau_81                  95613 non-null  float64\n",
      " 748   bureau_82                  95613 non-null  float64\n",
      " 749   bureau_83                  95613 non-null  float64\n",
      " 750   bureau_84                  95613 non-null  float64\n",
      " 751   bureau_85                  95613 non-null  float64\n",
      " 752   bureau_86                  95613 non-null  float64\n",
      " 753   bureau_87                  95613 non-null  float64\n",
      " 754   bureau_88                  95613 non-null  float64\n",
      " 755   bureau_89                  95613 non-null  float64\n",
      " 756   bureau_90                  95613 non-null  float64\n",
      " 757   bureau_91                  95613 non-null  float64\n",
      " 758   bureau_92                  95613 non-null  float64\n",
      " 759   bureau_93                  95613 non-null  float64\n",
      " 760   bureau_94                  95613 non-null  float64\n",
      " 761   bureau_95                  95613 non-null  float64\n",
      " 762   bureau_96                  95613 non-null  float64\n",
      " 763   bureau_97                  95613 non-null  float64\n",
      " 764   bureau_98                  95613 non-null  float64\n",
      " 765   bureau_99                  95613 non-null  float64\n",
      " 766   bureau_100                 95613 non-null  float64\n",
      " 767   bureau_101                 95613 non-null  float64\n",
      " 768   bureau_102                 95613 non-null  float64\n",
      " 769   bureau_103                 95613 non-null  float64\n",
      " 770   bureau_104                 95613 non-null  float64\n",
      " 771   bureau_105                 95613 non-null  float64\n",
      " 772   bureau_106                 95613 non-null  float64\n",
      " 773   bureau_107                 95613 non-null  float64\n",
      " 774   bureau_108                 95613 non-null  float64\n",
      " 775   bureau_109                 95613 non-null  float64\n",
      " 776   bureau_110                 95613 non-null  float64\n",
      " 777   bureau_111                 95613 non-null  float64\n",
      " 778   bureau_112                 95613 non-null  float64\n",
      " 779   bureau_113                 95613 non-null  float64\n",
      " 780   bureau_114                 95613 non-null  float64\n",
      " 781   bureau_115                 95613 non-null  float64\n",
      " 782   bureau_116                 95613 non-null  float64\n",
      " 783   bureau_117                 95613 non-null  float64\n",
      " 784   bureau_118                 95613 non-null  float64\n",
      " 785   bureau_119                 95613 non-null  float64\n",
      " 786   bureau_120                 95613 non-null  float64\n",
      " 787   bureau_121                 95613 non-null  float64\n",
      " 788   bureau_122                 95613 non-null  float64\n",
      " 789   bureau_123                 95613 non-null  float64\n",
      " 790   bureau_124                 95613 non-null  float64\n",
      " 791   bureau_125                 95613 non-null  float64\n",
      " 792   bureau_126                 95613 non-null  float64\n",
      " 793   bureau_127                 95613 non-null  float64\n",
      " 794   bureau_128                 95613 non-null  float64\n",
      " 795   bureau_129                 95613 non-null  float64\n",
      " 796   bureau_130                 95613 non-null  float64\n",
      " 797   bureau_131                 95613 non-null  float64\n",
      " 798   bureau_132                 95613 non-null  float64\n",
      " 799   bureau_133                 95613 non-null  float64\n",
      " 800   bureau_134                 95613 non-null  float64\n",
      " 801   bureau_135                 95613 non-null  float64\n",
      " 802   bureau_136                 95613 non-null  float64\n",
      " 803   bureau_137                 95613 non-null  float64\n",
      " 804   bureau_138                 95613 non-null  float64\n",
      " 805   bureau_139                 95489 non-null  float64\n",
      " 806   bureau_140                 95374 non-null  float64\n",
      " 807   bureau_141                 95252 non-null  float64\n",
      " 808   bureau_142                 95613 non-null  float64\n",
      " 809   bureau_143                 95456 non-null  float64\n",
      " 810   bureau_144                 95532 non-null  float64\n",
      " 811   bureau_145                 88271 non-null  float64\n",
      " 812   bureau_146                 94384 non-null  float64\n",
      " 813   bureau_147                 65378 non-null  float64\n",
      " 814   bureau_148                 6242 non-null   float64\n",
      " 815   bureau_149                 95613 non-null  float64\n",
      " 816   bureau_150                 95613 non-null  float64\n",
      " 817   bureau_151                 95613 non-null  float64\n",
      " 818   bureau_152                 95613 non-null  float64\n",
      " 819   bureau_153                 95613 non-null  float64\n",
      " 820   bureau_154                 95613 non-null  float64\n",
      " 821   bureau_155                 95613 non-null  float64\n",
      " 822   bureau_156                 95613 non-null  float64\n",
      " 823   bureau_157                 95613 non-null  float64\n",
      " 824   bureau_158                 95613 non-null  float64\n",
      " 825   bureau_159                 95613 non-null  float64\n",
      " 826   bureau_160                 95613 non-null  float64\n",
      " 827   bureau_161                 95613 non-null  float64\n",
      " 828   bureau_162                 95613 non-null  float64\n",
      " 829   bureau_163                 95613 non-null  float64\n",
      " 830   bureau_164                 95613 non-null  float64\n",
      " 831   bureau_165                 95613 non-null  float64\n",
      " 832   bureau_166                 95613 non-null  float64\n",
      " 833   bureau_167                 95613 non-null  float64\n",
      " 834   bureau_168                 95613 non-null  float64\n",
      " 835   bureau_169                 95613 non-null  float64\n",
      " 836   bureau_170                 95613 non-null  float64\n",
      " 837   bureau_171                 95613 non-null  float64\n",
      " 838   bureau_172                 95613 non-null  float64\n",
      " 839   bureau_173                 95613 non-null  float64\n",
      " 840   bureau_174                 95613 non-null  float64\n",
      " 841   bureau_175                 95613 non-null  float64\n",
      " 842   bureau_176                 95613 non-null  float64\n",
      " 843   bureau_177                 95613 non-null  float64\n",
      " 844   bureau_178                 95613 non-null  float64\n",
      " 845   bureau_179                 95613 non-null  float64\n",
      " 846   bureau_180                 95613 non-null  float64\n",
      " 847   bureau_181                 95613 non-null  float64\n",
      " 848   bureau_182                 95613 non-null  float64\n",
      " 849   bureau_183                 95613 non-null  float64\n",
      " 850   bureau_184                 95613 non-null  float64\n",
      " 851   bureau_185                 95613 non-null  float64\n",
      " 852   bureau_186                 95613 non-null  float64\n",
      " 853   bureau_187                 95613 non-null  float64\n",
      " 854   bureau_188                 95613 non-null  float64\n",
      " 855   bureau_189                 95613 non-null  float64\n",
      " 856   bureau_190                 95613 non-null  float64\n",
      " 857   bureau_191                 95613 non-null  float64\n",
      " 858   bureau_192                 95613 non-null  float64\n",
      " 859   bureau_193                 95613 non-null  float64\n",
      " 860   bureau_194                 95613 non-null  float64\n",
      " 861   bureau_195                 95613 non-null  float64\n",
      " 862   bureau_196                 95613 non-null  float64\n",
      " 863   bureau_197                 95613 non-null  float64\n",
      " 864   bureau_198                 95613 non-null  float64\n",
      " 865   bureau_199                 95613 non-null  float64\n",
      " 866   bureau_200                 95613 non-null  float64\n",
      " 867   bureau_201                 95613 non-null  float64\n",
      " 868   bureau_202                 95613 non-null  float64\n",
      " 869   bureau_203                 95613 non-null  float64\n",
      " 870   bureau_204                 95613 non-null  float64\n",
      " 871   bureau_205                 95613 non-null  float64\n",
      " 872   bureau_206                 95613 non-null  float64\n",
      " 873   bureau_207                 95613 non-null  float64\n",
      " 874   bureau_208                 95613 non-null  float64\n",
      " 875   bureau_209                 95613 non-null  float64\n",
      " 876   bureau_210                 95613 non-null  float64\n",
      " 877   bureau_211                 95613 non-null  float64\n",
      " 878   bureau_212                 95613 non-null  float64\n",
      " 879   bureau_213                 95613 non-null  float64\n",
      " 880   bureau_214                 95613 non-null  float64\n",
      " 881   bureau_215                 95613 non-null  float64\n",
      " 882   bureau_216                 95613 non-null  float64\n",
      " 883   bureau_217                 95613 non-null  float64\n",
      " 884   bureau_218                 95613 non-null  float64\n",
      " 885   bureau_219                 95613 non-null  float64\n",
      " 886   bureau_220                 95613 non-null  float64\n",
      " 887   bureau_221                 95613 non-null  float64\n",
      " 888   bureau_222                 95613 non-null  float64\n",
      " 889   bureau_223                 95613 non-null  float64\n",
      " 890   bureau_224                 95613 non-null  float64\n",
      " 891   bureau_225                 95613 non-null  float64\n",
      " 892   bureau_226                 95613 non-null  float64\n",
      " 893   bureau_227                 95613 non-null  float64\n",
      " 894   bureau_228                 95613 non-null  float64\n",
      " 895   bureau_229                 95613 non-null  float64\n",
      " 896   bureau_230                 95613 non-null  float64\n",
      " 897   bureau_231                 95613 non-null  float64\n",
      " 898   bureau_232                 95613 non-null  float64\n",
      " 899   bureau_233                 95613 non-null  float64\n",
      " 900   bureau_234                 95613 non-null  float64\n",
      " 901   bureau_235                 95613 non-null  float64\n",
      " 902   bureau_236                 95613 non-null  float64\n",
      " 903   bureau_237                 95613 non-null  float64\n",
      " 904   bureau_238                 95613 non-null  float64\n",
      " 905   bureau_239                 95613 non-null  float64\n",
      " 906   bureau_240                 95613 non-null  float64\n",
      " 907   bureau_241                 95613 non-null  float64\n",
      " 908   bureau_242                 95613 non-null  float64\n",
      " 909   bureau_243                 95613 non-null  float64\n",
      " 910   bureau_244                 95613 non-null  float64\n",
      " 911   bureau_245                 95613 non-null  float64\n",
      " 912   bureau_246                 95613 non-null  float64\n",
      " 913   bureau_247                 95613 non-null  float64\n",
      " 914   bureau_248                 95613 non-null  float64\n",
      " 915   bureau_249                 95613 non-null  float64\n",
      " 916   bureau_250                 95613 non-null  float64\n",
      " 917   bureau_251                 95613 non-null  float64\n",
      " 918   bureau_252                 95613 non-null  float64\n",
      " 919   bureau_253                 95613 non-null  float64\n",
      " 920   bureau_254                 95613 non-null  float64\n",
      " 921   bureau_255                 95613 non-null  float64\n",
      " 922   bureau_256                 95613 non-null  float64\n",
      " 923   bureau_257                 95613 non-null  float64\n",
      " 924   bureau_258                 95613 non-null  float64\n",
      " 925   bureau_259                 95613 non-null  float64\n",
      " 926   bureau_260                 95613 non-null  float64\n",
      " 927   bureau_261                 95613 non-null  float64\n",
      " 928   bureau_262                 95613 non-null  float64\n",
      " 929   bureau_263                 95613 non-null  float64\n",
      " 930   bureau_264                 95613 non-null  float64\n",
      " 931   bureau_265                 95613 non-null  float64\n",
      " 932   bureau_266                 95613 non-null  float64\n",
      " 933   bureau_267                 95613 non-null  float64\n",
      " 934   bureau_268                 95613 non-null  float64\n",
      " 935   bureau_269                 95613 non-null  float64\n",
      " 936   bureau_270                 95613 non-null  float64\n",
      " 937   bureau_271                 95613 non-null  float64\n",
      " 938   bureau_272                 95613 non-null  float64\n",
      " 939   bureau_273                 95613 non-null  float64\n",
      " 940   bureau_274                 95613 non-null  float64\n",
      " 941   bureau_275                 95613 non-null  float64\n",
      " 942   bureau_276                 95613 non-null  float64\n",
      " 943   bureau_277                 95613 non-null  float64\n",
      " 944   bureau_278                 95613 non-null  float64\n",
      " 945   bureau_279                 95613 non-null  float64\n",
      " 946   bureau_280                 95613 non-null  float64\n",
      " 947   bureau_281                 95613 non-null  float64\n",
      " 948   bureau_282                 95613 non-null  float64\n",
      " 949   bureau_283                 95613 non-null  float64\n",
      " 950   bureau_284                 95613 non-null  float64\n",
      " 951   bureau_285                 95613 non-null  float64\n",
      " 952   bureau_286                 95613 non-null  float64\n",
      " 953   bureau_287                 95613 non-null  float64\n",
      " 954   bureau_288                 95613 non-null  float64\n",
      " 955   bureau_289                 95613 non-null  float64\n",
      " 956   bureau_290                 95613 non-null  float64\n",
      " 957   bureau_291                 95613 non-null  float64\n",
      " 958   bureau_292                 95613 non-null  float64\n",
      " 959   bureau_293                 95613 non-null  float64\n",
      " 960   bureau_294                 95613 non-null  float64\n",
      " 961   bureau_295                 95613 non-null  float64\n",
      " 962   bureau_296                 95613 non-null  float64\n",
      " 963   bureau_297                 95613 non-null  float64\n",
      " 964   bureau_298                 95613 non-null  float64\n",
      " 965   bureau_299                 95613 non-null  float64\n",
      " 966   bureau_300                 95613 non-null  float64\n",
      " 967   bureau_301                 95613 non-null  float64\n",
      " 968   bureau_302                 95613 non-null  float64\n",
      " 969   bureau_303                 95613 non-null  float64\n",
      " 970   bureau_304                 95613 non-null  float64\n",
      " 971   bureau_305                 95613 non-null  float64\n",
      " 972   bureau_306                 95613 non-null  float64\n",
      " 973   bureau_307                 95613 non-null  float64\n",
      " 974   bureau_308                 95613 non-null  float64\n",
      " 975   bureau_309                 95613 non-null  float64\n",
      " 976   bureau_310                 95613 non-null  float64\n",
      " 977   bureau_311                 95613 non-null  float64\n",
      " 978   bureau_312                 95613 non-null  float64\n",
      " 979   bureau_313                 95613 non-null  float64\n",
      " 980   bureau_314                 95613 non-null  float64\n",
      " 981   bureau_315                 95613 non-null  float64\n",
      " 982   bureau_316                 95613 non-null  float64\n",
      " 983   bureau_317                 95613 non-null  float64\n",
      " 984   bureau_318                 95613 non-null  float64\n",
      " 985   bureau_319                 95613 non-null  float64\n",
      " 986   bureau_320                 95613 non-null  float64\n",
      " 987   bureau_321                 95613 non-null  float64\n",
      " 988   bureau_322                 95613 non-null  float64\n",
      " 989   bureau_323                 95613 non-null  float64\n",
      " 990   bureau_324                 95613 non-null  float64\n",
      " 991   bureau_325                 95613 non-null  float64\n",
      " 992   bureau_326                 95613 non-null  float64\n",
      " 993   bureau_327                 95613 non-null  float64\n",
      " 994   bureau_328                 95613 non-null  float64\n",
      " 995   bureau_329                 95613 non-null  float64\n",
      " 996   bureau_330                 95613 non-null  float64\n",
      " 997   bureau_331                 95613 non-null  float64\n",
      " 998   bureau_332                 95613 non-null  float64\n",
      " 999   bureau_333                 95613 non-null  float64\n",
      " 1000  bureau_334                 95613 non-null  float64\n",
      " 1001  bureau_335                 95613 non-null  float64\n",
      " 1002  bureau_336                 95613 non-null  float64\n",
      " 1003  bureau_337                 95613 non-null  float64\n",
      " 1004  bureau_338                 95613 non-null  float64\n",
      " 1005  bureau_339                 95613 non-null  float64\n",
      " 1006  bureau_340                 95613 non-null  float64\n",
      " 1007  bureau_341                 95613 non-null  float64\n",
      " 1008  bureau_342                 95613 non-null  float64\n",
      " 1009  bureau_343                 95613 non-null  float64\n",
      " 1010  bureau_344                 95613 non-null  float64\n",
      " 1011  bureau_345                 95613 non-null  float64\n",
      " 1012  bureau_346                 95613 non-null  float64\n",
      " 1013  bureau_347                 95613 non-null  float64\n",
      " 1014  bureau_348                 95613 non-null  float64\n",
      " 1015  bureau_349                 95613 non-null  float64\n",
      " 1016  bureau_350                 95613 non-null  float64\n",
      " 1017  bureau_351                 95613 non-null  float64\n",
      " 1018  bureau_352                 95613 non-null  float64\n",
      " 1019  bureau_353                 95613 non-null  float64\n",
      " 1020  bureau_354                 95613 non-null  float64\n",
      " 1021  bureau_355                 95613 non-null  float64\n",
      " 1022  bureau_356                 95613 non-null  float64\n",
      " 1023  bureau_357                 95613 non-null  float64\n",
      " 1024  bureau_358                 95613 non-null  float64\n",
      " 1025  bureau_359                 95613 non-null  float64\n",
      " 1026  bureau_360                 95613 non-null  float64\n",
      " 1027  bureau_361                 95613 non-null  float64\n",
      " 1028  bureau_362                 95613 non-null  float64\n",
      " 1029  bureau_363                 95613 non-null  float64\n",
      " 1030  bureau_364                 95613 non-null  float64\n",
      " 1031  bureau_365                 95613 non-null  float64\n",
      " 1032  bureau_366                 95613 non-null  float64\n",
      " 1033  bureau_367                 95613 non-null  float64\n",
      " 1034  bureau_368                 95613 non-null  float64\n",
      " 1035  bureau_369                 95613 non-null  float64\n",
      " 1036  bureau_370                 95613 non-null  float64\n",
      " 1037  bureau_371                 95613 non-null  float64\n",
      " 1038  bureau_372                 95613 non-null  float64\n",
      " 1039  bureau_373                 95613 non-null  float64\n",
      " 1040  bureau_374                 95613 non-null  float64\n",
      " 1041  bureau_375                 95613 non-null  float64\n",
      " 1042  bureau_376                 95613 non-null  float64\n",
      " 1043  bureau_377                 95613 non-null  float64\n",
      " 1044  bureau_378                 95613 non-null  float64\n",
      " 1045  bureau_379                 95613 non-null  float64\n",
      " 1046  bureau_380                 95613 non-null  float64\n",
      " 1047  bureau_381                 95613 non-null  float64\n",
      " 1048  bureau_382                 95613 non-null  float64\n",
      " 1049  bureau_383                 95613 non-null  float64\n",
      " 1050  bureau_384                 95613 non-null  float64\n",
      " 1051  bureau_385                 95613 non-null  float64\n",
      " 1052  bureau_386                 95613 non-null  float64\n",
      " 1053  bureau_387                 95613 non-null  float64\n",
      " 1054  bureau_388                 95613 non-null  float64\n",
      " 1055  bureau_389                 95613 non-null  float64\n",
      " 1056  bureau_390                 95613 non-null  float64\n",
      " 1057  bureau_391                 95613 non-null  float64\n",
      " 1058  bureau_392                 95613 non-null  float64\n",
      " 1059  bureau_393                 95613 non-null  float64\n",
      " 1060  bureau_394                 95613 non-null  float64\n",
      " 1061  bureau_395                 95613 non-null  float64\n",
      " 1062  bureau_396                 95613 non-null  float64\n",
      " 1063  bureau_397                 95613 non-null  float64\n",
      " 1064  bureau_398                 95613 non-null  float64\n",
      " 1065  bureau_399                 95613 non-null  float64\n",
      " 1066  bureau_400                 95613 non-null  float64\n",
      " 1067  bureau_401                 95613 non-null  float64\n",
      " 1068  bureau_402                 95613 non-null  float64\n",
      " 1069  bureau_403                 95613 non-null  float64\n",
      " 1070  bureau_404                 95613 non-null  float64\n",
      " 1071  bureau_405                 95613 non-null  float64\n",
      " 1072  bureau_406                 95613 non-null  float64\n",
      " 1073  bureau_407                 95613 non-null  float64\n",
      " 1074  bureau_408                 95613 non-null  float64\n",
      " 1075  bureau_409                 95613 non-null  float64\n",
      " 1076  bureau_410                 95613 non-null  float64\n",
      " 1077  bureau_411                 95613 non-null  float64\n",
      " 1078  bureau_412                 95613 non-null  float64\n",
      " 1079  bureau_413                 95613 non-null  float64\n",
      " 1080  bureau_414                 95613 non-null  float64\n",
      " 1081  bureau_415                 95613 non-null  float64\n",
      " 1082  bureau_416                 95613 non-null  float64\n",
      " 1083  bureau_417                 95613 non-null  float64\n",
      " 1084  bureau_418                 95613 non-null  float64\n",
      " 1085  bureau_419                 95613 non-null  float64\n",
      " 1086  bureau_420                 95613 non-null  float64\n",
      " 1087  bureau_421                 95613 non-null  float64\n",
      " 1088  bureau_422                 95613 non-null  float64\n",
      " 1089  bureau_423                 95613 non-null  float64\n",
      " 1090  bureau_424                 95613 non-null  float64\n",
      " 1091  bureau_425                 95613 non-null  float64\n",
      " 1092  bureau_426                 95613 non-null  float64\n",
      " 1093  bureau_427                 95613 non-null  float64\n",
      " 1094  bureau_428                 95613 non-null  float64\n",
      " 1095  bureau_429                 95613 non-null  float64\n",
      " 1096  bureau_430                 95613 non-null  float64\n",
      " 1097  bureau_431                 95411 non-null  float64\n",
      " 1098  bureau_432                 94133 non-null  float64\n",
      " 1099  bureau_433                 23025 non-null  float64\n",
      " 1100  bureau_434                 55918 non-null  float64\n",
      " 1101  bureau_435                 22315 non-null  float64\n",
      " 1102  bureau_436                 0 non-null      float64\n",
      " 1103  bureau_437                 31499 non-null  float64\n",
      " 1104  bureau_438                 13999 non-null  float64\n",
      " 1105  bureau_439                 89838 non-null  float64\n",
      " 1106  bureau_440                 63932 non-null  float64\n",
      " 1107  bureau_441                 94062 non-null  float64\n",
      " 1108  bureau_442                 93419 non-null  float64\n",
      " 1109  bureau_443                 90884 non-null  float64\n",
      " 1110  bureau_444                 11866 non-null  float64\n",
      " 1111  bureau_445                 44832 non-null  float64\n",
      " 1112  bureau_446                 16092 non-null  float64\n",
      " 1113  bureau_447                 0 non-null      float64\n",
      " 1114  bureau_448                 9653 non-null   float64\n",
      " 1115  bureau_449                 5688 non-null   float64\n",
      " 1116  bureau_450                 88060 non-null  float64\n",
      " 1117  bureau_451                 41425 non-null  float64\n",
      " 1118  bureau_452                 91833 non-null  float64\n",
      " 1119  onus_attribute_2           96806 non-null  float64\n",
      " 1120  onus_attribute_3           96806 non-null  int64  \n",
      " 1121  onus_attribute_4           96806 non-null  int64  \n",
      " 1122  onus_attribute_5           96804 non-null  float64\n",
      " 1123  onus_attribute_6           61143 non-null  float64\n",
      " 1124  onus_attribute_7           51852 non-null  float64\n",
      " 1125  onus_attribute_8           68568 non-null  float64\n",
      " 1126  onus_attribute_9           66375 non-null  float64\n",
      " 1127  onus_attribute_10          96804 non-null  float64\n",
      " 1128  onus_attribute_11          69434 non-null  float64\n",
      " 1129  onus_attribute_12          68022 non-null  float64\n",
      " 1130  onus_attribute_13          96804 non-null  float64\n",
      " 1131  onus_attribute_14          69438 non-null  float64\n",
      " 1132  onus_attribute_15          68031 non-null  float64\n",
      " 1133  onus_attribute_16          96804 non-null  float64\n",
      " 1134  onus_attribute_17          96806 non-null  float64\n",
      " 1135  onus_attribute_18          96806 non-null  int64  \n",
      " 1136  onus_attribute_19          96806 non-null  int64  \n",
      " 1137  onus_attribute_20          96806 non-null  float64\n",
      " 1138  onus_attribute_21          96806 non-null  int64  \n",
      " 1139  onus_attribute_22          96806 non-null  int64  \n",
      " 1140  onus_attribute_23          96806 non-null  float64\n",
      " 1141  onus_attribute_24          96806 non-null  int64  \n",
      " 1142  onus_attribute_25          96806 non-null  int64  \n",
      " 1143  onus_attribute_26          96806 non-null  int64  \n",
      " 1144  onus_attribute_27          96806 non-null  int64  \n",
      " 1145  onus_attribute_28          96806 non-null  int64  \n",
      " 1146  onus_attribute_29          96806 non-null  int64  \n",
      " 1147  onus_attribute_30          96806 non-null  int64  \n",
      " 1148  onus_attribute_31          96806 non-null  int64  \n",
      " 1149  onus_attribute_32          96806 non-null  int64  \n",
      " 1150  onus_attribute_33          96806 non-null  int64  \n",
      " 1151  onus_attribute_34          96806 non-null  int64  \n",
      " 1152  onus_attribute_35          96806 non-null  int64  \n",
      " 1153  onus_attribute_36          96806 non-null  int64  \n",
      " 1154  onus_attribute_37          96806 non-null  int64  \n",
      " 1155  onus_attribute_38          96806 non-null  int64  \n",
      " 1156  onus_attribute_39          96806 non-null  int64  \n",
      " 1157  onus_attribute_40          96806 non-null  int64  \n",
      " 1158  onus_attribute_41          96806 non-null  int64  \n",
      " 1159  onus_attribute_42          96806 non-null  int64  \n",
      " 1160  bureau_enquiry_1           94212 non-null  float64\n",
      " 1161  bureau_enquiry_2           94212 non-null  float64\n",
      " 1162  bureau_enquiry_3           94212 non-null  float64\n",
      " 1163  bureau_enquiry_4           94212 non-null  float64\n",
      " 1164  bureau_enquiry_5           94212 non-null  float64\n",
      " 1165  bureau_enquiry_6           94212 non-null  float64\n",
      " 1166  bureau_enquiry_7           94212 non-null  float64\n",
      " 1167  bureau_enquiry_8           94212 non-null  float64\n",
      " 1168  bureau_enquiry_9           94212 non-null  float64\n",
      " 1169  bureau_enquiry_10          94212 non-null  float64\n",
      " 1170  bureau_enquiry_11          94212 non-null  float64\n",
      " 1171  bureau_enquiry_12          94212 non-null  float64\n",
      " 1172  bureau_enquiry_13          94212 non-null  float64\n",
      " 1173  bureau_enquiry_14          94212 non-null  float64\n",
      " 1174  bureau_enquiry_15          94212 non-null  float64\n",
      " 1175  bureau_enquiry_16          94212 non-null  float64\n",
      " 1176  bureau_enquiry_17          94212 non-null  float64\n",
      " 1177  bureau_enquiry_18          94212 non-null  float64\n",
      " 1178  bureau_enquiry_19          94212 non-null  float64\n",
      " 1179  bureau_enquiry_20          94212 non-null  float64\n",
      " 1180  bureau_enquiry_21          94212 non-null  float64\n",
      " 1181  bureau_enquiry_22          94212 non-null  float64\n",
      " 1182  bureau_enquiry_23          94212 non-null  float64\n",
      " 1183  bureau_enquiry_24          94212 non-null  float64\n",
      " 1184  bureau_enquiry_25          94212 non-null  float64\n",
      " 1185  bureau_enquiry_26          94212 non-null  float64\n",
      " 1186  bureau_enquiry_27          94212 non-null  float64\n",
      " 1187  bureau_enquiry_28          94212 non-null  float64\n",
      " 1188  bureau_enquiry_29          94212 non-null  float64\n",
      " 1189  bureau_enquiry_30          94212 non-null  float64\n",
      " 1190  bureau_enquiry_31          94212 non-null  float64\n",
      " 1191  bureau_enquiry_32          94212 non-null  float64\n",
      " 1192  bureau_enquiry_33          94212 non-null  float64\n",
      " 1193  bureau_enquiry_34          94212 non-null  float64\n",
      " 1194  bureau_enquiry_35          94212 non-null  float64\n",
      " 1195  bureau_enquiry_36          94212 non-null  float64\n",
      " 1196  bureau_enquiry_37          94212 non-null  float64\n",
      " 1197  bureau_enquiry_38          94212 non-null  float64\n",
      " 1198  bureau_enquiry_39          94212 non-null  float64\n",
      " 1199  bureau_enquiry_40          94212 non-null  float64\n",
      " 1200  bureau_enquiry_41          94212 non-null  float64\n",
      " 1201  bureau_enquiry_42          94212 non-null  float64\n",
      " 1202  bureau_enquiry_43          94212 non-null  float64\n",
      " 1203  bureau_enquiry_44          94212 non-null  float64\n",
      " 1204  bureau_enquiry_45          94212 non-null  float64\n",
      " 1205  bureau_enquiry_46          94212 non-null  float64\n",
      " 1206  bureau_enquiry_47          94212 non-null  float64\n",
      " 1207  bureau_enquiry_48          94212 non-null  float64\n",
      " 1208  bureau_enquiry_49          94212 non-null  float64\n",
      " 1209  bureau_enquiry_50          94212 non-null  float64\n",
      " 1210  onus_attribute_43          11610 non-null  float64\n",
      " 1211  onus_attribute_44          11610 non-null  float64\n",
      " 1212  onus_attribute_45          11610 non-null  float64\n",
      " 1213  onus_attribute_46          11610 non-null  float64\n",
      " 1214  onus_attribute_47          11610 non-null  float64\n",
      " 1215  onus_attribute_48          11610 non-null  float64\n",
      "dtypes: float64(1189), int64(27)\n",
      "memory usage: 898.1 MB\n"
     ]
    }
   ],
   "source": [
    "dev_dataset.info(memory_usage=\"deep\", max_cols=1216)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "f8c020b9-094f-427c-8481-e749253c8d5e",
   "metadata": {},
   "outputs": [],
   "source": [
    "onus_attributes = [col for col in dev_dataset.columns if col.startswith(\"onus\")]\n",
    "transaction_attributes = [col for col in dev_dataset.columns if col.startswith(\"transaction\")]\n",
    "bureau_attributes = [col for col in dev_dataset.columns if col.startswith(\"bureau\")]\n",
    "bureau_enquiry_attributes = [col for col in dev_dataset.columns if col.startswith(\"bureau_enquiry\")]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "3ed94329-faf5-4650-af23-0160d7924e2c",
   "metadata": {},
   "outputs": [],
   "source": [
    "# ## Transaction analysis\n",
    "# %%\n",
    "tran_dataset = dev_dataset.iloc[:, 1:665].copy()  # Ensuring a copy to avoid SettingWithCopyWarning\n",
    "tran_dataset[\"bad_flag\"] = dev_dataset[\"bad_flag\"]\n",
    "\n",
    "#tran_dataset=dev_dataset.iloc[ : ,1:665]\n",
    "#tran_dataset[\"bad_flag\"]=pd.DataFrame(dev_dataset[\"bad_flag\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "7a1249d5-3162-4036-91f8-712e0478ce20",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bad_flag</th>\n",
       "      <th>onus_attribute_1</th>\n",
       "      <th>transaction_attribute_1</th>\n",
       "      <th>transaction_attribute_2</th>\n",
       "      <th>transaction_attribute_3</th>\n",
       "      <th>transaction_attribute_4</th>\n",
       "      <th>transaction_attribute_5</th>\n",
       "      <th>transaction_attribute_6</th>\n",
       "      <th>transaction_attribute_7</th>\n",
       "      <th>transaction_attribute_8</th>\n",
       "      <th>...</th>\n",
       "      <th>transaction_attribute_653</th>\n",
       "      <th>transaction_attribute_654</th>\n",
       "      <th>transaction_attribute_655</th>\n",
       "      <th>transaction_attribute_656</th>\n",
       "      <th>transaction_attribute_657</th>\n",
       "      <th>transaction_attribute_658</th>\n",
       "      <th>transaction_attribute_659</th>\n",
       "      <th>transaction_attribute_660</th>\n",
       "      <th>transaction_attribute_661</th>\n",
       "      <th>transaction_attribute_662</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>count</th>\n",
       "      <td>96806.000000</td>\n",
       "      <td>7.157500e+04</td>\n",
       "      <td>71575.000000</td>\n",
       "      <td>71575.000000</td>\n",
       "      <td>71575.000000</td>\n",
       "      <td>71575.000000</td>\n",
       "      <td>71575.000000</td>\n",
       "      <td>71575.000000</td>\n",
       "      <td>71575.000000</td>\n",
       "      <td>71575.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>71575.000000</td>\n",
       "      <td>71575.000000</td>\n",
       "      <td>71575.000000</td>\n",
       "      <td>71575.000000</td>\n",
       "      <td>71575.000000</td>\n",
       "      <td>71575.000000</td>\n",
       "      <td>71575.000000</td>\n",
       "      <td>71575.000000</td>\n",
       "      <td>71575.000000</td>\n",
       "      <td>71575.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>mean</th>\n",
       "      <td>0.014173</td>\n",
       "      <td>1.542391e+05</td>\n",
       "      <td>9.570769</td>\n",
       "      <td>0.002207</td>\n",
       "      <td>4.092854</td>\n",
       "      <td>77.306435</td>\n",
       "      <td>0.006315</td>\n",
       "      <td>67.043550</td>\n",
       "      <td>0.081034</td>\n",
       "      <td>0.000210</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000790</td>\n",
       "      <td>0.004160</td>\n",
       "      <td>0.016778</td>\n",
       "      <td>0.004551</td>\n",
       "      <td>0.031221</td>\n",
       "      <td>0.030402</td>\n",
       "      <td>0.271082</td>\n",
       "      <td>0.272400</td>\n",
       "      <td>0.146834</td>\n",
       "      <td>0.000604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>std</th>\n",
       "      <td>0.118203</td>\n",
       "      <td>1.729925e+05</td>\n",
       "      <td>1513.967595</td>\n",
       "      <td>0.119240</td>\n",
       "      <td>301.580599</td>\n",
       "      <td>3164.987013</td>\n",
       "      <td>0.129545</td>\n",
       "      <td>2516.330899</td>\n",
       "      <td>14.203615</td>\n",
       "      <td>0.034053</td>\n",
       "      <td>...</td>\n",
       "      <td>0.009330</td>\n",
       "      <td>0.028060</td>\n",
       "      <td>0.094083</td>\n",
       "      <td>0.120712</td>\n",
       "      <td>0.458011</td>\n",
       "      <td>0.140984</td>\n",
       "      <td>0.734534</td>\n",
       "      <td>0.734516</td>\n",
       "      <td>0.598994</td>\n",
       "      <td>0.016403</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>min</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>2.500000e+04</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-109800.476600</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>-3498.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>-0.124333</td>\n",
       "      <td>-0.007479</td>\n",
       "      <td>-0.132793</td>\n",
       "      <td>-0.361607</td>\n",
       "      <td>-0.200000</td>\n",
       "      <td>-0.001115</td>\n",
       "      <td>-0.351142</td>\n",
       "      <td>-0.351142</td>\n",
       "      <td>-0.500000</td>\n",
       "      <td>-0.114195</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>25%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>5.900000e+04</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.041325</td>\n",
       "      <td>0.042630</td>\n",
       "      <td>0.001976</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>50%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000e+05</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.149862</td>\n",
       "      <td>0.151380</td>\n",
       "      <td>0.033403</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>75%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.810000e+05</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001007</td>\n",
       "      <td>0.002222</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.006824</td>\n",
       "      <td>0.000481</td>\n",
       "      <td>0.390488</td>\n",
       "      <td>0.392351</td>\n",
       "      <td>0.175411</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>90%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>3.350000e+05</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.007174</td>\n",
       "      <td>0.022959</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.051491</td>\n",
       "      <td>0.076096</td>\n",
       "      <td>0.675989</td>\n",
       "      <td>0.676770</td>\n",
       "      <td>0.427556</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>95%</th>\n",
       "      <td>0.000000</td>\n",
       "      <td>4.500000e+05</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.015553</td>\n",
       "      <td>0.069950</td>\n",
       "      <td>0.004259</td>\n",
       "      <td>0.157926</td>\n",
       "      <td>0.201441</td>\n",
       "      <td>0.899377</td>\n",
       "      <td>0.899760</td>\n",
       "      <td>0.624406</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>max</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>2.800000e+06</td>\n",
       "      <td>398936.791000</td>\n",
       "      <td>25.000000</td>\n",
       "      <td>55000.000000</td>\n",
       "      <td>358986.000000</td>\n",
       "      <td>12.000000</td>\n",
       "      <td>358986.000000</td>\n",
       "      <td>3150.000000</td>\n",
       "      <td>8.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.936854</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>9.560813</td>\n",
       "      <td>18.896210</td>\n",
       "      <td>93.688219</td>\n",
       "      <td>15.667014</td>\n",
       "      <td>129.879402</td>\n",
       "      <td>129.879402</td>\n",
       "      <td>93.688219</td>\n",
       "      <td>2.313612</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 664 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "           bad_flag  onus_attribute_1  transaction_attribute_1  \\\n",
       "count  96806.000000      7.157500e+04             71575.000000   \n",
       "mean       0.014173      1.542391e+05                 9.570769   \n",
       "std        0.118203      1.729925e+05              1513.967595   \n",
       "min        0.000000      2.500000e+04                 0.000000   \n",
       "25%        0.000000      5.900000e+04                 0.000000   \n",
       "50%        0.000000      1.000000e+05                 0.000000   \n",
       "75%        0.000000      1.810000e+05                 0.000000   \n",
       "90%        0.000000      3.350000e+05                 0.000000   \n",
       "95%        0.000000      4.500000e+05                 0.000000   \n",
       "max        1.000000      2.800000e+06            398936.791000   \n",
       "\n",
       "       transaction_attribute_2  transaction_attribute_3  \\\n",
       "count             71575.000000             71575.000000   \n",
       "mean                  0.002207                 4.092854   \n",
       "std                   0.119240               301.580599   \n",
       "min                   0.000000                 0.000000   \n",
       "25%                   0.000000                 0.000000   \n",
       "50%                   0.000000                 0.000000   \n",
       "75%                   0.000000                 0.000000   \n",
       "90%                   0.000000                 0.000000   \n",
       "95%                   0.000000                 0.000000   \n",
       "max                  25.000000             55000.000000   \n",
       "\n",
       "       transaction_attribute_4  transaction_attribute_5  \\\n",
       "count             71575.000000             71575.000000   \n",
       "mean                 77.306435                 0.006315   \n",
       "std                3164.987013                 0.129545   \n",
       "min             -109800.476600                 0.000000   \n",
       "25%                   0.000000                 0.000000   \n",
       "50%                   0.000000                 0.000000   \n",
       "75%                   0.000000                 0.000000   \n",
       "90%                   0.000000                 0.000000   \n",
       "95%                   0.000000                 0.000000   \n",
       "max              358986.000000                12.000000   \n",
       "\n",
       "       transaction_attribute_6  transaction_attribute_7  \\\n",
       "count             71575.000000             71575.000000   \n",
       "mean                 67.043550                 0.081034   \n",
       "std                2516.330899                14.203615   \n",
       "min               -3498.000000                 0.000000   \n",
       "25%                   0.000000                 0.000000   \n",
       "50%                   0.000000                 0.000000   \n",
       "75%                   0.000000                 0.000000   \n",
       "90%                   0.000000                 0.000000   \n",
       "95%                   0.000000                 0.000000   \n",
       "max              358986.000000              3150.000000   \n",
       "\n",
       "       transaction_attribute_8  ...  transaction_attribute_653  \\\n",
       "count             71575.000000  ...               71575.000000   \n",
       "mean                  0.000210  ...                   0.000790   \n",
       "std                   0.034053  ...                   0.009330   \n",
       "min                   0.000000  ...                  -0.124333   \n",
       "25%                   0.000000  ...                   0.000000   \n",
       "50%                   0.000000  ...                   0.000000   \n",
       "75%                   0.000000  ...                   0.000000   \n",
       "90%                   0.000000  ...                   0.000000   \n",
       "95%                   0.000000  ...                   0.000000   \n",
       "max                   8.000000  ...                   0.936854   \n",
       "\n",
       "       transaction_attribute_654  transaction_attribute_655  \\\n",
       "count               71575.000000               71575.000000   \n",
       "mean                    0.004160                   0.016778   \n",
       "std                     0.028060                   0.094083   \n",
       "min                    -0.007479                  -0.132793   \n",
       "25%                     0.000000                   0.000000   \n",
       "50%                     0.000000                   0.000000   \n",
       "75%                     0.001007                   0.002222   \n",
       "90%                     0.007174                   0.022959   \n",
       "95%                     0.015553                   0.069950   \n",
       "max                     1.000000                   9.560813   \n",
       "\n",
       "       transaction_attribute_656  transaction_attribute_657  \\\n",
       "count               71575.000000               71575.000000   \n",
       "mean                    0.004551                   0.031221   \n",
       "std                     0.120712                   0.458011   \n",
       "min                    -0.361607                  -0.200000   \n",
       "25%                     0.000000                   0.000000   \n",
       "50%                     0.000000                   0.000000   \n",
       "75%                     0.000000                   0.006824   \n",
       "90%                     0.000000                   0.051491   \n",
       "95%                     0.004259                   0.157926   \n",
       "max                    18.896210                  93.688219   \n",
       "\n",
       "       transaction_attribute_658  transaction_attribute_659  \\\n",
       "count               71575.000000               71575.000000   \n",
       "mean                    0.030402                   0.271082   \n",
       "std                     0.140984                   0.734534   \n",
       "min                    -0.001115                  -0.351142   \n",
       "25%                     0.000000                   0.041325   \n",
       "50%                     0.000000                   0.149862   \n",
       "75%                     0.000481                   0.390488   \n",
       "90%                     0.076096                   0.675989   \n",
       "95%                     0.201441                   0.899377   \n",
       "max                    15.667014                 129.879402   \n",
       "\n",
       "       transaction_attribute_660  transaction_attribute_661  \\\n",
       "count               71575.000000               71575.000000   \n",
       "mean                    0.272400                   0.146834   \n",
       "std                     0.734516                   0.598994   \n",
       "min                    -0.351142                  -0.500000   \n",
       "25%                     0.042630                   0.001976   \n",
       "50%                     0.151380                   0.033403   \n",
       "75%                     0.392351                   0.175411   \n",
       "90%                     0.676770                   0.427556   \n",
       "95%                     0.899760                   0.624406   \n",
       "max                   129.879402                  93.688219   \n",
       "\n",
       "       transaction_attribute_662  \n",
       "count               71575.000000  \n",
       "mean                    0.000604  \n",
       "std                     0.016403  \n",
       "min                    -0.114195  \n",
       "25%                     0.000000  \n",
       "50%                     0.000000  \n",
       "75%                     0.000000  \n",
       "90%                     0.000000  \n",
       "95%                     0.000000  \n",
       "max                     2.313612  \n",
       "\n",
       "[10 rows x 664 columns]"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tran_dataset.describe(percentiles=[0.25, 0.5, 0.75, 0.90, 0.95])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "048f5380-0521-4e91-a903-50a055ed76a4",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bad_flag                         0\n",
       "onus_attribute_1             25231\n",
       "transaction_attribute_1      25231\n",
       "transaction_attribute_2      25231\n",
       "transaction_attribute_3      25231\n",
       "                             ...  \n",
       "transaction_attribute_658    25231\n",
       "transaction_attribute_659    25231\n",
       "transaction_attribute_660    25231\n",
       "transaction_attribute_661    25231\n",
       "transaction_attribute_662    25231\n",
       "Length: 664, dtype: int64"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tran_dataset.isnull().sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "id": "bc348dc9-55b6-4bba-bf11-d41d15e89c6e",
   "metadata": {},
   "outputs": [],
   "source": [
    "'''# Only fill NaN in columns that exist in transaction_attributes\n",
    "for column in transaction_attributes:\n",
    "    if column in tran_dataset.columns:\n",
    "    tran_dataset[column]= tran_dataset[column].fillna(tran_dataset[column].mean())'''\n",
    "\n",
    "#filling median\n",
    "''''for column in transaction_attributes:\n",
    "    tran_dataset[column].fillna(tran_dataset[column].median(), inplace=True)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "id": "b325cfba-87d0-41de-b2ea-93b2869d29f8",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>bad_flag</th>\n",
       "      <th>onus_attribute_1</th>\n",
       "      <th>transaction_attribute_1</th>\n",
       "      <th>transaction_attribute_2</th>\n",
       "      <th>transaction_attribute_3</th>\n",
       "      <th>transaction_attribute_4</th>\n",
       "      <th>transaction_attribute_5</th>\n",
       "      <th>transaction_attribute_6</th>\n",
       "      <th>transaction_attribute_7</th>\n",
       "      <th>transaction_attribute_8</th>\n",
       "      <th>...</th>\n",
       "      <th>transaction_attribute_653</th>\n",
       "      <th>transaction_attribute_654</th>\n",
       "      <th>transaction_attribute_655</th>\n",
       "      <th>transaction_attribute_656</th>\n",
       "      <th>transaction_attribute_657</th>\n",
       "      <th>transaction_attribute_658</th>\n",
       "      <th>transaction_attribute_659</th>\n",
       "      <th>transaction_attribute_660</th>\n",
       "      <th>transaction_attribute_661</th>\n",
       "      <th>transaction_attribute_662</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>9.570769</td>\n",
       "      <td>0.002207</td>\n",
       "      <td>4.092854</td>\n",
       "      <td>77.306435</td>\n",
       "      <td>0.006315</td>\n",
       "      <td>67.04355</td>\n",
       "      <td>0.081034</td>\n",
       "      <td>0.00021</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000790</td>\n",
       "      <td>0.004160</td>\n",
       "      <td>0.016778</td>\n",
       "      <td>0.004551</td>\n",
       "      <td>0.031221</td>\n",
       "      <td>0.030402</td>\n",
       "      <td>0.271082</td>\n",
       "      <td>0.272400</td>\n",
       "      <td>0.146834</td>\n",
       "      <td>0.000604</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>221000.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.001627</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000151</td>\n",
       "      <td>0.020436</td>\n",
       "      <td>0.111137</td>\n",
       "      <td>0.183021</td>\n",
       "      <td>0.324152</td>\n",
       "      <td>0.324152</td>\n",
       "      <td>0.320594</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>25000.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.019143</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.009693</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.004387</td>\n",
       "      <td>0.117333</td>\n",
       "      <td>0.356340</td>\n",
       "      <td>0.356340</td>\n",
       "      <td>0.167810</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>86000.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003203</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.106848</td>\n",
       "      <td>0.106848</td>\n",
       "      <td>0.095351</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>215000.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.002247</td>\n",
       "      <td>0.002247</td>\n",
       "      <td>0.002247</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96801</th>\n",
       "      <td>0</td>\n",
       "      <td>156000.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.233492</td>\n",
       "      <td>0.233492</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96802</th>\n",
       "      <td>0</td>\n",
       "      <td>46000.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.078221</td>\n",
       "      <td>0.078221</td>\n",
       "      <td>0.051337</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96803</th>\n",
       "      <td>0</td>\n",
       "      <td>95000.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003132</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.014207</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.921664</td>\n",
       "      <td>0.921664</td>\n",
       "      <td>0.009728</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96804</th>\n",
       "      <td>0</td>\n",
       "      <td>88000.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.001000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003829</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.070867</td>\n",
       "      <td>0.070867</td>\n",
       "      <td>0.003829</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>96805</th>\n",
       "      <td>0</td>\n",
       "      <td>94000.0</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.00000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.003780</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.715895</td>\n",
       "      <td>0.715895</td>\n",
       "      <td>0.003780</td>\n",
       "      <td>0.000000</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>96806 rows × 664 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "       bad_flag  onus_attribute_1  transaction_attribute_1  \\\n",
       "0             0               NaN                 9.570769   \n",
       "1             0          221000.0                 0.000000   \n",
       "2             0           25000.0                 0.000000   \n",
       "3             0           86000.0                 0.000000   \n",
       "4             0          215000.0                 0.000000   \n",
       "...         ...               ...                      ...   \n",
       "96801         0          156000.0                 0.000000   \n",
       "96802         0           46000.0                 0.000000   \n",
       "96803         0           95000.0                 0.000000   \n",
       "96804         0           88000.0                 0.000000   \n",
       "96805         0           94000.0                 0.000000   \n",
       "\n",
       "       transaction_attribute_2  transaction_attribute_3  \\\n",
       "0                     0.002207                 4.092854   \n",
       "1                     0.000000                 0.000000   \n",
       "2                     0.000000                 0.000000   \n",
       "3                     0.000000                 0.000000   \n",
       "4                     0.000000                 0.000000   \n",
       "...                        ...                      ...   \n",
       "96801                 0.000000                 0.000000   \n",
       "96802                 0.000000                 0.000000   \n",
       "96803                 0.000000                 0.000000   \n",
       "96804                 0.000000                 0.000000   \n",
       "96805                 0.000000                 0.000000   \n",
       "\n",
       "       transaction_attribute_4  transaction_attribute_5  \\\n",
       "0                    77.306435                 0.006315   \n",
       "1                     0.000000                 0.000000   \n",
       "2                     0.000000                 0.000000   \n",
       "3                     0.000000                 0.000000   \n",
       "4                     0.000000                 0.000000   \n",
       "...                        ...                      ...   \n",
       "96801                 0.000000                 0.000000   \n",
       "96802                 0.000000                 0.000000   \n",
       "96803                 0.000000                 0.000000   \n",
       "96804                 0.000000                 0.000000   \n",
       "96805                 0.000000                 0.000000   \n",
       "\n",
       "       transaction_attribute_6  transaction_attribute_7  \\\n",
       "0                     67.04355                 0.081034   \n",
       "1                      0.00000                 0.000000   \n",
       "2                      0.00000                 0.000000   \n",
       "3                      0.00000                 0.000000   \n",
       "4                      0.00000                 0.000000   \n",
       "...                        ...                      ...   \n",
       "96801                  0.00000                 0.000000   \n",
       "96802                  0.00000                 0.000000   \n",
       "96803                  0.00000                 0.000000   \n",
       "96804                  0.00000                 0.000000   \n",
       "96805                  0.00000                 0.000000   \n",
       "\n",
       "       transaction_attribute_8  ...  transaction_attribute_653  \\\n",
       "0                      0.00021  ...                   0.000790   \n",
       "1                      0.00000  ...                   0.001627   \n",
       "2                      0.00000  ...                   0.019143   \n",
       "3                      0.00000  ...                   0.000000   \n",
       "4                      0.00000  ...                   0.000000   \n",
       "...                        ...  ...                        ...   \n",
       "96801                  0.00000  ...                   0.000000   \n",
       "96802                  0.00000  ...                   0.000000   \n",
       "96803                  0.00000  ...                   0.000000   \n",
       "96804                  0.00000  ...                   0.000000   \n",
       "96805                  0.00000  ...                   0.000000   \n",
       "\n",
       "       transaction_attribute_654  transaction_attribute_655  \\\n",
       "0                       0.004160                   0.016778   \n",
       "1                       0.000000                   0.000151   \n",
       "2                       0.000000                   0.009693   \n",
       "3                       0.000000                   0.000000   \n",
       "4                       0.000000                   0.000000   \n",
       "...                          ...                        ...   \n",
       "96801                   0.000000                   0.000000   \n",
       "96802                   0.000000                   0.000000   \n",
       "96803                   0.003132                   0.000000   \n",
       "96804                   0.001000                   0.000000   \n",
       "96805                   0.003780                   0.000000   \n",
       "\n",
       "       transaction_attribute_656  transaction_attribute_657  \\\n",
       "0                       0.004551                   0.031221   \n",
       "1                       0.020436                   0.111137   \n",
       "2                       0.000000                   0.004387   \n",
       "3                       0.000000                   0.003203   \n",
       "4                       0.000000                   0.000000   \n",
       "...                          ...                        ...   \n",
       "96801                   0.000000                   0.000000   \n",
       "96802                   0.000000                   0.000000   \n",
       "96803                   0.000000                   0.014207   \n",
       "96804                   0.000000                   0.003829   \n",
       "96805                   0.000000                   0.000000   \n",
       "\n",
       "       transaction_attribute_658  transaction_attribute_659  \\\n",
       "0                       0.030402                   0.271082   \n",
       "1                       0.183021                   0.324152   \n",
       "2                       0.117333                   0.356340   \n",
       "3                       0.000000                   0.106848   \n",
       "4                       0.000000                   0.002247   \n",
       "...                          ...                        ...   \n",
       "96801                   0.000000                   0.233492   \n",
       "96802                   0.000000                   0.078221   \n",
       "96803                   0.000000                   0.921664   \n",
       "96804                   0.000000                   0.070867   \n",
       "96805                   0.000000                   0.715895   \n",
       "\n",
       "       transaction_attribute_660  transaction_attribute_661  \\\n",
       "0                       0.272400                   0.146834   \n",
       "1                       0.324152                   0.320594   \n",
       "2                       0.356340                   0.167810   \n",
       "3                       0.106848                   0.095351   \n",
       "4                       0.002247                   0.002247   \n",
       "...                          ...                        ...   \n",
       "96801                   0.233492                   0.000000   \n",
       "96802                   0.078221                   0.051337   \n",
       "96803                   0.921664                   0.009728   \n",
       "96804                   0.070867                   0.003829   \n",
       "96805                   0.715895                   0.003780   \n",
       "\n",
       "       transaction_attribute_662  \n",
       "0                       0.000604  \n",
       "1                       0.000000  \n",
       "2                       0.000000  \n",
       "3                       0.000000  \n",
       "4                       0.000000  \n",
       "...                          ...  \n",
       "96801                   0.000000  \n",
       "96802                   0.000000  \n",
       "96803                   0.000000  \n",
       "96804                   0.000000  \n",
       "96805                   0.000000  \n",
       "\n",
       "[96806 rows x 664 columns]"
      ]
     },
     "execution_count": 50,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'tran_dataset"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "b2a98a6b-f210-4c23-8716-0968314e31b9",
   "metadata": {},
   "outputs": [],
   "source": [
    "# trtransaction_attributes aur tran_dataset mein mismatch remove karne ke liye\n",
    "\n",
    "transaction_attributes = [col for col in dev_dataset.columns if col.startswith(\"transaction\")]\n",
    "transaction_attributes = [col for col in transaction_attributes if col in tran_dataset.columns]\n",
    "tran_dataset.columns = tran_dataset.columns.str.strip()  # Remove extra spaces\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "4e715019-c0aa-49c2-bf4a-7398fa4e5e94",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "bad_flag                         0\n",
       "onus_attribute_1             25231\n",
       "transaction_attribute_1      25231\n",
       "transaction_attribute_2      25231\n",
       "transaction_attribute_3      25231\n",
       "                             ...  \n",
       "transaction_attribute_658    25231\n",
       "transaction_attribute_659    25231\n",
       "transaction_attribute_660    25231\n",
       "transaction_attribute_661    25231\n",
       "transaction_attribute_662    25231\n",
       "Length: 664, dtype: int64"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tran_dataset.isnull().sum()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "e77628c3-ead2-478c-b0fe-2217d5d38d88",
   "metadata": {},
   "source": [
    "# ## Outlier removal\n",
    "# %%\n",
    "Q1 = tran_dataset.quantile(0.25)\n",
    "Q3 = tran_dataset.quantile(0.75)\n",
    "IQR = Q3 - Q1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 60,
   "id": "3da53ea6-8916-453d-8e7f-eba8d0b48cb8",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Outlier removal with per-column quantiles\n",
    "'tran_dataset_no_outliers = tran_dataset.copy()  # Copy the original dataset\n",
    "\n",
    "'for col in transaction_attributes:\n",
    " '   Q1 = tran_dataset_no_outliers[col].quantile(0.25)  # Column-specific Q1\n",
    "  '  Q3 = tran_dataset_no_outliers[col].quantile(0.75)  # Column-specific Q3\n",
    "  '  IQR = Q3 - Q1\n",
    "    # Filter rows within the IQR range\n",
    " '   tran_dataset_no_outliers = tran_dataset_no_outliers[\n",
    "  '''      ~((tran_dataset_no_outliers[col] < (Q1 - 1.5 * IQR)) | (tran_dataset_no_outliers[col] > (Q3 + 1.5 * IQR)))\n",
    "    ]\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "id": "cc022073-7cb8-4ad9-8e27-ba8099265ead",
   "metadata": {},
   "outputs": [],
   "source": [
    "## outliers = (tran_dataset < (Q1 - 1.5 * IQR)) | (tran_dataset > (Q3 + 1.5 * IQR))\n",
    "## tran_dataset_no_outliers = tran_dataset[~outliers.any(axis=1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "id": "25973a33-108f-4f03-b237-296c47493df6",
   "metadata": {},
   "outputs": [
    {
     "ename": "SyntaxError",
     "evalue": "incomplete input (2811976646.py, line 2)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;36m  Cell \u001b[1;32mIn[94], line 2\u001b[1;36m\u001b[0m\n\u001b[1;33m    '''skewness = tran_dataset_no_outliers[transaction_attributes].skew()\u001b[0m\n\u001b[1;37m    ^\u001b[0m\n\u001b[1;31mSyntaxError\u001b[0m\u001b[1;31m:\u001b[0m incomplete input\n"
     ]
    }
   ],
   "source": [
    "# Calculate skewness after correct outlier removal\n",
    "'''skewness = tran_dataset_no_outliers[transaction_attributes].skew()\n",
    "print(skewness)"
   ]
  },
  {
   "cell_type": "raw",
   "id": "3492d1ba-14c4-4d2e-ac68-8cb8b073f719",
   "metadata": {},
   "source": [
    "for col in transaction_attributes:\n",
    "    if col in tran_dataset_no_outliers.columns:\n",
    "        print(f\"Skewness of {col}: {tran_dataset_no_outliers[col].skew()}\")"
   ]
  },
  {
   "cell_type": "raw",
   "id": "60950ca1-4ec6-4e46-96af-4266337f72f6",
   "metadata": {},
   "source": [
    "pt = PowerTransformer(method='yeo-johnson')\n",
    "tran_dataset_transformed = pd.DataFrame(pt.fit_transform(tran_dataset), columns=tran_dataset.columns)\n",
    "tran_dataset_transformed.describe()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 206,
   "id": "a3c74433-13fd-4732-8137-a4c669a06fb2",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Index(['bad_flag', 'onus_attribute_1', 'transaction_attribute_1',\n",
      "       'transaction_attribute_2', 'transaction_attribute_3',\n",
      "       'transaction_attribute_4', 'transaction_attribute_5',\n",
      "       'transaction_attribute_6', 'transaction_attribute_7',\n",
      "       'transaction_attribute_8',\n",
      "       ...\n",
      "       'transaction_attribute_653', 'transaction_attribute_654',\n",
      "       'transaction_attribute_655', 'transaction_attribute_656',\n",
      "       'transaction_attribute_657', 'transaction_attribute_658',\n",
      "       'transaction_attribute_659', 'transaction_attribute_660',\n",
      "       'transaction_attribute_661', 'transaction_attribute_662'],\n",
      "      dtype='object', length=664)\n"
     ]
    }
   ],
   "source": [
    "print(tran_dataset.columns)  # List all column names"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 208,
   "id": "b67e7da3-dc72-4099-ac9c-0e74989f428f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['transaction_attribute_1', 'transaction_attribute_2', 'transaction_attribute_3', 'transaction_attribute_4', 'transaction_attribute_5', 'transaction_attribute_6', 'transaction_attribute_7', 'transaction_attribute_8', 'transaction_attribute_9', 'transaction_attribute_10', 'transaction_attribute_11', 'transaction_attribute_12', 'transaction_attribute_13', 'transaction_attribute_14', 'transaction_attribute_15', 'transaction_attribute_16', 'transaction_attribute_17', 'transaction_attribute_18', 'transaction_attribute_19', 'transaction_attribute_20', 'transaction_attribute_21', 'transaction_attribute_22', 'transaction_attribute_23', 'transaction_attribute_24', 'transaction_attribute_25', 'transaction_attribute_26', 'transaction_attribute_27', 'transaction_attribute_28', 'transaction_attribute_29', 'transaction_attribute_30', 'transaction_attribute_31', 'transaction_attribute_32', 'transaction_attribute_33', 'transaction_attribute_34', 'transaction_attribute_35', 'transaction_attribute_36', 'transaction_attribute_37', 'transaction_attribute_38', 'transaction_attribute_39', 'transaction_attribute_40', 'transaction_attribute_41', 'transaction_attribute_42', 'transaction_attribute_43', 'transaction_attribute_44', 'transaction_attribute_45', 'transaction_attribute_46', 'transaction_attribute_47', 'transaction_attribute_48', 'transaction_attribute_49', 'transaction_attribute_50', 'transaction_attribute_51', 'transaction_attribute_52', 'transaction_attribute_53', 'transaction_attribute_54', 'transaction_attribute_55', 'transaction_attribute_56', 'transaction_attribute_57', 'transaction_attribute_58', 'transaction_attribute_59', 'transaction_attribute_60', 'transaction_attribute_61', 'transaction_attribute_62', 'transaction_attribute_63', 'transaction_attribute_64', 'transaction_attribute_65', 'transaction_attribute_66', 'transaction_attribute_67', 'transaction_attribute_68', 'transaction_attribute_69', 'transaction_attribute_70', 'transaction_attribute_71', 'transaction_attribute_72', 'transaction_attribute_73', 'transaction_attribute_74', 'transaction_attribute_75', 'transaction_attribute_76', 'transaction_attribute_77', 'transaction_attribute_78', 'transaction_attribute_79', 'transaction_attribute_80', 'transaction_attribute_81', 'transaction_attribute_82', 'transaction_attribute_83', 'transaction_attribute_84', 'transaction_attribute_85', 'transaction_attribute_86', 'transaction_attribute_87', 'transaction_attribute_88', 'transaction_attribute_89', 'transaction_attribute_90', 'transaction_attribute_91', 'transaction_attribute_92', 'transaction_attribute_93', 'transaction_attribute_94', 'transaction_attribute_95', 'transaction_attribute_96', 'transaction_attribute_97', 'transaction_attribute_98', 'transaction_attribute_99', 'transaction_attribute_100', 'transaction_attribute_101', 'transaction_attribute_102', 'transaction_attribute_103', 'transaction_attribute_104', 'transaction_attribute_105', 'transaction_attribute_106', 'transaction_attribute_107', 'transaction_attribute_108', 'transaction_attribute_109', 'transaction_attribute_110', 'transaction_attribute_111', 'transaction_attribute_112', 'transaction_attribute_113', 'transaction_attribute_114', 'transaction_attribute_115', 'transaction_attribute_116', 'transaction_attribute_117', 'transaction_attribute_118', 'transaction_attribute_119', 'transaction_attribute_120', 'transaction_attribute_121', 'transaction_attribute_122', 'transaction_attribute_123', 'transaction_attribute_124', 'transaction_attribute_125', 'transaction_attribute_126', 'transaction_attribute_127', 'transaction_attribute_128', 'transaction_attribute_129', 'transaction_attribute_130', 'transaction_attribute_131', 'transaction_attribute_132', 'transaction_attribute_133', 'transaction_attribute_134', 'transaction_attribute_135', 'transaction_attribute_136', 'transaction_attribute_137', 'transaction_attribute_138', 'transaction_attribute_139', 'transaction_attribute_140', 'transaction_attribute_141', 'transaction_attribute_142', 'transaction_attribute_143', 'transaction_attribute_144', 'transaction_attribute_145', 'transaction_attribute_146', 'transaction_attribute_147', 'transaction_attribute_148', 'transaction_attribute_149', 'transaction_attribute_150', 'transaction_attribute_151', 'transaction_attribute_152', 'transaction_attribute_153', 'transaction_attribute_154', 'transaction_attribute_155', 'transaction_attribute_156', 'transaction_attribute_157', 'transaction_attribute_158', 'transaction_attribute_159', 'transaction_attribute_160', 'transaction_attribute_161', 'transaction_attribute_162', 'transaction_attribute_163', 'transaction_attribute_164', 'transaction_attribute_165', 'transaction_attribute_166', 'transaction_attribute_167', 'transaction_attribute_168', 'transaction_attribute_169', 'transaction_attribute_170', 'transaction_attribute_171', 'transaction_attribute_172', 'transaction_attribute_173', 'transaction_attribute_174', 'transaction_attribute_175', 'transaction_attribute_176', 'transaction_attribute_177', 'transaction_attribute_178', 'transaction_attribute_179', 'transaction_attribute_180', 'transaction_attribute_181', 'transaction_attribute_182', 'transaction_attribute_183', 'transaction_attribute_184', 'transaction_attribute_185', 'transaction_attribute_186', 'transaction_attribute_187', 'transaction_attribute_188', 'transaction_attribute_189', 'transaction_attribute_190', 'transaction_attribute_191', 'transaction_attribute_192', 'transaction_attribute_193', 'transaction_attribute_194', 'transaction_attribute_195', 'transaction_attribute_196', 'transaction_attribute_197', 'transaction_attribute_198', 'transaction_attribute_199', 'transaction_attribute_200', 'transaction_attribute_201', 'transaction_attribute_202', 'transaction_attribute_203', 'transaction_attribute_204', 'transaction_attribute_205', 'transaction_attribute_206', 'transaction_attribute_207', 'transaction_attribute_208', 'transaction_attribute_209', 'transaction_attribute_210', 'transaction_attribute_211', 'transaction_attribute_212', 'transaction_attribute_213', 'transaction_attribute_214', 'transaction_attribute_215', 'transaction_attribute_216', 'transaction_attribute_217', 'transaction_attribute_218', 'transaction_attribute_219', 'transaction_attribute_220', 'transaction_attribute_221', 'transaction_attribute_222', 'transaction_attribute_223', 'transaction_attribute_224', 'transaction_attribute_225', 'transaction_attribute_226', 'transaction_attribute_227', 'transaction_attribute_228', 'transaction_attribute_229', 'transaction_attribute_230', 'transaction_attribute_231', 'transaction_attribute_232', 'transaction_attribute_233', 'transaction_attribute_234', 'transaction_attribute_235', 'transaction_attribute_236', 'transaction_attribute_237', 'transaction_attribute_238', 'transaction_attribute_239', 'transaction_attribute_240', 'transaction_attribute_241', 'transaction_attribute_242', 'transaction_attribute_243', 'transaction_attribute_244', 'transaction_attribute_245', 'transaction_attribute_246', 'transaction_attribute_247', 'transaction_attribute_248', 'transaction_attribute_249', 'transaction_attribute_250', 'transaction_attribute_251', 'transaction_attribute_252', 'transaction_attribute_253', 'transaction_attribute_254', 'transaction_attribute_255', 'transaction_attribute_256', 'transaction_attribute_257', 'transaction_attribute_258', 'transaction_attribute_259', 'transaction_attribute_260', 'transaction_attribute_261', 'transaction_attribute_262', 'transaction_attribute_263', 'transaction_attribute_264', 'transaction_attribute_265', 'transaction_attribute_266', 'transaction_attribute_267', 'transaction_attribute_268', 'transaction_attribute_269', 'transaction_attribute_270', 'transaction_attribute_271', 'transaction_attribute_272', 'transaction_attribute_273', 'transaction_attribute_274', 'transaction_attribute_275', 'transaction_attribute_276', 'transaction_attribute_277', 'transaction_attribute_278', 'transaction_attribute_279', 'transaction_attribute_280', 'transaction_attribute_281', 'transaction_attribute_282', 'transaction_attribute_283', 'transaction_attribute_284', 'transaction_attribute_285', 'transaction_attribute_286', 'transaction_attribute_287', 'transaction_attribute_288', 'transaction_attribute_289', 'transaction_attribute_290', 'transaction_attribute_291', 'transaction_attribute_292', 'transaction_attribute_293', 'transaction_attribute_294', 'transaction_attribute_295', 'transaction_attribute_296', 'transaction_attribute_297', 'transaction_attribute_298', 'transaction_attribute_299', 'transaction_attribute_300', 'transaction_attribute_301', 'transaction_attribute_302', 'transaction_attribute_303', 'transaction_attribute_304', 'transaction_attribute_305', 'transaction_attribute_306', 'transaction_attribute_307', 'transaction_attribute_308', 'transaction_attribute_309', 'transaction_attribute_310', 'transaction_attribute_311', 'transaction_attribute_312', 'transaction_attribute_313', 'transaction_attribute_314', 'transaction_attribute_315', 'transaction_attribute_316', 'transaction_attribute_317', 'transaction_attribute_318', 'transaction_attribute_319', 'transaction_attribute_320', 'transaction_attribute_321', 'transaction_attribute_322', 'transaction_attribute_323', 'transaction_attribute_324', 'transaction_attribute_325', 'transaction_attribute_326', 'transaction_attribute_327', 'transaction_attribute_328', 'transaction_attribute_329', 'transaction_attribute_330', 'transaction_attribute_331', 'transaction_attribute_332', 'transaction_attribute_333', 'transaction_attribute_334', 'transaction_attribute_335', 'transaction_attribute_336', 'transaction_attribute_337', 'transaction_attribute_338', 'transaction_attribute_339', 'transaction_attribute_340', 'transaction_attribute_341', 'transaction_attribute_342', 'transaction_attribute_343', 'transaction_attribute_344', 'transaction_attribute_345', 'transaction_attribute_346', 'transaction_attribute_347', 'transaction_attribute_348', 'transaction_attribute_349', 'transaction_attribute_350', 'transaction_attribute_351', 'transaction_attribute_352', 'transaction_attribute_353', 'transaction_attribute_354', 'transaction_attribute_355', 'transaction_attribute_356', 'transaction_attribute_357', 'transaction_attribute_358', 'transaction_attribute_359', 'transaction_attribute_360', 'transaction_attribute_361', 'transaction_attribute_362', 'transaction_attribute_363', 'transaction_attribute_364', 'transaction_attribute_365', 'transaction_attribute_366', 'transaction_attribute_367', 'transaction_attribute_368', 'transaction_attribute_369', 'transaction_attribute_370', 'transaction_attribute_371', 'transaction_attribute_372', 'transaction_attribute_373', 'transaction_attribute_374', 'transaction_attribute_375', 'transaction_attribute_376', 'transaction_attribute_377', 'transaction_attribute_378', 'transaction_attribute_379', 'transaction_attribute_380', 'transaction_attribute_381', 'transaction_attribute_382', 'transaction_attribute_383', 'transaction_attribute_384', 'transaction_attribute_385', 'transaction_attribute_386', 'transaction_attribute_387', 'transaction_attribute_388', 'transaction_attribute_389', 'transaction_attribute_390', 'transaction_attribute_391', 'transaction_attribute_392', 'transaction_attribute_393', 'transaction_attribute_394', 'transaction_attribute_395', 'transaction_attribute_396', 'transaction_attribute_397', 'transaction_attribute_398', 'transaction_attribute_399', 'transaction_attribute_400', 'transaction_attribute_401', 'transaction_attribute_402', 'transaction_attribute_403', 'transaction_attribute_404', 'transaction_attribute_405', 'transaction_attribute_406', 'transaction_attribute_407', 'transaction_attribute_408', 'transaction_attribute_409', 'transaction_attribute_410', 'transaction_attribute_411', 'transaction_attribute_412', 'transaction_attribute_413', 'transaction_attribute_414', 'transaction_attribute_415', 'transaction_attribute_416', 'transaction_attribute_417', 'transaction_attribute_418', 'transaction_attribute_419', 'transaction_attribute_420', 'transaction_attribute_421', 'transaction_attribute_422', 'transaction_attribute_423', 'transaction_attribute_424', 'transaction_attribute_425', 'transaction_attribute_426', 'transaction_attribute_427', 'transaction_attribute_428', 'transaction_attribute_429', 'transaction_attribute_430', 'transaction_attribute_431', 'transaction_attribute_432', 'transaction_attribute_433', 'transaction_attribute_434', 'transaction_attribute_435', 'transaction_attribute_436', 'transaction_attribute_437', 'transaction_attribute_438', 'transaction_attribute_439', 'transaction_attribute_440', 'transaction_attribute_441', 'transaction_attribute_442', 'transaction_attribute_443', 'transaction_attribute_444', 'transaction_attribute_445', 'transaction_attribute_446', 'transaction_attribute_447', 'transaction_attribute_448', 'transaction_attribute_449', 'transaction_attribute_450', 'transaction_attribute_451', 'transaction_attribute_452', 'transaction_attribute_453', 'transaction_attribute_454', 'transaction_attribute_455', 'transaction_attribute_456', 'transaction_attribute_457', 'transaction_attribute_458', 'transaction_attribute_459', 'transaction_attribute_460', 'transaction_attribute_461', 'transaction_attribute_462', 'transaction_attribute_463', 'transaction_attribute_464', 'transaction_attribute_465', 'transaction_attribute_466', 'transaction_attribute_467', 'transaction_attribute_468', 'transaction_attribute_469', 'transaction_attribute_470', 'transaction_attribute_471', 'transaction_attribute_472', 'transaction_attribute_473', 'transaction_attribute_474', 'transaction_attribute_475', 'transaction_attribute_476', 'transaction_attribute_477', 'transaction_attribute_478', 'transaction_attribute_479', 'transaction_attribute_480', 'transaction_attribute_481', 'transaction_attribute_482', 'transaction_attribute_483', 'transaction_attribute_484', 'transaction_attribute_485', 'transaction_attribute_486', 'transaction_attribute_487', 'transaction_attribute_488', 'transaction_attribute_489', 'transaction_attribute_490', 'transaction_attribute_491', 'transaction_attribute_492', 'transaction_attribute_493', 'transaction_attribute_494', 'transaction_attribute_495', 'transaction_attribute_496', 'transaction_attribute_497', 'transaction_attribute_498', 'transaction_attribute_499', 'transaction_attribute_500', 'transaction_attribute_501', 'transaction_attribute_502', 'transaction_attribute_503', 'transaction_attribute_504', 'transaction_attribute_505', 'transaction_attribute_506', 'transaction_attribute_507', 'transaction_attribute_508', 'transaction_attribute_509', 'transaction_attribute_510', 'transaction_attribute_511', 'transaction_attribute_512', 'transaction_attribute_513', 'transaction_attribute_514', 'transaction_attribute_515', 'transaction_attribute_516', 'transaction_attribute_517', 'transaction_attribute_518', 'transaction_attribute_519', 'transaction_attribute_520', 'transaction_attribute_521', 'transaction_attribute_522', 'transaction_attribute_523', 'transaction_attribute_524', 'transaction_attribute_525', 'transaction_attribute_526', 'transaction_attribute_527', 'transaction_attribute_528', 'transaction_attribute_529', 'transaction_attribute_530', 'transaction_attribute_531', 'transaction_attribute_532', 'transaction_attribute_533', 'transaction_attribute_534', 'transaction_attribute_535', 'transaction_attribute_536', 'transaction_attribute_537', 'transaction_attribute_538', 'transaction_attribute_539', 'transaction_attribute_540', 'transaction_attribute_541', 'transaction_attribute_542', 'transaction_attribute_543', 'transaction_attribute_544', 'transaction_attribute_545', 'transaction_attribute_546', 'transaction_attribute_547', 'transaction_attribute_548', 'transaction_attribute_549', 'transaction_attribute_550', 'transaction_attribute_551', 'transaction_attribute_552', 'transaction_attribute_553', 'transaction_attribute_554', 'transaction_attribute_555', 'transaction_attribute_556', 'transaction_attribute_557', 'transaction_attribute_558', 'transaction_attribute_559', 'transaction_attribute_560', 'transaction_attribute_561', 'transaction_attribute_562', 'transaction_attribute_563', 'transaction_attribute_564', 'transaction_attribute_565', 'transaction_attribute_566', 'transaction_attribute_567', 'transaction_attribute_568', 'transaction_attribute_569', 'transaction_attribute_570', 'transaction_attribute_571', 'transaction_attribute_572', 'transaction_attribute_573', 'transaction_attribute_574', 'transaction_attribute_575', 'transaction_attribute_576', 'transaction_attribute_577', 'transaction_attribute_578', 'transaction_attribute_579', 'transaction_attribute_580', 'transaction_attribute_581', 'transaction_attribute_582', 'transaction_attribute_583', 'transaction_attribute_584', 'transaction_attribute_585', 'transaction_attribute_586', 'transaction_attribute_587', 'transaction_attribute_588', 'transaction_attribute_589', 'transaction_attribute_590', 'transaction_attribute_591', 'transaction_attribute_592', 'transaction_attribute_593', 'transaction_attribute_594', 'transaction_attribute_595', 'transaction_attribute_596', 'transaction_attribute_597', 'transaction_attribute_598', 'transaction_attribute_599', 'transaction_attribute_600', 'transaction_attribute_601', 'transaction_attribute_602', 'transaction_attribute_603', 'transaction_attribute_604', 'transaction_attribute_605', 'transaction_attribute_606', 'transaction_attribute_607', 'transaction_attribute_608', 'transaction_attribute_609', 'transaction_attribute_610', 'transaction_attribute_611', 'transaction_attribute_612', 'transaction_attribute_613', 'transaction_attribute_614', 'transaction_attribute_615', 'transaction_attribute_616', 'transaction_attribute_617', 'transaction_attribute_618', 'transaction_attribute_619', 'transaction_attribute_620', 'transaction_attribute_621', 'transaction_attribute_622', 'transaction_attribute_623', 'transaction_attribute_624', 'transaction_attribute_625', 'transaction_attribute_626', 'transaction_attribute_627', 'transaction_attribute_628', 'transaction_attribute_629', 'transaction_attribute_630', 'transaction_attribute_631', 'transaction_attribute_632', 'transaction_attribute_633', 'transaction_attribute_634', 'transaction_attribute_635', 'transaction_attribute_636', 'transaction_attribute_637', 'transaction_attribute_638', 'transaction_attribute_639', 'transaction_attribute_640', 'transaction_attribute_641', 'transaction_attribute_642', 'transaction_attribute_643', 'transaction_attribute_644', 'transaction_attribute_645', 'transaction_attribute_646', 'transaction_attribute_647', 'transaction_attribute_648', 'transaction_attribute_649', 'transaction_attribute_650', 'transaction_attribute_651', 'transaction_attribute_652', 'transaction_attribute_653', 'transaction_attribute_654', 'transaction_attribute_655', 'transaction_attribute_656', 'transaction_attribute_657', 'transaction_attribute_658', 'transaction_attribute_659', 'transaction_attribute_660', 'transaction_attribute_661', 'transaction_attribute_662']\n"
     ]
    }
   ],
   "source": [
    "print(transaction_attributes)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 144,
   "id": "3cd6a632-0d40-4716-b294-40c94a36f731",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.preprocessing import StandardScaler\n",
    "\n",
    "# Standardize the data to improve KNN imputation results\n",
    "scaler = StandardScaler()\n",
    "scaled_data = scaler.fit_transform(tran_dataset)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "a8da87ca-d958-4704-b4ec-b168694604f8",
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.sparse import csr_matrix, save_npz, load_npz\n",
    "from sklearn.impute import KNNImputer\n",
    "import os\n",
    "import psutil"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 195,
   "id": "fd11d4e5-585e-4a8b-8989-aa258e536ce2",
   "metadata": {},
   "outputs": [],
   "source": [
    "sparse_data = csr_matrix(tran_dataset.values)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 196,
   "id": "daf85839-31c5-440e-af79-83665d8a387b",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<96806x664 sparse matrix of type '<class 'numpy.float64'>'\n",
       "\twith 25407200 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 196,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sparse_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 197,
   "id": "aa398281-50b6-4b6b-9111-40c057a91943",
   "metadata": {},
   "outputs": [],
   "source": [
    "knn_imputer = KNNImputer(n_neighbors=5, weights=\"uniform\", metric=\"nan_euclidean\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 198,
   "id": "380cab84-13a2-484e-bc99-a88bc75055b2",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Define a function to process data in batches\n",
    "def knn_impute_in_batches(sparse_matrix, batch_size, output_file):\n",
    "    rows = sparse_matrix.shape[0]\n",
    "    results = []\n",
    "    for start in range(0, rows, batch_size):\n",
    "        end = min(start + batch_size, rows)\n",
    "        \n",
    "        # Monitoring memory usage before processing each batch\n",
    "        memory_info = psutil.virtual_memory()\n",
    "        print(f\"Memory Usage: {memory_info.percent}% (Available: {memory_info.available / (1024 ** 3):.2f} GB)\")\n",
    "        \n",
    "        # If memory usage exceeds a threshold, reduce the batch size\n",
    "        if memory_info.percent > 80:\n",
    "            batch_size = max(1000, batch_size // 2)  # Adjust batch size down\n",
    "\n",
    "        batch = sparse_matrix[start:end].toarray()  # Convert a batch to dense\n",
    "        print(f\"Processing rows {start} to {end}...\")\n",
    "\n",
    "        # Perform KNN imputation\n",
    "        imputed_batch = knn_imputer.fit_transform(batch)  \n",
    "        results.append(imputed_batch)\n",
    "\n",
    "        # Save intermediate results to disk after each batch\n",
    "        imputed_batch_df = pd.DataFrame(imputed_batch, columns=transaction_attributes)\n",
    "        imputed_batch_df.to_csv(output_file, mode='a', header=(start == 0), index=False)  # Append results\n",
    "        \n",
    "    return results"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5f971b70-7106-492c-80ba-04aa89107a95",
   "metadata": {},
   "outputs": [],
   "source": [
    "tran_dataset.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 204,
   "id": "5fb20791-83a4-42e2-a66b-db810a050d4f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Memory Usage: 72.0% (Available: 4.41 GB)\n",
      "Processing rows 0 to 10000...\n"
     ]
    },
    {
     "ename": "ValueError",
     "evalue": "Shape of passed values is (10000, 664), indices imply (10000, 662)",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mValueError\u001b[0m                                Traceback (most recent call last)",
      "Cell \u001b[1;32mIn[204], line 8\u001b[0m\n\u001b[0;32m      5\u001b[0m output_file \u001b[38;5;241m=\u001b[39m \u001b[38;5;124mr\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mD:\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mMACHINE LEARNING\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mHackathon\u001b[39m\u001b[38;5;124m\\\u001b[39m\u001b[38;5;124mimputed_data.csv\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[0;32m      7\u001b[0m \u001b[38;5;66;03m# Start the batch-wise imputation process\u001b[39;00m\n\u001b[1;32m----> 8\u001b[0m \u001b[43mknn_impute_in_batches\u001b[49m\u001b[43m(\u001b[49m\u001b[43msparse_data\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mbatch_size\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43moutput_file\u001b[49m\u001b[43m)\u001b[49m\n",
      "Cell \u001b[1;32mIn[198], line 24\u001b[0m, in \u001b[0;36mknn_impute_in_batches\u001b[1;34m(sparse_matrix, batch_size, output_file)\u001b[0m\n\u001b[0;32m     21\u001b[0m     results\u001b[38;5;241m.\u001b[39mappend(imputed_batch)\n\u001b[0;32m     23\u001b[0m     \u001b[38;5;66;03m# Save intermediate results to disk after each batch\u001b[39;00m\n\u001b[1;32m---> 24\u001b[0m     imputed_batch_df \u001b[38;5;241m=\u001b[39m \u001b[43mpd\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mDataFrame\u001b[49m\u001b[43m(\u001b[49m\u001b[43mimputed_batch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtransaction_attributes\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m     25\u001b[0m     imputed_batch_df\u001b[38;5;241m.\u001b[39mto_csv(output_file, mode\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m'\u001b[39m\u001b[38;5;124ma\u001b[39m\u001b[38;5;124m'\u001b[39m, header\u001b[38;5;241m=\u001b[39m(start \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m), index\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mFalse\u001b[39;00m)  \u001b[38;5;66;03m# Append results\u001b[39;00m\n\u001b[0;32m     27\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m results\n",
      "File \u001b[1;32mD:\\anaconda\\Lib\\site-packages\\pandas\\core\\frame.py:827\u001b[0m, in \u001b[0;36mDataFrame.__init__\u001b[1;34m(self, data, index, columns, dtype, copy)\u001b[0m\n\u001b[0;32m    816\u001b[0m         mgr \u001b[38;5;241m=\u001b[39m dict_to_mgr(\n\u001b[0;32m    817\u001b[0m             \u001b[38;5;66;03m# error: Item \"ndarray\" of \"Union[ndarray, Series, Index]\" has no\u001b[39;00m\n\u001b[0;32m    818\u001b[0m             \u001b[38;5;66;03m# attribute \"name\"\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    824\u001b[0m             copy\u001b[38;5;241m=\u001b[39m_copy,\n\u001b[0;32m    825\u001b[0m         )\n\u001b[0;32m    826\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m--> 827\u001b[0m         mgr \u001b[38;5;241m=\u001b[39m \u001b[43mndarray_to_mgr\u001b[49m\u001b[43m(\u001b[49m\n\u001b[0;32m    828\u001b[0m \u001b[43m            \u001b[49m\u001b[43mdata\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    829\u001b[0m \u001b[43m            \u001b[49m\u001b[43mindex\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    830\u001b[0m \u001b[43m            \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    831\u001b[0m \u001b[43m            \u001b[49m\u001b[43mdtype\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mdtype\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    832\u001b[0m \u001b[43m            \u001b[49m\u001b[43mcopy\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcopy\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    833\u001b[0m \u001b[43m            \u001b[49m\u001b[43mtyp\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mmanager\u001b[49m\u001b[43m,\u001b[49m\n\u001b[0;32m    834\u001b[0m \u001b[43m        \u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    836\u001b[0m \u001b[38;5;66;03m# For data is list-like, or Iterable (will consume into list)\u001b[39;00m\n\u001b[0;32m    837\u001b[0m \u001b[38;5;28;01melif\u001b[39;00m is_list_like(data):\n",
      "File \u001b[1;32mD:\\anaconda\\Lib\\site-packages\\pandas\\core\\internals\\construction.py:336\u001b[0m, in \u001b[0;36mndarray_to_mgr\u001b[1;34m(values, index, columns, dtype, copy, typ)\u001b[0m\n\u001b[0;32m    331\u001b[0m \u001b[38;5;66;03m# _prep_ndarraylike ensures that values.ndim == 2 at this point\u001b[39;00m\n\u001b[0;32m    332\u001b[0m index, columns \u001b[38;5;241m=\u001b[39m _get_axes(\n\u001b[0;32m    333\u001b[0m     values\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m0\u001b[39m], values\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m1\u001b[39m], index\u001b[38;5;241m=\u001b[39mindex, columns\u001b[38;5;241m=\u001b[39mcolumns\n\u001b[0;32m    334\u001b[0m )\n\u001b[1;32m--> 336\u001b[0m \u001b[43m_check_values_indices_shape_match\u001b[49m\u001b[43m(\u001b[49m\u001b[43mvalues\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mindex\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mcolumns\u001b[49m\u001b[43m)\u001b[49m\n\u001b[0;32m    338\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m typ \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124marray\u001b[39m\u001b[38;5;124m\"\u001b[39m:\n\u001b[0;32m    339\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28missubclass\u001b[39m(values\u001b[38;5;241m.\u001b[39mdtype\u001b[38;5;241m.\u001b[39mtype, \u001b[38;5;28mstr\u001b[39m):\n",
      "File \u001b[1;32mD:\\anaconda\\Lib\\site-packages\\pandas\\core\\internals\\construction.py:420\u001b[0m, in \u001b[0;36m_check_values_indices_shape_match\u001b[1;34m(values, index, columns)\u001b[0m\n\u001b[0;32m    418\u001b[0m passed \u001b[38;5;241m=\u001b[39m values\u001b[38;5;241m.\u001b[39mshape\n\u001b[0;32m    419\u001b[0m implied \u001b[38;5;241m=\u001b[39m (\u001b[38;5;28mlen\u001b[39m(index), \u001b[38;5;28mlen\u001b[39m(columns))\n\u001b[1;32m--> 420\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mShape of passed values is \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mpassed\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m, indices imply \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mimplied\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n",
      "\u001b[1;31mValueError\u001b[0m: Shape of passed values is (10000, 664), indices imply (10000, 662)"
     ]
    }
   ],
   "source": [
    "# Choose an initial batch size based on the dataset size and available memory\n",
    "batch_size = 10000 if tran_dataset.shape[1] < 700 else 5000\n",
    "\n",
    "# Output file path (modified to store in D:\\MACHINE LEARNING\\Hackathon)\n",
    "output_file = r\"D:\\MACHINE LEARNING\\Hackathon\\imputed_data.csv\"\n",
    "\n",
    "# Start the batch-wise imputation process\n",
    "knn_impute_in_batches(sparse_data, batch_size, output_file)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "53f08ee9-53d6-4025-adc5-8119c6a70285",
   "metadata": {},
   "outputs": [],
   "source": [
    "# Optionally, load the final imputed data (if needed for further analysis)\n",
    "imputed_df = pd.read_csv(output_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e85a36cf-16b4-4429-a018-0b7b40e993d1",
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.experimental import enable_iterative_imputer  # noqa\n",
    "from sklearn.impute import IterativeImputer\n",
    "from sklearn.ensemble import RandomForestRegressor\n",
    "\n",
    "# Initialize IterativeImputer with Random Forest regressor\n",
    "iterative_imputer = IterativeImputer(estimator=RandomForestRegressor(n_estimators=100), random_state=0)\n",
    "\n",
    "# Fit and transform the dataset\n",
    "tran_dataset_imputed = iterative_imputer.fit_transform(tran_dataset)\n",
    "\n",
    "# Convert back to DataFrame for easier handling\n",
    "tran_dataset_imputed = pd.DataFrame(tran_dataset_imputed, columns=tran_dataset.columns)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e8afc647-93dd-403e-8eb9-d136e909a5a8",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}