FR_HOME_CREDIT_missing value 999 without agg_baseline_model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For Preprocessing\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import pandas as pd\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.metrics import accuracy_score\n",
    "from xgboost import plot_importance\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "import os\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from sklearn.externals import joblib\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "import dask.dataframe as dd\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import f1_score,recall_score,precision_score\n",
    "from dask.distributed import Client"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>SK_ID_CURR</th>\n",
       "      <th>TARGET</th>\n",
       "      <th>NAME_CONTRACT_TYPE</th>\n",
       "      <th>CODE_GENDER</th>\n",
       "      <th>FLAG_OWN_CAR</th>\n",
       "      <th>FLAG_OWN_REALTY</th>\n",
       "      <th>CNT_CHILDREN</th>\n",
       "      <th>AMT_INCOME_TOTAL</th>\n",
       "      <th>AMT_CREDIT</th>\n",
       "      <th>AMT_ANNUITY</th>\n",
       "      <th>...</th>\n",
       "      <th>FLAG_DOCUMENT_18</th>\n",
       "      <th>FLAG_DOCUMENT_19</th>\n",
       "      <th>FLAG_DOCUMENT_20</th>\n",
       "      <th>FLAG_DOCUMENT_21</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_HOUR</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_DAY</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_WEEK</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_MON</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_QRT</th>\n",
       "      <th>AMT_REQ_CREDIT_BUREAU_YEAR</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>100002</td>\n",
       "      <td>1</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>202500.0</td>\n",
       "      <td>406597.5</td>\n",
       "      <td>24700.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>1.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>100003</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>N</td>\n",
       "      <td>0</td>\n",
       "      <td>270000.0</td>\n",
       "      <td>1293502.5</td>\n",
       "      <td>35698.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>100004</td>\n",
       "      <td>0</td>\n",
       "      <td>Revolving loans</td>\n",
       "      <td>M</td>\n",
       "      <td>Y</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>67500.0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>6750.0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>100006</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>F</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>135000.0</td>\n",
       "      <td>312682.5</td>\n",
       "      <td>29686.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>100007</td>\n",
       "      <td>0</td>\n",
       "      <td>Cash loans</td>\n",
       "      <td>M</td>\n",
       "      <td>N</td>\n",
       "      <td>Y</td>\n",
       "      <td>0</td>\n",
       "      <td>121500.0</td>\n",
       "      <td>513000.0</td>\n",
       "      <td>21865.5</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "      <td>0.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 122 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   SK_ID_CURR  TARGET NAME_CONTRACT_TYPE CODE_GENDER FLAG_OWN_CAR  \\\n",
       "0      100002       1         Cash loans           M            N   \n",
       "1      100003       0         Cash loans           F            N   \n",
       "2      100004       0    Revolving loans           M            Y   \n",
       "3      100006       0         Cash loans           F            N   \n",
       "4      100007       0         Cash loans           M            N   \n",
       "\n",
       "  FLAG_OWN_REALTY  CNT_CHILDREN  AMT_INCOME_TOTAL  AMT_CREDIT  AMT_ANNUITY  \\\n",
       "0               Y             0          202500.0    406597.5      24700.5   \n",
       "1               N             0          270000.0   1293502.5      35698.5   \n",
       "2               Y             0           67500.0    135000.0       6750.0   \n",
       "3               Y             0          135000.0    312682.5      29686.5   \n",
       "4               Y             0          121500.0    513000.0      21865.5   \n",
       "\n",
       "              ...              FLAG_DOCUMENT_18 FLAG_DOCUMENT_19  \\\n",
       "0             ...                             0                0   \n",
       "1             ...                             0                0   \n",
       "2             ...                             0                0   \n",
       "3             ...                             0                0   \n",
       "4             ...                             0                0   \n",
       "\n",
       "  FLAG_DOCUMENT_20 FLAG_DOCUMENT_21 AMT_REQ_CREDIT_BUREAU_HOUR  \\\n",
       "0                0                0                        0.0   \n",
       "1                0                0                        0.0   \n",
       "2                0                0                        0.0   \n",
       "3                0                0                        NaN   \n",
       "4                0                0                        0.0   \n",
       "\n",
       "  AMT_REQ_CREDIT_BUREAU_DAY  AMT_REQ_CREDIT_BUREAU_WEEK  \\\n",
       "0                       0.0                         0.0   \n",
       "1                       0.0                         0.0   \n",
       "2                       0.0                         0.0   \n",
       "3                       NaN                         NaN   \n",
       "4                       0.0                         0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_MON  AMT_REQ_CREDIT_BUREAU_QRT  \\\n",
       "0                        0.0                        0.0   \n",
       "1                        0.0                        0.0   \n",
       "2                        0.0                        0.0   \n",
       "3                        NaN                        NaN   \n",
       "4                        0.0                        0.0   \n",
       "\n",
       "   AMT_REQ_CREDIT_BUREAU_YEAR  \n",
       "0                         1.0  \n",
       "1                         0.0  \n",
       "2                         0.0  \n",
       "3                         NaN  \n",
       "4                         0.0  \n",
       "\n",
       "[5 rows x 122 columns]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X = pd.read_csv(\"application_train.csv\")\n",
    "df_test = pd.read_csv(\"application_test.csv\")\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "F    202448\n",
       "M    105059\n",
       "Name: CODE_GENDER, dtype: int64"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X['CODE_GENDER'].replace('XNA',np.nan, inplace=True)\n",
    "X['CODE_GENDER'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "X['DAYS_LAST_PHONE_CHANGE'].replace(0, np.nan, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "X['DAYS_EMPLOYED'].replace({365243: np.nan}, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "# AGGREGATION_RECIPIES = [\n",
    "#     (['CODE_GENDER', 'NAME_EDUCATION_TYPE'], [('AMT_ANNUITY', 'max')                                             \n",
    "                                              \n",
    "#                                               ]),\n",
    "#     (['CODE_GENDER', 'ORGANIZATION_TYPE'], [\n",
    "#                                             ('AMT_INCOME_TOTAL', 'mean'),\n",
    "#                                             ('DAYS_REGISTRATION', 'mean')\n",
    "#                                             ]),\n",
    "#     (['CODE_GENDER', 'REG_CITY_NOT_WORK_CITY'], [\n",
    "#                                                  ('CNT_CHILDREN', 'mean')]),\n",
    " \n",
    "#     (['NAME_EDUCATION_TYPE', 'OCCUPATION_TYPE'], [('AMT_CREDIT', 'mean'),\n",
    "#                                                   ('AMT_REQ_CREDIT_BUREAU_YEAR', 'mean'),\n",
    "#                                                   ('APARTMENTS_AVG', 'mean'),\n",
    "#                                                   ('BASEMENTAREA_AVG', 'mean'),\n",
    "#                                                   ('NONLIVINGAREA_AVG', 'mean'),\n",
    "#                                                   ('OWN_CAR_AGE', 'mean'),\n",
    "#                                                   ('YEARS_BUILD_AVG', 'mean')]),\n",
    "#     (['NAME_EDUCATION_TYPE', 'OCCUPATION_TYPE', 'REG_CITY_NOT_WORK_CITY'], [('ELEVATORS_AVG', 'mean')\n",
    "#                                                                             ])\n",
    "# ]\n",
    "\n",
    "# groupby_aggregate_names = []\n",
    "# for groupby_cols, specs in tqdm(AGGREGATION_RECIPIES):\n",
    "#     group_object = X.groupby(groupby_cols)\n",
    "#     for select, agg in tqdm(specs):\n",
    "#         groupby_aggregate_name = '{}_{}_{}'.format('_'.join(groupby_cols), agg, select)\n",
    "#         X = X.merge(group_object[select]\n",
    "#                               .agg(agg)\n",
    "#                               .reset_index()\n",
    "#                               .rename(index=str,\n",
    "#                                       columns={select: groupby_aggregate_name})\n",
    "#                               [groupby_cols + [groupby_aggregate_name]],\n",
    "#                               on=groupby_cols,\n",
    "#                               how='left')\n",
    "#         groupby_aggregate_names.append(groupby_aggregate_name)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# X_agg = X[groupby_aggregate_names + ['TARGET']]\n",
    "# X_agg_corr = abs(X_agg.corr())\n",
    "# X_agg_corr.sort_values('TARGET', ascending=False)['TARGET']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X=X.drop(['AMT_ANNUITY','OWN_CAR_AGE','AMT_INCOME_TOTAL','DAYS_REGISTRATION','CNT_CHILDREN','AMT_CREDIT',\n",
    "#           'AMT_REQ_CREDIT_BUREAU_YEAR','APARTMENTS_AVG','BASEMENTAREA_AVG','NONLIVINGAREA_AVG',\n",
    "#           'YEARS_BUILD_AVG','ELEVATORS_AVG'\n",
    "#          ],axis=1)\n",
    "# #Dropped 12 columns and replaced them with their corresponding aggregated columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3 columns were label encoded.\n"
     ]
    }
   ],
   "source": [
    "\n",
    "le = LabelEncoder()\n",
    "le_count = 0\n",
    "\n",
    "# Iterate through the columns\n",
    "for col in X:\n",
    "    if X[col].dtype == 'object':\n",
    "        # If 2 or fewer unique categories\n",
    "        if len(list(X[col].unique())) <= 2:\n",
    "            # Train on the training data\n",
    "            le.fit(X[col])\n",
    "            # Transform both training and testing data\n",
    "            X[col] = le.transform(X[col])\n",
    "           # df_test[col] = le.transform(df_test[col])\n",
    "            \n",
    "            # Keep track of how many columns were label encoded\n",
    "            le_count += 1\n",
    "            \n",
    "print('%d columns were label encoded.' % le_count)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Training Features shape:  (307511, 242)\n",
      "Index(['SK_ID_CURR', 'TARGET', 'NAME_CONTRACT_TYPE', 'FLAG_OWN_CAR',\n",
      "       'FLAG_OWN_REALTY', 'CNT_CHILDREN', 'AMT_INCOME_TOTAL', 'AMT_CREDIT',\n",
      "       'AMT_ANNUITY', 'AMT_GOODS_PRICE',\n",
      "       ...\n",
      "       'HOUSETYPE_MODE_terraced house', 'WALLSMATERIAL_MODE_Block',\n",
      "       'WALLSMATERIAL_MODE_Mixed', 'WALLSMATERIAL_MODE_Monolithic',\n",
      "       'WALLSMATERIAL_MODE_Others', 'WALLSMATERIAL_MODE_Panel',\n",
      "       'WALLSMATERIAL_MODE_Stone, brick', 'WALLSMATERIAL_MODE_Wooden',\n",
      "       'EMERGENCYSTATE_MODE_No', 'EMERGENCYSTATE_MODE_Yes'],\n",
      "      dtype='object', length=242)\n"
     ]
    }
   ],
   "source": [
    "# one-hot encoding of categorical variables\n",
    "X = pd.get_dummies(X)\n",
    "#df_test = pd.get_dummies(df_test)\n",
    "list1=X.columns\n",
    "\n",
    "print('Training Features shape: ', X.shape)\n",
    "#print('Testing Features shape: ', df_test.shape)\n",
    "print(list1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to calculate missing values by column# Funct \n",
    "def missing_values_table(df):\n",
    "        # Total missing values\n",
    "        mis_val = df.isnull().sum()\n",
    "        \n",
    "        # Percentage of missing values\n",
    "        mis_val_percent = 100 * df.isnull().sum() / len(df)\n",
    "        \n",
    "        # Make a table with the results\n",
    "        mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)\n",
    "        # Rename the columns\n",
    "        mis_val_table_ren_columns = mis_val_table.rename(\n",
    "        columns = {0 : 'Missing Values', 1 : '% of Total Values'})\n",
    "        \n",
    "        # Sort the table by percentage of missing descending\n",
    "        mis_val_table_ren_columns = mis_val_table_ren_columns[\n",
    "            mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(\n",
    "        '% of Total Values', ascending=False).round(1)\n",
    "        \n",
    "        # Print some summary information\n",
    "        print (\"Your selected dataframe has \" + str(df.shape[1]) + \" columns.\\n\"      \n",
    "            \"There are \" + str(mis_val_table_ren_columns.shape[0]) +\n",
    "              \" columns that have missing values.\")\n",
    "        \n",
    "        # Return the dataframe with missing information\n",
    "        return mis_val_table_ren_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Your selected dataframe has 242 columns.\n",
      "There are 62 columns that have missing values.\n"
     ]
    }
   ],
   "source": [
    "# Missing values statistics\n",
    "missing_values = missing_values_table(X)\n",
    "mis_col=list(missing_values.index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #Have commented the dropping (not dropping any columns for now) Replace missing with -999\n",
    "# Mis=X[mis_col][:]\n",
    "# X_mis=(Mis.isnull().sum() / Mis.shape[0] * 100.00)\n",
    "# df_mis=pd.DataFrame(np.c_[X_mis.index,X_mis],columns=['Colname','miss_p'])\n",
    "# drop_col=df_mis[df_mis['miss_p']>65]['Colname'].values\n",
    "# print('number of columns before dropping->',len(X.columns))\n",
    "# #X=X.drop(drop_col,axis=1)\n",
    "# print('number of columns after dropping->',len(X.columns))\n",
    "X.replace(np.nan,-999, inplace=True)\n",
    "#X.astype('float32',inplace=True)\n",
    "#print(newX[].describe())\n",
    "y=X['TARGET']\n",
    "X=X.drop('TARGET',axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iterate below functions only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "def split_train_test(X , y):\n",
    "    return(train_test_split( X , y , test_size=0.2, random_state=42))\n",
    "\n",
    "X_train, X_test, y_train, y_test =split_train_test(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Improved number of features-> 83\n"
     ]
    }
   ],
   "source": [
    "def feature_eng(X_train,y_train,X_test,y_test):\n",
    "    train_col=list(X_train.columns)\n",
    "    test_col=list(X_test.columns)    \n",
    "    clf = ExtraTreesClassifier(n_estimators=50)\n",
    "    clf = clf.fit(X_train, y_train)\n",
    "    model = SelectFromModel(clf, prefit=True)\n",
    "    new_trainX = model.transform(X_train)\n",
    "    n=np.shape(new_trainX)[1]\n",
    "    importances = clf.feature_importances_\n",
    "    indices = np.argsort(importances)[::-1]\n",
    "    clist=[]\n",
    "    for f in range(n):     \n",
    "        clist.append(indices[f])\n",
    "    new_train_col=[train_col[i] for i in clist]\n",
    "    new_test_col=[test_col[i] for i in clist]\n",
    "    X_train=pd.DataFrame(X_train)\n",
    "    X_test=pd.DataFrame(X_test)\n",
    "    y_train=pd.DataFrame(y_train)\n",
    "    y_test=pd.DataFrame(y_test)\n",
    "    df_new_train=pd.DataFrame(X_train.loc[:,new_train_col],columns=new_train_col)            \n",
    "    df_new_test=pd.DataFrame(X_test.loc[:,new_test_col],columns=new_test_col)\n",
    "    print(\"Improved number of features->\",len(clist))    \n",
    "    return(df_new_train,df_new_test)\n",
    "\n",
    "df_new_train,df_new_test=feature_eng(X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_model_eval_metric(modelname,modelobj,X_train,Y_train,X_test,Y_test):\n",
    "    # In order to let the models run faster we use dask dataframe\n",
    "    \n",
    "    modelobj.fit(X_train,Y_train)\n",
    "    ypred_train=modelobj.predict(X_train)\n",
    "    ypred_test=modelobj.predict(X_test)\n",
    "    y_train_proba = modelobj.predict_proba(X_train)[:,1]\n",
    "    y_test_proba = modelobj.predict_proba(X_test)[:,1]\n",
    "    print('\\n')\n",
    "    print(modelname,\"model:\")\n",
    "    confmat=confusion_matrix(Y_train,ypred_train)\n",
    "    print(\"Train Confusion Matrix:\")\n",
    "    print(confmat)\n",
    "    confmat=confusion_matrix(Y_test,ypred_test)\n",
    "    print(\"Test Confusion Matrix:\")\n",
    "    print(confmat)\n",
    "    metric=pd.DataFrame([[0]*5]*2,columns=['PRECISION','RECALL','F1_SCORE','ROC_AUC_SCORE','ACCURACY'],index=['Train','Test'],dtype='float32')\n",
    "    metric['PRECISION']['Train']=round(precision_score(Y_train, ypred_train),3)\n",
    "    metric['RECALL']['Train']=recall_score(Y_train, ypred_train)\n",
    "    metric['F1_SCORE']['Train']=round(f1_score(Y_train, ypred_train),3)\n",
    "    metric['ROC_AUC_SCORE']['Train']=roc_auc_score(Y_train,y_train_proba)\n",
    "    metric['ACCURACY']['Train']=accuracy_score(Y_train,ypred_train)\n",
    "    metric['PRECISION']['Test']=precision_score(Y_test, ypred_test)\n",
    "    metric['RECALL']['Test']=recall_score(Y_test, ypred_test)\n",
    "    metric['F1_SCORE']['Test']=f1_score(Y_test, ypred_test)\n",
    "    metric['ROC_AUC_SCORE']['Test']=roc_auc_score(Y_test,y_test_proba)\n",
    "    metric['ACCURACY']['Test']=accuracy_score(Y_test,ypred_test)\n",
    "    print(metric)\n",
    "   \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "K-Nearest Neighbor 1 model:\n",
      "Train Confusion Matrix:\n",
      "[[225409    723]\n",
      " [ 18724   1152]]\n",
      "Test Confusion Matrix:\n",
      "[[56177   377]\n",
      " [ 4890    59]]\n",
      "       PRECISION    RECALL  F1_SCORE  ROC_AUC_SCORE  ACCURACY\n",
      "Train   0.614000  0.057959  0.106000       0.887909  0.920950\n",
      "Test    0.135321  0.011922  0.021913       0.541189  0.914362\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\ensemble\\forest.py:248: FutureWarning: The default value of n_estimators will change from 10 in version 0.20 to 100 in 0.22.\n",
      "  \"10 in version 0.20 to 100 in 0.22.\", FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Random Forest model:\n",
      "Train Confusion Matrix:\n",
      "[[226129      3]\n",
      " [  3626  16250]]\n",
      "Test Confusion Matrix:\n",
      "[[56457    97]\n",
      " [ 4916    33]]\n",
      "       PRECISION    RECALL  F1_SCORE  ROC_AUC_SCORE  ACCURACY\n",
      "Train   1.000000  0.817569  0.900000       0.999822  0.985248\n",
      "Test    0.253846  0.006668  0.012995       0.627320  0.918492\n",
      "\n",
      "\n",
      "Decision Tree model:\n",
      "Train Confusion Matrix:\n",
      "[[226132      0]\n",
      " [     0  19876]]\n",
      "Test Confusion Matrix:\n",
      "[[51488  5066]\n",
      " [ 4091   858]]\n",
      "       PRECISION    RECALL  F1_SCORE  ROC_AUC_SCORE  ACCURACY\n",
      "Train   1.000000  1.000000  1.000000       1.000000  1.000000\n",
      "Test    0.144835  0.173368  0.157822       0.541895  0.851113\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\linear_model\\logistic.py:432: FutureWarning: Default solver will be changed to 'lbfgs' in 0.22. Specify a solver to silence this warning.\n",
      "  FutureWarning)\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "\n",
      "Logistic Regression model:\n",
      "Train Confusion Matrix:\n",
      "[[226126      6]\n",
      " [ 19875      1]]\n",
      "Test Confusion Matrix:\n",
      "[[56552     2]\n",
      " [ 4949     0]]\n",
      "       PRECISION   RECALL  F1_SCORE  ROC_AUC_SCORE  ACCURACY\n",
      "Train      0.143  0.00005       0.0       0.635367  0.919186\n",
      "Test       0.000  0.00000       0.0       0.640536  0.919500\n"
     ]
    }
   ],
   "source": [
    "def run_all_models(df_new_train,y_train,df_new_test,y_test):\n",
    "    # K Nearest Neighbor\n",
    "    knearest = KNeighborsClassifier()\n",
    "    run_model_eval_metric('K-Nearest Neighbor ',knearest,df_new_train,y_train,df_new_test,y_test)\n",
    "    \n",
    "#     knearest = KNeighborsClassifier()\n",
    "#     run_model_eval_metric('K-Nearest Neighbor 3',knearest,df_new_train,y_train,df_new_test,y_test)\n",
    "\n",
    "    # Random Forest\n",
    "    randfor = RandomForestClassifier()\n",
    "    run_model_eval_metric('Random Forest',randfor,df_new_train,y_train,df_new_test,y_test)\n",
    "\n",
    "    # Decision Trees\n",
    "    dectree = DecisionTreeClassifier()\n",
    "    run_model_eval_metric('Decision Tree',dectree,df_new_train,y_train,df_new_test,y_test)\n",
    "\n",
    "    # Logistic Regression\n",
    "    logreg = LogisticRegression()\n",
    "    run_model_eval_metric('Logistic Regression',logreg,df_new_train,y_train,df_new_test,y_test)\n",
    "\n",
    "run_all_models(df_new_train,y_train,df_new_test,y_test)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #Run Base Models\n",
    "# X_train, X_test, y_train, y_test =split_train_test(X,y)\n",
    "# df_new_train,df_new_test=feature_eng(X_train, y_train, X_test, y_test)\n",
    "# df_new_train=dd.from_pandas(df_new_train,npartitions=3)\n",
    "# df_new_test=dd.from_pandas(df_new_test,npartitions=3)\n",
    "# y_train=dd.from_pandas(y_train,npartitions=3)\n",
    "# y_test=dd.from_pandas(y_test,npartitions=3)\n",
    "# run_all_models(df_new_train,y_train,df_new_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #Handling Imbalance problem\n",
    "# X_tr, X_test, y_tr, y_test =split_train_test(X,y)\n",
    "# '''\n",
    "# Use SMOTE to handle imbalance problem\n",
    "# '''\n",
    "# # Class0 size is 10 times that of Class1 in train set\n",
    "# print('Y train before resampling',Counter(y_tr))\n",
    "# Imb_sm = SMOTE(random_state=42)\n",
    "# X_train, y_train = Imb_sm.fit_resample(X_tr, y_tr)\n",
    "# print('Y train after resampling',Counter(y_train))\n",
    "# X_train=pd.DataFrame(X_train)\n",
    "# y_train=pd.DataFrame(y_train)\n",
    "# df_new_train,df_new_test=feature_eng(X_train, y_train, X_test, y_test)\n",
    "# df_new_train=dd.from_pandas(df_new_train,npartitions=3)\n",
    "# df_new_test=dd.from_pandas(df_new_test,npartitions=3)\n",
    "# y_train=dd.from_pandas(y_train,npartitions=3)\n",
    "# y_test=dd.from_pandas(y_test,npartitions=3)\n",
    "# run_all_models(df_new_train,y_train,df_new_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}