HOME_CREDIT_misMean01-normalised_standardscalar.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# For Preprocessing\n",
    "from sklearn import preprocessing\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "import pandas as pd\n",
    "from sklearn.linear_model import LogisticRegression\n",
    "from sklearn.neighbors import KNeighborsClassifier\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.svm import SVC\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.preprocessing import StandardScaler\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.metrics import accuracy_score\n",
    "from xgboost import plot_importance\n",
    "from sklearn.metrics import confusion_matrix\n",
    "import numpy as np\n",
    "from collections import Counter\n",
    "from sklearn.ensemble import ExtraTreesClassifier\n",
    "from xgboost import XGBClassifier\n",
    "import xgboost as xgb\n",
    "from sklearn.tree import DecisionTreeClassifier\n",
    "import os\n",
    "from tqdm import tqdm_notebook as tqdm\n",
    "from sklearn.externals import joblib\n",
    "%matplotlib inline\n",
    "import seaborn as sns\n",
    "import dask.dataframe as dd\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from imblearn.over_sampling import SMOTE\n",
    "from sklearn.metrics import classification_report\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from sklearn.metrics import f1_score,recall_score,precision_score"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = pd.read_csv(\"application_train.csv\")\n",
    "df_test = pd.read_csv(\"application_test.csv\")\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X['CODE_GENDER'].replace('XNA',np.nan, inplace=True)\n",
    "X['CODE_GENDER'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X['DAYS_LAST_PHONE_CHANGE'].replace(0, np.nan, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X['DAYS_EMPLOYED'].replace({365243: np.nan}, inplace = True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "le = LabelEncoder()\n",
    "le_count = 0\n",
    "\n",
    "# Iterate through the columns\n",
    "for col in X:\n",
    "    if X[col].dtype == 'object':\n",
    "        # If 2 or fewer unique categories\n",
    "        if len(list(X[col].unique())) <= 2:\n",
    "            # Train on the training data\n",
    "            le.fit(X[col])\n",
    "            # Transform both training and testing data\n",
    "            X[col] = le.transform(X[col])\n",
    "           # df_test[col] = le.transform(df_test[col])\n",
    "            \n",
    "            # Keep track of how many columns were label encoded\n",
    "            le_count += 1\n",
    "            \n",
    "print('%d columns were label encoded.' % le_count)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# one-hot encoding of categorical variables\n",
    "X = pd.get_dummies(X)\n",
    "#df_test = pd.get_dummies(df_test)\n",
    "list1=X.columns\n",
    "\n",
    "print('Training Features shape: ', X.shape)\n",
    "#print('Testing Features shape: ', df_test.shape)\n",
    "print(list1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Function to calculate missing values by column# Funct \n",
    "def missing_values_table(df):\n",
    "        # Total missing values\n",
    "        mis_val = df.isnull().sum()\n",
    "        \n",
    "        # Percentage of missing values\n",
    "        mis_val_percent = 100 * df.isnull().sum() / len(df)\n",
    "        \n",
    "        # Make a table with the results\n",
    "        mis_val_table = pd.concat([mis_val, mis_val_percent], axis=1)\n",
    "        # Rename the columns\n",
    "        mis_val_table_ren_columns = mis_val_table.rename(\n",
    "        columns = {0 : 'Missing Values', 1 : '% of Total Values'})\n",
    "        \n",
    "        # Sort the table by percentage of missing descending\n",
    "        mis_val_table_ren_columns = mis_val_table_ren_columns[\n",
    "            mis_val_table_ren_columns.iloc[:,1] != 0].sort_values(\n",
    "        '% of Total Values', ascending=False).round(1)\n",
    "        \n",
    "        # Print some summary information\n",
    "        print (\"Your selected dataframe has \" + str(df.shape[1]) + \" columns.\\n\"      \n",
    "            \"There are \" + str(mis_val_table_ren_columns.shape[0]) +\n",
    "              \" columns that have missing values.\")\n",
    "        \n",
    "        # Return the dataframe with missing information\n",
    "        return mis_val_table_ren_columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Missing values statistics\n",
    "missing_values = missing_values_table(X)\n",
    "mis_col=list(missing_values.index)\n",
    "Mis=X[mis_col][:]\n",
    "\n",
    "for col in mis_col:\n",
    "    X[col+'_miss']=[0]*len(X[col])\n",
    "    X[col+'_miss'][list(X[np.isnan(X[col])].index)]=1\n",
    "\n",
    "print(X.describe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.base import TransformerMixin\n",
    "\n",
    "class DataFrameImputer(TransformerMixin):\n",
    "\n",
    "    def __init__(self):\n",
    "        \"\"\"Impute missing values.\n",
    "\n",
    "        Columns of dtype object are imputed with the most frequent value \n",
    "        in column.\n",
    "\n",
    "        Columns of other types are imputed with mean of column.\n",
    "\n",
    "        \"\"\"\n",
    "    def fit(self, X, y=None):\n",
    "\n",
    "        self.fill = pd.Series([X[c].value_counts().index[0]\n",
    "            if X[c].dtype == np.dtype('O') else X[c].mean() for c in X],\n",
    "            index=X.columns)\n",
    "\n",
    "        return self\n",
    "\n",
    "    def transform(self, X, y=None):\n",
    "        return X.fillna(self.fill)\n",
    "    \n",
    "print('before...')    \n",
    "Mis=X[mis_col][:]\n",
    "print('Mis - ',Mis)\n",
    "\n",
    "xt = DataFrameImputer().fit_transform(Mis)\n",
    "\n",
    "print('after...')\n",
    "print(xt)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# #Have commented the dropping (not dropping any columns for now) Replace missing with -999\n",
    "\n",
    "# X_mis=(Mis.isnull().sum() / Mis.shape[0] * 100.00)\n",
    "# df_mis=pd.DataFrame(np.c_[X_mis.index,X_mis],columns=['Colname','miss_p'])\n",
    "# drop_col=df_mis[df_mis['miss_p']>65]['Colname'].values\n",
    "# print('number of columns before dropping->',len(X.columns))\n",
    "# #X=X.drop(drop_col,axis=1)\n",
    "# print('number of columns after dropping->',len(X.columns))\n",
    "# X.replace(np.nan,-999, inplace=True)\n",
    "# #X.astype('float32',inplace=True)\n",
    "#print(newX[].describe())\n",
    "for mis in mis_col:\n",
    "    X[mis]=xt[mis]\n",
    "print(X.shape)\n",
    "print(X.describe)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# iterate below functions only"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "y=X['TARGET']\n",
    "X=X.drop('TARGET',axis=1)\n",
    "\n",
    "def split_train_test(X , y):\n",
    "    return(train_test_split( X , y , test_size=0.2, random_state=42))\n",
    "\n",
    "X_train, X_test, y_train, y_test =split_train_test(X,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def feature_eng(X_train,y_train,X_test,y_test):\n",
    "    train_col=list(X_train.columns)\n",
    "    test_col=list(X_test.columns)    \n",
    "    clf = ExtraTreesClassifier(n_estimators=50)\n",
    "    clf = clf.fit(X_train, y_train)\n",
    "    model = SelectFromModel(clf, prefit=True)\n",
    "    new_trainX = model.transform(X_train)\n",
    "    n=np.shape(new_trainX)[1]\n",
    "    importances = clf.feature_importances_\n",
    "    indices = np.argsort(importances)[::-1]\n",
    "    clist=[]\n",
    "    for f in range(n):     \n",
    "        clist.append(indices[f])\n",
    "    new_train_col=[train_col[i] for i in clist]\n",
    "    new_test_col=[test_col[i] for i in clist]\n",
    "    X_train=pd.DataFrame(X_train)\n",
    "    X_test=pd.DataFrame(X_test)\n",
    "    y_train=pd.DataFrame(y_train)\n",
    "    y_test=pd.DataFrame(y_test)\n",
    "    df_new_train=pd.DataFrame(X_train.loc[:,new_train_col],columns=new_train_col)            \n",
    "    df_new_test=pd.DataFrame(X_test.loc[:,new_test_col],columns=new_test_col)\n",
    "    print(\"Improved number of features->\",len(clist))    \n",
    "    return(df_new_train,df_new_test)\n",
    "\n",
    "df_new_train,df_new_test=feature_eng(X_train, y_train, X_test, y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "scaler1 = StandardScaler().fit(df_new_train)\n",
    "df_new_train = scaler1.transform(df_new_train)\n",
    "\n",
    "\n",
    "scaler2 = StandardScaler().fit(df_new_test)\n",
    "df_new_test = scaler2.transform(df_new_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_model_eval_metric(modelname,modelobj,X_train,Y_train,X_test,Y_test):\n",
    "    # In order to let the models run faster we use dask dataframe\n",
    "#     X_train=dd.from_pandas(X_train,npartitions=3)\n",
    "#     X_test=dd.from_pandas(X_test,npartitions=3)\n",
    "#     Y_train=dd.from_pandas(Y_train,npartitions=3)\n",
    "#     Y_test=dd.from_pandas(Y_test,npartitions=3)\n",
    "\n",
    "    modelobj.fit(X_train,Y_train)\n",
    "    ypred_train=modelobj.predict(X_train)\n",
    "    ypred_test=modelobj.predict(X_test)\n",
    "    y_train_proba = modelobj.predict_proba(X_train)[:,1]\n",
    "    y_test_proba = modelobj.predict_proba(X_test)[:,1]\n",
    "    print(modelname,\"model:\")\n",
    "    confmat=confusion_matrix(Y_train,ypred_train)\n",
    "    print(\"Train Confusion Matrix:\")\n",
    "    print(confmat)\n",
    "    confmat=confusion_matrix(Y_test,ypred_test)\n",
    "    print(\"Test Confusion Matrix:\")\n",
    "    print(confmat)\n",
    "    metric=pd.DataFrame([[0]*5]*2,columns=['PRECISION','RECALL','F1_SCORE','ROC_AUC_SCORE','ACCURACY'],index=['Train','Test'],dtype='float32')\n",
    "    metric['PRECISION']['Train']=precision_score(Y_train, ypred_train)\n",
    "    metric['RECALL']['Train']=recall_score(Y_train, ypred_train)\n",
    "    metric['F1_SCORE']['Train']=f1_score(Y_train, ypred_train)\n",
    "    metric['ROC_AUC_SCORE']['Train']=roc_auc_score(Y_train,y_train_proba)\n",
    "    metric['ACCURACY']['Train']=accuracy_score(Y_train,ypred_train)\n",
    "    metric['PRECISION']['Test']=precision_score(Y_test, ypred_test)\n",
    "    metric['RECALL']['Test']=recall_score(Y_test, ypred_test)\n",
    "    metric['F1_SCORE']['Test']=f1_score(Y_test, ypred_test)\n",
    "    metric['ROC_AUC_SCORE']['Test']=roc_auc_score(Y_test,y_test_proba)\n",
    "    metric['ACCURACY']['Test']=accuracy_score(Y_test,ypred_test)\n",
    "    print(metric)\n",
    "   \n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "def run_all_models(df_new_train,y_train,df_new_test,y_test):\n",
    "    # K Nearest Neighbor\n",
    "    knearest = KNeighborsClassifier(algorithm= 'auto', n_jobs= -1, n_neighbors= 1)\n",
    "    run_model_eval_metric('K-Nearest Neighbor 1',knearest,df_new_train,y_train,df_new_test,y_test)\n",
    "    \n",
    "    knearest = KNeighborsClassifier(algorithm= 'auto', n_jobs= -1, n_neighbors= 3)\n",
    "    run_model_eval_metric('K-Nearest Neighbor 3',knearest,df_new_train,y_train,df_new_test,y_test)\n",
    "\n",
    "    # Random Forest\n",
    "    randfor = RandomForestClassifier(n_estimators= 5, max_features= 'auto', max_depth= 70, bootstrap= True, n_jobs= -1)\n",
    "    run_model_eval_metric('Random Forest',randfor,df_new_train,y_train,df_new_test,y_test)\n",
    "\n",
    "#     # Decision Trees\n",
    "#     dectree = DecisionTreeClassifier(criterion=\"gini\",max_depth=15,min_samples_leaf=25)\n",
    "#     run_model_eval_metric('Decision Tree',dectree,df_new_train,y_train,df_new_test,y_test)\n",
    "\n",
    "#     # Logistic Regression\n",
    "#     logreg = LogisticRegression()\n",
    "#     run_model_eval_metric('Logistic Regression',logreg,df_new_train,y_train,df_new_test,y_test)\n",
    "\n",
    "#     #SVM\n",
    "#     svm = SVC()\n",
    "#     svm_ypred_train,svm_ypred_test=run_model_eval_metric('SVM',svm,df_new_train,y_train,df_new_test,y_test)\n",
    "\n",
    "run_all_models(df_new_train,y_train,df_new_test,y_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}