LightGBM before Smote and after Smote on imputed csv.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lightgbm import LGBMClassifier\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import roc_auc_score, roc_curve\n",
    "from sklearn.model_selection import KFold, StratifiedKFold\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from xgboost import XGBClassifier\n",
    "from sklearn.model_selection import cross_val_score, StratifiedKFold,train_test_split\n",
    "from sklearn.metrics import classification_report,confusion_matrix\n",
    "from sklearn.metrics import roc_auc_score\n",
    "from imblearn.over_sampling import SMOTE"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "(307511, 247)\n",
      "(307511, 252)\n"
     ]
    }
   ],
   "source": [
    "app_train = pd.read_csv('application_train_imputed.csv')\n",
    "print(app_train.shape)\n",
    "app_train['DAYS_EMPLOYED'].replace(365243, np.nan, inplace= True)\n",
    "    # Some simple new features (percentages)\n",
    "app_train['DAYS_EMPLOYED_PERC'] = app_train['DAYS_EMPLOYED'] / app_train['DAYS_BIRTH']\n",
    "app_train['INCOME_CREDIT_PERC'] = app_train['AMT_INCOME_TOTAL'] / app_train['AMT_CREDIT']\n",
    "app_train['INCOME_PER_PERSON'] = app_train['AMT_INCOME_TOTAL'] / app_train['CNT_FAM_MEMBERS']\n",
    "app_train['ANNUITY_INCOME_PERC'] = app_train['AMT_ANNUITY'] / app_train['AMT_INCOME_TOTAL']\n",
    "app_train['PAYMENT_RATE'] = app_train['AMT_ANNUITY'] / app_train['AMT_CREDIT']\n",
    "\n",
    "print(app_train.shape)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "y = app_train['TARGET']\n",
    "X=app_train.drop(['TARGET'],axis=1)\n",
    "# X.drop(columns=['Unnamed: 0'],axis=1,inplace=True)\n",
    "# Remove above line of Unnamed: 0 after making changes to application_train_imputed.csv"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Imputed csv + self added 5 cols "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Postive examples in train set: 226148\n",
      "Negative examples in train set: 19860\n",
      "Postive examples in test set: 56538\n",
      "Negative examples in test set: 4965\n"
     ]
    }
   ],
   "source": [
    "X_train_later, X_test, y_train_later, y_test = train_test_split(X, y, test_size=0.2, stratify=y, random_state=1)\n",
    "print(\"Postive examples in train set: {}\".format(np.sum(y_train_later==0)))\n",
    "print(\"Negative examples in train set: {}\".format(np.sum(y_train_later==1)))\n",
    "\n",
    "print(\"Postive examples in test set: {}\".format(np.sum(y_test==0)))\n",
    "print(\"Negative examples in test set: {}\".format(np.sum(y_test==1)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "# X_train, X_val, y_train, y_val = train_test_split(X_train_later, y_train_later, test_size=0.2, stratify=y_train, random_state=1)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf11 = LGBMClassifier(\n",
    "            nthread=4,\n",
    "            n_estimators=10000,\n",
    "            learning_rate=0.02,\n",
    "            num_leaves=34,\n",
    "            colsample_bytree=0.9497036,\n",
    "            subsample=0.8715623,\n",
    "            max_depth=8,\n",
    "            reg_alpha=0.041545473,\n",
    "            reg_lambda=0.0735294,\n",
    "            min_split_gain=0.0222415,\n",
    "            min_child_weight=39.3259775,\n",
    "            silent=-1,\n",
    "            verbose=-1, )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "clf11.fit(X_train_later,y_train_later)\n",
    "y_pred=clf11.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    61012\n",
       "1.0      491\n",
       "Name: 0, dtype: int64"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newdf=pd.DataFrame(data=y_pred)\n",
    "newdf.iloc[:,0].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.92      1.00      0.96     56538\n",
      "        1.0       0.46      0.05      0.08      4965\n",
      "\n",
      "avg / total       0.89      0.92      0.89     61503\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[56273,   265],\n",
       "       [ 4739,   226]], dtype=int64)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# With SMOTE + above all"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Postive examples in train set: 226148\n",
      "Negative examples in train set: 19860\n",
      "Postive examples in test set: 56538\n",
      "Negative examples in test set: 4965\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\utils\\deprecation.py:77: DeprecationWarning: Function _ratio_float is deprecated; Use a float for 'ratio' is deprecated from version 0.2. The support will be removed in 0.4. Use a dict, str, or a callable instead.\n",
      "  warnings.warn(msg, category=DeprecationWarning)\n"
     ]
    }
   ],
   "source": [
    "X_train_later, X_test, y_train_later, y_test = train_test_split(X, y, test_size=0.2, stratify=y, random_state=1)\n",
    "print(\"Postive examples in train set: {}\".format(np.sum(y_train_later==0)))\n",
    "print(\"Negative examples in train set: {}\".format(np.sum(y_train_later==1)))\n",
    "\n",
    "print(\"Postive examples in test set: {}\".format(np.sum(y_test==0)))\n",
    "print(\"Negative examples in test set: {}\".format(np.sum(y_test==1)))\n",
    "\n",
    "sm = SMOTE(random_state=12, ratio = 1.0)\n",
    "X_train_res, y_train_res = sm.fit_sample(X_train_later, y_train_later)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "clf12 = LGBMClassifier(\n",
    "            nthread=4,\n",
    "            n_estimators=100,\n",
    "            learning_rate=0.2,\n",
    "            num_leaves=300,\n",
    "            colsample_bytree=0.9497036,\n",
    "            subsample=0.8715623,\n",
    "            max_depth=8,\n",
    "            reg_alpha=0.041545473,\n",
    "            reg_lambda=0.0735294,\n",
    "            min_split_gain=0.0222415,\n",
    "            min_child_weight=39.3259775,\n",
    "            silent=-1,\n",
    "            verbose=-1, )"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### After smote size of train set : "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Postive examples in train set: 226148\n",
      "Negative examples in train set: 226148\n"
     ]
    }
   ],
   "source": [
    "print(\"Postive examples in train set: {}\".format(np.sum(y_train_res==0)))\n",
    "print(\"Negative examples in train set: {}\".format(np.sum(y_train_res==1)))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\ProgramData\\Anaconda3\\lib\\site-packages\\sklearn\\preprocessing\\label.py:151: DeprecationWarning: The truth value of an empty array is ambiguous. Returning False, but in future this will result in an error. Use `array.size > 0` to check that an array is not empty.\n",
      "  if diff:\n"
     ]
    }
   ],
   "source": [
    "clf12.fit(X_train_res,y_train_res)\n",
    "y_pred=clf12.predict(X_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.0    61174\n",
       "1.0      329\n",
       "Name: 0, dtype: int64"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "newdf=pd.DataFrame(data=y_pred)\n",
    "newdf.iloc[:,0].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "             precision    recall  f1-score   support\n",
      "\n",
      "        0.0       0.92      1.00      0.96     56538\n",
      "        1.0       0.52      0.03      0.06      4965\n",
      "\n",
      "avg / total       0.89      0.92      0.89     61503\n",
      "\n"
     ]
    }
   ],
   "source": [
    "print(classification_report(y_test, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[56379,   159],\n",
       "       [ 4795,   170]], dtype=int64)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "confusion_matrix(y_test, y_pred)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.0"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}