Titanic Data Cleaning,Making ready for model.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "import numpy as np\n",
    "from sklearn.preprocessing import OneHotEncoder,LabelEncoder\n",
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "from sklearn.ensemble.gradient_boosting import GradientBoostingClassifier\n",
    "from sklearn.feature_selection import SelectKBest\n",
    "from sklearn.model_selection import StratifiedKFold\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.model_selection import cross_val_score\n",
    "from sklearn.feature_selection import SelectFromModel\n",
    "from sklearn.linear_model import LogisticRegression, LogisticRegressionCV\n",
    "%matplotlib inline\n",
    "import warnings\n",
    "warnings.filterwarnings('ignore')\n",
    "warnings.filterwarnings('ignore', category=DeprecationWarning)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Loading the Data,Combining and removing unnecessary columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = pd.read_csv('train.csv')\n",
    "test = pd.read_csv('test.csv')\n",
    "\n",
    "# extracting and then removing the targets from the training data \n",
    "targets = train.Survived\n",
    "train.drop(['Survived'], 1, inplace=True)\n",
    "    \n",
    "\n",
    "# merging train data and test data for future feature engineering\n",
    "# we'll also remove the PassengerID since this is not an informative feature\n",
    "combined = train.append(test)\n",
    "combined.reset_index(inplace=True)\n",
    "combined.drop(['index', 'PassengerId'], inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(891,)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "targets.shape"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Processing Name to Generate Titles"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.Series.str.split.html\n",
    "<br>\n",
    "Description of pandas.series.str.split()\n",
    "<br>\n",
    "This method will be used to scrape the titles from Name column and to process the name column\n",
    "<br>\n",
    "*combined['Title'] = combined['Name'].str.split(',',expand=True)[1].str.split('.',expand=True)[0]* -This code did not work when combining dataframes using append method"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "titles = set()\n",
    "for name in train['Name']:\n",
    "    titles.add(name.split(',')[1].split('.')[0].strip())"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'Miss', 'Don', 'Major', 'Mr', 'Dr', 'Ms', 'Sir', 'Mlle', 'the Countess', 'Col', 'Rev', 'Capt', 'Jonkheer', 'Mme', 'Mrs', 'Lady', 'Master'}\n"
     ]
    }
   ],
   "source": [
    "print(titles)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "As We can see there are a lot of titles ,we will reduce them to five categories.\n",
    "<br>\n",
    "- Mr \n",
    "<br> \n",
    "- Miss\n",
    "<br> \n",
    "- Mrs\n",
    "<br> \n",
    "- Master\n",
    "<br> \n",
    "- Officer\n",
    "<br> \n",
    "- Royalty"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "Title_Dictionary = {\n",
    "    \"Capt\": \"Royalty\",\n",
    "    \"Col\": \"Royalty\",\n",
    "    \"Major\": \"Royalty\",\n",
    "    \"Jonkheer\": \"Royalty\",\n",
    "    \"Don\": \"Royalty\",\n",
    "    \"Sir\" : \"Royalty\",\n",
    "    \"Dr\": \"Royalty\",\n",
    "    \"Rev\": \"Royalty\",\n",
    "    \"the Countess\":\"Royalty\",\n",
    "    \"Mme\": \"Mrs\",\n",
    "    \"Mlle\": \"Miss\",\n",
    "    \"Ms\": \"Mrs\",\n",
    "    \"Mr\" : \"Mr\",\n",
    "    \"Mrs\" : \"Mrs\",\n",
    "    \"Miss\" : \"Miss\",\n",
    "    \"Master\" : \"Master\",\n",
    "    \"Lady\" : \"Royalty\"\n",
    "}\n",
    "\n",
    "def get_titles():\n",
    "    # we extract the title from each name\n",
    "    combined['Title'] = combined['Name'].map(lambda name:name.split(',')[1].split('.')[0].strip())\n",
    "    \n",
    "    # a map of more aggregated title\n",
    "    # we map each title\n",
    "    combined['Title'] = combined.Title.map(Title_Dictionary)\n",
    "    return combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "Title_Dictionary = {\n",
    "    \"Capt\": \"Officer\",\n",
    "    \"Col\": \"Officer\",\n",
    "    \"Major\": \"Officer\",\n",
    "    \"Jonkheer\": \"Royalty\",\n",
    "    \"Don\": \"Royalty\",\n",
    "    \"Sir\" : \"Royalty\",\n",
    "    \"Dr\": \"Officer\",\n",
    "    \"Rev\": \"Officer\",\n",
    "    \"the Countess\":\"Royalty\",\n",
    "    \"Mme\": \"Mrs\",\n",
    "    \"Mlle\": \"Miss\",\n",
    "    \"Ms\": \"Mrs\",\n",
    "    \"Mr\" : \"Mr\",\n",
    "    \"Mrs\" : \"Mrs\",\n",
    "    \"Miss\" : \"Miss\",\n",
    "    \"Master\" : \"Master\",\n",
    "    \"Lady\" : \"Royalty\"\n",
    "}\n",
    "\n",
    "def get_titles():\n",
    "    # we extract the title from each name\n",
    "    combined['Title'] = combined['Name'].map(lambda name:name.split(',')[1].split('.')[0].strip())\n",
    "    \n",
    "    # a map of more aggregated title\n",
    "    # we map each title\n",
    "    combined['Title'] = combined.Title.map(Title_Dictionary)\n",
    "    return combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined = get_titles()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "      <th>Title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1305</th>\n",
       "      <td>1</td>\n",
       "      <td>Oliva y Ocana, Dona. Fermina</td>\n",
       "      <td>female</td>\n",
       "      <td>39.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17758</td>\n",
       "      <td>108.9</td>\n",
       "      <td>C105</td>\n",
       "      <td>C</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Pclass                          Name     Sex   Age  SibSp  Parch  \\\n",
       "1305       1  Oliva y Ocana, Dona. Fermina  female  39.0      0      0   \n",
       "\n",
       "        Ticket   Fare Cabin Embarked Title  \n",
       "1305  PC 17758  108.9  C105        C   NaN  "
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined[combined['Title'].isnull()]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined['Title'].fillna('Royalty',inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>Sex</th>\n",
       "      <th>female</th>\n",
       "      <th>male</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Title</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>Master</th>\n",
       "      <td>0</td>\n",
       "      <td>61</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Miss</th>\n",
       "      <td>262</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mr</th>\n",
       "      <td>0</td>\n",
       "      <td>757</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Mrs</th>\n",
       "      <td>200</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Officer</th>\n",
       "      <td>1</td>\n",
       "      <td>22</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>Royalty</th>\n",
       "      <td>3</td>\n",
       "      <td>3</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "Sex      female  male\n",
       "Title                \n",
       "Master        0    61\n",
       "Miss        262     0\n",
       "Mr            0   757\n",
       "Mrs         200     0\n",
       "Officer       1    22\n",
       "Royalty       3     3"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pd.crosstab(combined['Title'],combined['Sex'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Processing Age"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "177\n"
     ]
    }
   ],
   "source": [
    "print(combined['Age'].iloc[:891].isnull().sum())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "**So we Have 177 null values in the train set which have to be imputed**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "mediantrain = combined.iloc[:891].groupby(['Sex','Pclass','Title']).median()['Age'].reset_index()[['Sex', 'Pclass', 'Title', 'Age']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sex</th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Title</th>\n",
       "      <th>Age</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>female</td>\n",
       "      <td>1</td>\n",
       "      <td>Miss</td>\n",
       "      <td>30.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>female</td>\n",
       "      <td>1</td>\n",
       "      <td>Mrs</td>\n",
       "      <td>40.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>female</td>\n",
       "      <td>1</td>\n",
       "      <td>Officer</td>\n",
       "      <td>49.0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>female</td>\n",
       "      <td>1</td>\n",
       "      <td>Royalty</td>\n",
       "      <td>40.5</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>female</td>\n",
       "      <td>2</td>\n",
       "      <td>Miss</td>\n",
       "      <td>24.0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "      Sex  Pclass    Title   Age\n",
       "0  female       1     Miss  30.0\n",
       "1  female       1      Mrs  40.0\n",
       "2  female       1  Officer  49.0\n",
       "3  female       1  Royalty  40.5\n",
       "4  female       2     Miss  24.0"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "mediantrain.head(5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This DataFrame will help us impute the age vaues\n",
    "<br>\n",
    "How to iterate through rows in a dataframe?\n",
    "<br>\n",
    "https://www.geeksforgeeks.org/different-ways-to-iterate-over-rows-in-pandas-dataframe/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Name</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Embarked</th>\n",
       "      <th>Title</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>30</th>\n",
       "      <td>1</td>\n",
       "      <td>Uruchurtu, Don. Manuel E</td>\n",
       "      <td>male</td>\n",
       "      <td>40.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17601</td>\n",
       "      <td>27.7208</td>\n",
       "      <td>NaN</td>\n",
       "      <td>C</td>\n",
       "      <td>Royalty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>599</th>\n",
       "      <td>1</td>\n",
       "      <td>Duff Gordon, Sir. Cosmo Edmund (\"Mr Morgan\")</td>\n",
       "      <td>male</td>\n",
       "      <td>49.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17485</td>\n",
       "      <td>56.9292</td>\n",
       "      <td>A20</td>\n",
       "      <td>C</td>\n",
       "      <td>Royalty</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>822</th>\n",
       "      <td>1</td>\n",
       "      <td>Reuchlin, Jonkheer. John George</td>\n",
       "      <td>male</td>\n",
       "      <td>38.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>19972</td>\n",
       "      <td>0.0000</td>\n",
       "      <td>NaN</td>\n",
       "      <td>S</td>\n",
       "      <td>Royalty</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "     Pclass                                          Name   Sex   Age  SibSp  \\\n",
       "30        1                      Uruchurtu, Don. Manuel E  male  40.0      0   \n",
       "599       1  Duff Gordon, Sir. Cosmo Edmund (\"Mr Morgan\")  male  49.0      1   \n",
       "822       1               Reuchlin, Jonkheer. John George  male  38.0      0   \n",
       "\n",
       "     Parch    Ticket     Fare Cabin Embarked    Title  \n",
       "30       0  PC 17601  27.7208   NaN        C  Royalty  \n",
       "599      0  PC 17485  56.9292   A20        C  Royalty  \n",
       "822      0     19972   0.0000   NaN        S  Royalty  "
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#example what the next block of code does\n",
    "combined[(combined['Sex']=='male')&(combined['Title']=='Royalty')&(combined['Pclass']==1)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "def fill_age(row):\n",
    "    condition = (\n",
    "        (mediantrain['Sex'] == row['Sex']) & \n",
    "        (mediantrain['Title'] == row['Title']) & \n",
    "        (mediantrain['Pclass'] == row['Pclass'])\n",
    "    ) \n",
    "    return mediantrain[condition]['Age'].values[0]\n",
    "\n",
    "\n",
    "def process_age():\n",
    "    global combined\n",
    "    # a function that fills the missing values of the Age variable\n",
    "    combined['Age'] = combined.apply(lambda row: fill_age(row) if np.isnan(row['Age']) else row['Age'], axis=1)\n",
    "    return combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined = process_age()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Processing Fares"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we simply fill the null values of fare with the mean in the train set and ignore the values which have zero fare for first iteration,After testing our model for the first time ,we will come back and check again if treating those values would make a difference"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined.Fare.fillna(combined.iloc[:891].Fare.mean(), inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 1309 entries, 0 to 1308\n",
      "Data columns (total 11 columns):\n",
      "Pclass      1309 non-null int64\n",
      "Name        1309 non-null object\n",
      "Sex         1309 non-null object\n",
      "Age         1309 non-null float64\n",
      "SibSp       1309 non-null int64\n",
      "Parch       1309 non-null int64\n",
      "Ticket      1309 non-null object\n",
      "Fare        1309 non-null float64\n",
      "Cabin       295 non-null object\n",
      "Embarked    1307 non-null object\n",
      "Title       1309 non-null object\n",
      "dtypes: float64(2), int64(3), object(6)\n",
      "memory usage: 112.6+ KB\n"
     ]
    }
   ],
   "source": [
    "combined.info()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Processing Titles"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "We will use pandas get dummies method to do one-hot encoding of Titles column and drop the names column\n",
    "<br> If you still want to use scikit learn's one hot method here is a very usefull link\n",
    "<br>http://www.insightsbot.com/blog/McTKK/python-one-hot-encoding-with-scikit-learn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_names():\n",
    "    global combined\n",
    "    # we clean the Name variable\n",
    "    combined.drop('Name', axis=1, inplace=True)\n",
    "    \n",
    "    # encoding in dummy variable\n",
    "    titles_dummies = pd.get_dummies(combined['Title'], prefix='Title')\n",
    "    combined = pd.concat([combined, titles_dummies], axis=1)\n",
    "    \n",
    "    # removing the title variable\n",
    "    combined.drop('Title', axis=1, inplace=True)\n",
    "    \n",
    "    return combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined = process_names()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Processing Embarked"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_embarked():\n",
    "    global combined\n",
    "    # two missing embarked values - filling them with the most frequent one in the train  set(S)\n",
    "    combined.Embarked.fillna('S', inplace=True)\n",
    "    # dummy encoding \n",
    "    embarked_dummies = pd.get_dummies(combined['Embarked'], prefix='Embarked')\n",
    "    combined = pd.concat([combined, embarked_dummies], axis=1)\n",
    "    combined.drop('Embarked', axis=1, inplace=True)\n",
    "    return combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined = process_embarked()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Title_Master</th>\n",
       "      <th>Title_Miss</th>\n",
       "      <th>Title_Mr</th>\n",
       "      <th>Title_Mrs</th>\n",
       "      <th>Title_Officer</th>\n",
       "      <th>Title_Royalty</th>\n",
       "      <th>Embarked_C</th>\n",
       "      <th>Embarked_Q</th>\n",
       "      <th>Embarked_S</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>22.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>38.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>female</td>\n",
       "      <td>26.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>35.0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>35.0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pclass     Sex   Age  SibSp  Parch            Ticket     Fare Cabin  \\\n",
       "0       3    male  22.0      1      0         A/5 21171   7.2500   NaN   \n",
       "1       1  female  38.0      1      0          PC 17599  71.2833   C85   \n",
       "2       3  female  26.0      0      0  STON/O2. 3101282   7.9250   NaN   \n",
       "3       1  female  35.0      1      0            113803  53.1000  C123   \n",
       "4       3    male  35.0      0      0            373450   8.0500   NaN   \n",
       "\n",
       "   Title_Master  Title_Miss  Title_Mr  Title_Mrs  Title_Officer  \\\n",
       "0             0           0         1          0              0   \n",
       "1             0           0         0          1              0   \n",
       "2             0           1         0          0              0   \n",
       "3             0           0         0          1              0   \n",
       "4             0           0         1          0              0   \n",
       "\n",
       "   Title_Royalty  Embarked_C  Embarked_Q  Embarked_S  \n",
       "0              0           0           0           1  \n",
       "1              0           1           0           0  \n",
       "2              0           0           0           1  \n",
       "3              0           0           0           1  \n",
       "4              0           0           0           1  "
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature Engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- **Creating Age Band**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0    (20.0, 24.0]\n",
       "1    (32.0, 39.0]\n",
       "2    (24.0, 26.0]\n",
       "3    (32.0, 39.0]\n",
       "4    (32.0, 39.0]\n",
       "Name: Age, dtype: category\n",
       "Categories (9, interval[float64]): [(0.169, 17.0] < (17.0, 20.0] < (20.0, 24.0] < (24.0, 26.0] ... (28.0, 32.0] < (32.0, 39.0] < (39.0, 47.0] < (47.0, 80.0]]"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined['Age'] = pd.qcut(combined['Age'],9)\n",
    "combined['Age'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [],
   "source": [
    "'''Age_bin = pd.get_dummies(combined['Age'])\n",
    "combined = pd.concat([combined, Age_bin], axis=1)\n",
    "combined.drop([\"Age\"], axis=1, inplace=True)'''\n",
    "\n",
    "combined['Age'] = LabelEncoder().fit_transform(combined['Age'])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- **Creating Fare Band**"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Title_Master</th>\n",
       "      <th>Title_Miss</th>\n",
       "      <th>Title_Mr</th>\n",
       "      <th>Title_Mrs</th>\n",
       "      <th>Title_Officer</th>\n",
       "      <th>Title_Royalty</th>\n",
       "      <th>Embarked_C</th>\n",
       "      <th>Embarked_Q</th>\n",
       "      <th>Embarked_S</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>7.2500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>71.2833</td>\n",
       "      <td>C85</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>female</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>7.9250</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>53.1000</td>\n",
       "      <td>C123</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>8.0500</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pclass     Sex  Age  SibSp  Parch            Ticket     Fare Cabin  \\\n",
       "0       3    male    2      1      0         A/5 21171   7.2500   NaN   \n",
       "1       1  female    6      1      0          PC 17599  71.2833   C85   \n",
       "2       3  female    3      0      0  STON/O2. 3101282   7.9250   NaN   \n",
       "3       1  female    6      1      0            113803  53.1000  C123   \n",
       "4       3    male    6      0      0            373450   8.0500   NaN   \n",
       "\n",
       "   Title_Master  Title_Miss  Title_Mr  Title_Mrs  Title_Officer  \\\n",
       "0             0           0         1          0              0   \n",
       "1             0           0         0          1              0   \n",
       "2             0           1         0          0              0   \n",
       "3             0           0         0          1              0   \n",
       "4             0           0         1          0              0   \n",
       "\n",
       "   Title_Royalty  Embarked_C  Embarked_Q  Embarked_S  \n",
       "0              0           0           0           1  \n",
       "1              0           1           0           0  \n",
       "2              0           0           0           1  \n",
       "3              0           0           0           1  \n",
       "4              0           0           0           1  "
      ]
     },
     "execution_count": 27,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined['Ticket_count'] = combined.Ticket.apply(lambda x: combined[combined['Ticket']==x].shape[0] ) #Counts how many people have the same ticket\n",
    "combined['Fare_tickect']= combined.apply(lambda x: x.Fare/x.Ticket_count,axis=1 )\n",
    "combined['Fare'] = pd.qcut(combined['Fare_tickect'],12)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined['Fare'] = LabelEncoder().fit_transform(combined['Fare'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined.drop(\"Fare_tickect\",axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Pclass</th>\n",
       "      <th>Sex</th>\n",
       "      <th>Age</th>\n",
       "      <th>SibSp</th>\n",
       "      <th>Parch</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Title_Master</th>\n",
       "      <th>Title_Miss</th>\n",
       "      <th>Title_Mr</th>\n",
       "      <th>Title_Mrs</th>\n",
       "      <th>Title_Officer</th>\n",
       "      <th>Title_Royalty</th>\n",
       "      <th>Embarked_C</th>\n",
       "      <th>Embarked_Q</th>\n",
       "      <th>Embarked_S</th>\n",
       "      <th>Ticket_count</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>11</td>\n",
       "      <td>C85</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>female</td>\n",
       "      <td>3</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1</td>\n",
       "      <td>female</td>\n",
       "      <td>6</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>113803</td>\n",
       "      <td>9</td>\n",
       "      <td>C123</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>3</td>\n",
       "      <td>male</td>\n",
       "      <td>6</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>373450</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   Pclass     Sex  Age  SibSp  Parch            Ticket  Fare Cabin  \\\n",
       "0       3    male    2      1      0         A/5 21171     2   NaN   \n",
       "1       1  female    6      1      0          PC 17599    11   C85   \n",
       "2       3  female    3      0      0  STON/O2. 3101282     5   NaN   \n",
       "3       1  female    6      1      0            113803     9  C123   \n",
       "4       3    male    6      0      0            373450     5   NaN   \n",
       "\n",
       "   Title_Master  Title_Miss  Title_Mr  Title_Mrs  Title_Officer  \\\n",
       "0             0           0         1          0              0   \n",
       "1             0           0         0          1              0   \n",
       "2             0           1         0          0              0   \n",
       "3             0           0         0          1              0   \n",
       "4             0           0         1          0              0   \n",
       "\n",
       "   Title_Royalty  Embarked_C  Embarked_Q  Embarked_S  Ticket_count  \n",
       "0              0           0           0           1             1  \n",
       "1              0           1           0           0             2  \n",
       "2              0           0           0           1             1  \n",
       "3              0           0           0           1             2  \n",
       "4              0           0           0           1             1  "
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Processing Pclass And Sex"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [],
   "source": [
    "Pclass_bin = pd.get_dummies(combined['Pclass'],prefix ='Class')\n",
    "Sex_bin = pd.get_dummies(combined['Sex'],drop_first=True,prefix ='Sex')\n",
    "combined = pd.concat([combined, Pclass_bin,Sex_bin], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined.drop(['Sex','Pclass'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Processing Family"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [],
   "source": [
    "def process_family():\n",
    "    \n",
    "    global combined\n",
    "    # introducing a new feature : the size of families (including the passenger)\n",
    "    combined['FamilySize'] = combined['Parch'] + combined['SibSp'] + 1\n",
    "    \n",
    "    # introducing other features based on the family size\n",
    "    combined['Singleton'] = combined['FamilySize'].map(lambda s: 1 if s == 1 else 0)\n",
    "    combined['SmallFamily'] = combined['FamilySize'].map(lambda s: 1 if 2 <= s <= 4 else 0)\n",
    "    combined['LargeFamily'] = combined['FamilySize'].map(lambda s: 1 if 5 <= s else 0)\n",
    "    \n",
    "    return combined"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined = process_family()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined.drop(['SibSp','Parch','FamilySize'],axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Processing Cabin"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Ticket</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Cabin</th>\n",
       "      <th>Title_Master</th>\n",
       "      <th>Title_Miss</th>\n",
       "      <th>Title_Mr</th>\n",
       "      <th>Title_Mrs</th>\n",
       "      <th>Title_Officer</th>\n",
       "      <th>Title_Royalty</th>\n",
       "      <th>...</th>\n",
       "      <th>Embarked_Q</th>\n",
       "      <th>Embarked_S</th>\n",
       "      <th>Ticket_count</th>\n",
       "      <th>Class_1</th>\n",
       "      <th>Class_2</th>\n",
       "      <th>Class_3</th>\n",
       "      <th>Sex_male</th>\n",
       "      <th>Singleton</th>\n",
       "      <th>SmallFamily</th>\n",
       "      <th>LargeFamily</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>A/5 21171</td>\n",
       "      <td>2</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>PC 17599</td>\n",
       "      <td>11</td>\n",
       "      <td>C85</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>STON/O2. 3101282</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>113803</td>\n",
       "      <td>9</td>\n",
       "      <td>C123</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>2</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>373450</td>\n",
       "      <td>5</td>\n",
       "      <td>NaN</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 21 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Age            Ticket  Fare Cabin  Title_Master  Title_Miss  Title_Mr  \\\n",
       "0    2         A/5 21171     2   NaN             0           0         1   \n",
       "1    6          PC 17599    11   C85             0           0         0   \n",
       "2    3  STON/O2. 3101282     5   NaN             0           1         0   \n",
       "3    6            113803     9  C123             0           0         0   \n",
       "4    6            373450     5   NaN             0           0         1   \n",
       "\n",
       "   Title_Mrs  Title_Officer  Title_Royalty  ...  Embarked_Q  Embarked_S  \\\n",
       "0          0              0              0  ...           0           1   \n",
       "1          1              0              0  ...           0           0   \n",
       "2          0              0              0  ...           0           1   \n",
       "3          1              0              0  ...           0           1   \n",
       "4          0              0              0  ...           0           1   \n",
       "\n",
       "   Ticket_count  Class_1  Class_2  Class_3  Sex_male  Singleton  SmallFamily  \\\n",
       "0             1        0        0        1         1          0            1   \n",
       "1             2        1        0        0         0          0            1   \n",
       "2             1        0        0        1         0          1            0   \n",
       "3             2        1        0        0         0          0            1   \n",
       "4             1        0        0        1         1          1            0   \n",
       "\n",
       "   LargeFamily  \n",
       "0            0  \n",
       "1            0  \n",
       "2            0  \n",
       "3            0  \n",
       "4            0  \n",
       "\n",
       "[5 rows x 21 columns]"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [],
   "source": [
    "train_cabin, test_cabin = set(), set()\n",
    "\n",
    "for c in combined.iloc[:891]['Cabin']:\n",
    "    try:\n",
    "        train_cabin.add(c[0])\n",
    "    except:\n",
    "        train_cabin.add('U')\n",
    "        \n",
    "for c in combined.iloc[891:]['Cabin']:\n",
    "    try:\n",
    "        test_cabin.add(c[0])\n",
    "    except:\n",
    "        test_cabin.add('U')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined['Deck'] = combined['Cabin'].apply(lambda s: s[0] if pd.notnull(s) else 'M')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "M      1014\n",
       "ABC     181\n",
       "DE       87\n",
       "FG       26\n",
       "T         1\n",
       "Name: Deck, dtype: int64"
      ]
     },
     "execution_count": 40,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined['Deck'] = combined['Deck'].replace(['A', 'B', 'C'], 'ABC')\n",
    "combined['Deck'] = combined['Deck'].replace(['D', 'E'], 'DE')\n",
    "combined['Deck'] = combined['Deck'].replace(['F', 'G'], 'FG')\n",
    "\n",
    "combined['Deck'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined.drop(['Cabin'], inplace=True, axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [],
   "source": [
    "cabin_dummies = pd.get_dummies(combined['Deck'], prefix='Deck')    \n",
    "combined = pd.concat([combined, cabin_dummies], axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 43,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined.drop('Deck', axis=1, inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 44,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'B', 'E', 'G', 'D', 'C', 'T', 'U', 'F', 'A'}\n"
     ]
    }
   ],
   "source": [
    "print(train_cabin)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 45,
   "metadata": {},
   "outputs": [],
   "source": [
    "combined.drop('Ticket',axis=1,inplace=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Age</th>\n",
       "      <th>Fare</th>\n",
       "      <th>Title_Master</th>\n",
       "      <th>Title_Miss</th>\n",
       "      <th>Title_Mr</th>\n",
       "      <th>Title_Mrs</th>\n",
       "      <th>Title_Officer</th>\n",
       "      <th>Title_Royalty</th>\n",
       "      <th>Embarked_C</th>\n",
       "      <th>Embarked_Q</th>\n",
       "      <th>...</th>\n",
       "      <th>Class_3</th>\n",
       "      <th>Sex_male</th>\n",
       "      <th>Singleton</th>\n",
       "      <th>SmallFamily</th>\n",
       "      <th>LargeFamily</th>\n",
       "      <th>Deck_ABC</th>\n",
       "      <th>Deck_DE</th>\n",
       "      <th>Deck_FG</th>\n",
       "      <th>Deck_M</th>\n",
       "      <th>Deck_T</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>2</td>\n",
       "      <td>2</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>6</td>\n",
       "      <td>11</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>6</td>\n",
       "      <td>9</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>6</td>\n",
       "      <td>5</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   Age  Fare  Title_Master  Title_Miss  Title_Mr  Title_Mrs  Title_Officer  \\\n",
       "0    2     2             0           0         1          0              0   \n",
       "1    6    11             0           0         0          1              0   \n",
       "2    3     5             0           1         0          0              0   \n",
       "3    6     9             0           0         0          1              0   \n",
       "4    6     5             0           0         1          0              0   \n",
       "\n",
       "   Title_Royalty  Embarked_C  Embarked_Q  ...  Class_3  Sex_male  Singleton  \\\n",
       "0              0           0           0  ...        1         1          0   \n",
       "1              0           1           0  ...        0         0          0   \n",
       "2              0           0           0  ...        1         0          1   \n",
       "3              0           0           0  ...        0         0          0   \n",
       "4              0           0           0  ...        1         1          1   \n",
       "\n",
       "   SmallFamily  LargeFamily  Deck_ABC  Deck_DE  Deck_FG  Deck_M  Deck_T  \n",
       "0            1            0         0        0        0       1       0  \n",
       "1            1            0         1        0        0       0       0  \n",
       "2            0            0         0        0        0       1       0  \n",
       "3            1            0         1        0        0       0       0  \n",
       "4            0            0         0        0        0       1       0  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 46,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "combined.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# It's Modelling Time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "trainc = combined.iloc[:891]\n",
    "testc = combined.iloc[891:]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "https://stackabuse.com/cross-validation-and-grid-search-for-model-selection-in-python/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 48,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compute_score(clf, X, y, scoring='accuracy'):\n",
    "    xval = cross_val_score(clf, X, y, cv = 5, scoring=scoring)\n",
    "    return np.mean(xval)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg = LogisticRegression()\n",
    "logreg_cv = LogisticRegressionCV()\n",
    "rf = RandomForestClassifier()\n",
    "gboost = GradientBoostingClassifier()\n",
    "\n",
    "models = [logreg, logreg_cv, rf, gboost]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 50,
   "metadata": {},
   "outputs": [],
   "source": [
    "logreg = LogisticRegression()\n",
    "logreg_cv = LogisticRegressionCV()\n",
    "rf = RandomForestClassifier()\n",
    "gboost = GradientBoostingClassifier()\n",
    "\n",
    "models = [logreg, logreg_cv, rf, gboost]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 51,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Cross-validation of : <class 'sklearn.linear_model.logistic.LogisticRegression'>\n",
      "CV score = 0.827202976678735\n",
      "****\n",
      "Cross-validation of : <class 'sklearn.linear_model.logistic.LogisticRegressionCV'>\n",
      "CV score = 0.8249557147401558\n",
      "****\n",
      "Cross-validation of : <class 'sklearn.ensemble.forest.RandomForestClassifier'>\n",
      "CV score = 0.8070092882902149\n",
      "****\n",
      "Cross-validation of : <class 'sklearn.ensemble.gradient_boosting.GradientBoostingClassifier'>\n",
      "CV score = 0.8249304646060146\n",
      "****\n"
     ]
    }
   ],
   "source": [
    "for model in models:\n",
    "    print('Cross-validation of : {0}'.format(model.__class__))\n",
    "    score = compute_score(clf=model, X=trainc, y=targets, scoring='accuracy')\n",
    "    print('CV score = {0}'.format(score))\n",
    "    print('****')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 55,
   "metadata": {},
   "outputs": [],
   "source": [
    "run_gs = False\n",
    "\n",
    "if run_gs:\n",
    "    parameter_grid = {\n",
    "                 'max_depth' : [4, 6, 8],\n",
    "                 'n_estimators': [50, 10],\n",
    "                 'max_features': ['sqrt', 'auto', 'log2'],\n",
    "                 'min_samples_split': [2, 3, 10],\n",
    "                 'min_samples_leaf': [1, 3, 10],\n",
    "                 'bootstrap': [True, False],\n",
    "                 }\n",
    "    forest = RandomForestClassifier()\n",
    "    cross_validation = StratifiedKFold(n_splits=5)\n",
    "\n",
    "    grid_search = GridSearchCV(forest,\n",
    "                               scoring='accuracy',\n",
    "                               param_grid=parameter_grid,\n",
    "                               cv=cross_validation,\n",
    "                               verbose=1\n",
    "                              )\n",
    "\n",
    "    grid_search.fit(trainc, targets)\n",
    "    model = grid_search\n",
    "    parameters = grid_search.best_params_\n",
    "\n",
    "    print('Best score: {}'.format(grid_search.best_score_))\n",
    "    print('Best parameters: {}'.format(grid_search.best_params_))\n",
    "    \n",
    "else: \n",
    "    parameters = {'bootstrap': False, 'min_samples_leaf': 3, 'n_estimators': 50, \n",
    "                  'min_samples_split': 10, 'max_features': 'sqrt', 'max_depth': 6}\n",
    "    \n",
    "    model = RandomForestClassifier(**parameters)\n",
    "    model.fit(trainc, targets)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 56,
   "metadata": {},
   "outputs": [],
   "source": [
    "output = model.predict(testc).astype(int)\n",
    "df_output = pd.DataFrame()\n",
    "aux = pd.read_csv('test.csv')\n",
    "df_output['PassengerId'] = aux['PassengerId']\n",
    "df_output['Survived'] = output\n",
    "df_output[['PassengerId','Survived']].to_csv('submission.csv', index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}