imdb_good_movie.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# run me once\n",
    "import nltk\n",
    "nltk.download('stopwords')\n",
    "nltk.download('punkt')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import seaborn as sns\n",
    "import ppscore as pps\n",
    "import shap\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "from os import path\n",
    "from pandas_profiling import ProfileReport\n",
    "from nltk import word_tokenize\n",
    "from nltk.corpus import stopwords\n",
    "from scipy.spatial import distance\n",
    "from tqdm import tqdm\n",
    "from imblearn.over_sampling import RandomOverSampler\n",
    "from imblearn.pipeline import Pipeline\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.feature_selection import RFECV\n",
    "from sklearn.metrics import average_precision_score\n",
    "from sklearn.metrics import precision_recall_curve\n",
    "from sklearn.metrics import PrecisionRecallDisplay\n",
    "from sklearn.metrics import roc_curve\n",
    "from sklearn.metrics import RocCurveDisplay\n",
    "from sklearn.ensemble import GradientBoostingRegressor"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sns.set()\n",
    "stop_words = stopwords.words('english')"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Config"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setting to config parameters\n",
    "\n",
    "imdb_data_path = \"./data/\" #set to path to folder containing imdb data\n",
    "rating_col = \"weighted_average_vote\"\n",
    "title_id = \"imdb_title_id\"\n",
    "name_id = \"imdb_name_id\"\n",
    "rating_value = 7.5\n",
    "target = \"rating_{}\".format(rating_value)\n",
    "compute_desc_knn = True # set to True if you wish to recompute the description embedding KNN\n",
    "compute_grid_search = True # set to True if you wish to recompute the model hyperparameter gridsearch\n",
    "compute_rfe = True # set to true if you wish to recompute the recursive feature selection"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data ingestion"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "movies = pd.read_csv(path.join(imdb_data_path, \"IMDb movies.csv\"))\n",
    "names = pd.read_csv(path.join(imdb_data_path, \"IMDb names.csv\"))\n",
    "ratings = pd.read_csv(path.join(imdb_data_path, \"IMDb ratings.csv\"))\n",
    "title_principals = pd.read_csv(path.join(imdb_data_path, \"IMDb title_principals.csv\"))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# load the GloVe vectors in a dictionary:\n",
    "\n",
    "embeddings_index = {}\n",
    "\n",
    "if compute_desc_knn:\n",
    "    f = open('./data/glove.840B.300d.txt', encoding=\"utf8\")\n",
    "    for line in tqdm(f):\n",
    "        values = line.split()\n",
    "        word = values[0]\n",
    "        try:\n",
    "            coefs = np.asarray(values[1:], dtype='float32')\n",
    "            embeddings_index[word] = coefs\n",
    "        except ValueError:\n",
    "            pass\n",
    "    f.close()\n",
    "\n",
    "    print('Found %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data exploration"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true,
    "tags": []
   },
   "outputs": [],
   "source": [
    "# using pandas profiling to explore the datasets\n",
    "\n",
    "movies_profile = ProfileReport(movies)\n",
    "movies_profile"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "features to explore:\n",
    "* date_published to day count\n",
    "* genre (one hot)\n",
    "* duration (filter outlier)\n",
    "* country (one hot)\n",
    "* language (one hot)\n",
    "* description (word to vec)\n",
    "* budget (but to many missing)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "tags": []
   },
   "outputs": [],
   "source": [
    "names_profile = ProfileReport(names)\n",
    "names_profile"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "features to explore:\n",
    "* age\n",
    "* is dead"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "target is weighted_average_vote or mean_vote"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "title_principals.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "title_principals[\"category\"].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "title_principals[\"job\"].value_counts()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "allows to link movies to names\n",
    "feature to explore:\n",
    "* number of movie with rating above > 7.5 for each category"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data formatting"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# setting the id to the datasets\n",
    "\n",
    "movies = movies.set_index(title_id)\n",
    "ratings = ratings.set_index(title_id)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature engineering"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Target computation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ratings[target] = ratings[rating_col] > rating_value\n",
    "ratings[target] = ratings[target].astype(int)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Historical ratings for film crew"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# add movie rating and date pulbished to title_principals\n",
    "rating_history = title_principals.merge(ratings[[target]], left_on=title_id, right_index=True, how=\"left\")\n",
    "rating_history = rating_history.merge(movies[[\"date_published\"]], left_on=title_id, right_index=True, how=\"left\")\n",
    "\n",
    "rating_history_columns = [title_id, name_id, \"category\", target, \"date_published\"]\n",
    "rating_history_select = rating_history[rating_history_columns]\n",
    "rating_history_select.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# join rating_history on itself to know on which other movies the movie crew worked\n",
    "rating_history_join = rating_history_select.merge(rating_history_select, on=name_id, suffixes = (\"\", \"_history\"), how=\"left\")\n",
    "\n",
    "# keep only movies older than the current movie\n",
    "rating_history_filter = rating_history_join[\"date_published\"] > rating_history_join[\"date_published_history\"]\n",
    "rating_history_filtered = rating_history_join.loc[rating_history_filter]\n",
    "\n",
    "# get the number of older movies for which the film crew worked, and the number of those above the rating target\n",
    "rating_history_agg = rating_history_filtered.groupby([title_id, \"category\"])[\"{}_history\".format(target)].agg([\"count\", \"sum\"])\n",
    "# compute the ratio of movies reaching the rating target\n",
    "rating_history_agg['ratio'] = rating_history_agg['sum']/rating_history_agg['count']\n",
    "\n",
    "# rename the columns\n",
    "rating_history_agg = rating_history_agg.unstack(-1)\n",
    "rating_history_agg.columns = [\"target_history_{}_{}\".format(x[0], x[1]) for x in rating_history_agg.columns]\n",
    "\n",
    "# fill missing values\n",
    "rating_history_agg = rating_history_agg.fillna(0)\n",
    "# the count is actually 0, the ratio is undefined but the model will be able to pick it using the count\n",
    "\n",
    "rating_history_agg.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Movies"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the number of days since the movie was released\n",
    "movies[\"date_published\"] = pd.to_datetime(movies[\"date_published\"], errors=\"coerce\")\n",
    "movies[\"age\"] = movies[\"date_published\"] - movies[\"date_published\"].min()\n",
    "movies[\"age\"] = movies[\"age\"].dt.days"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get one hot encoded values for film genre, language and country\n",
    "movies_genre = movies['genre'].str.replace(\" \", \"\").str.get_dummies(sep=\",\")\n",
    "movies_country = movies['country'].str.replace(\" \", \"\").str.get_dummies(sep=\",\")\n",
    "movies_language = movies['language'].str.replace(\" \", \"\").str.get_dummies(sep=\",\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this function creates a normalized vector for the whole sentence\n",
    "def sent2vec(s):\n",
    "    words = str(s).lower()#.decode('utf-8')\n",
    "    words = word_tokenize(words)\n",
    "    words = [w for w in words if not w in stop_words]\n",
    "    words = [w for w in words if w.isalpha()]\n",
    "    M = []\n",
    "    for w in words:\n",
    "        try:\n",
    "            M.append(embeddings_index[w])\n",
    "        except:\n",
    "            continue\n",
    "    M = np.array(M)\n",
    "    v = M.sum(axis=0)\n",
    "    if type(v) != np.ndarray:\n",
    "        return np.zeros(300)\n",
    "    return v / np.sqrt((v ** 2).sum())\n",
    "\n",
    "if compute_desc_knn:\n",
    "\n",
    "    # get word embeddings for movie description\n",
    "    movies_desc_emb = movies[\"description\"].apply(sent2vec)\n",
    "    movies_desc_emb = pd.DataFrame(movies_desc_emb.tolist(), index=movies_desc_emb.index)\n",
    "\n",
    "    # join movie date and rating\n",
    "    movies_desc_emb[\"date_published\"] = movies[\"date_published\"]\n",
    "    movies_desc_emb[rating_col] = ratings[rating_col]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# this function computes the mean rating of the n closest older movies based on the description embeddings\n",
    "def get_mean_filtered_knn(current_movies_description_emb, all_movies_description_emb, n_neighbours = 30, n_dim_embeddings = 300):\n",
    "    \n",
    "    scope = all_movies_description_emb[\"date_published\"] < current_movies_description_emb[\"date_published\"]\n",
    "    older_movies_description_emb = all_movies_description_emb.loc[scope]\n",
    "    \n",
    "    if older_movies_description_emb.shape[0] < n_neighbours:\n",
    "        return np.nan\n",
    "    \n",
    "    emb_columns = range(n_dim_embeddings)\n",
    "    \n",
    "    # computing the euclidean distance between the embeddings of the current movie and all older movies\n",
    "    res = distance.cdist(current_movies_description_emb[emb_columns].values.reshape(1, -1), older_movies_description_emb[emb_columns])[0]\n",
    "    \n",
    "    # selection the n closest neighbours\n",
    "    neighbours_id = np.argsort(res)[:n_neighbours]\n",
    "    \n",
    "    # getting the mean rating the ne clossest nieghbours\n",
    "    return older_movies_description_emb.iloc[neighbours_id][rating_col].mean()\n",
    "\n",
    "if compute_desc_knn:\n",
    "\n",
    "    desc_emb_knn30_mean = movies_desc_emb.apply(lambda row: get_mean_filtered_knn(row, movies_desc_emb), axis=1)\n",
    "\n",
    "    # saving the result as pickle file because it is a heavy computation\n",
    "    desc_emb_knn30_mean.to_pickle(\"desc_emb_knn30_mean.pkl\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "if not compute_desc_knn:\n",
    "    # load the pre computed data\n",
    "    desc_emb_knn30_mean = pd.read_pickle(\"./desc_emb_knn30_mean.pkl\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Data join"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from movies\n",
    "movies_columns = [\"duration\", \"age\"]\n",
    "feature_table = movies.loc[:, movies_columns]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from ratings\n",
    "feature_table.loc[:, target] = ratings[target]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# fromm the embeddings knn \n",
    "feature_table.loc[:, \"desc_knn_mean_rating\"] = desc_emb_knn30_mean"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# join all one encoded columns which represents at least  1% of the movies\n",
    "\n",
    "one_hot_encoded_dfs = [movies_genre, movies_country, movies_language]\n",
    "\n",
    "for one_hot_encoded_df in one_hot_encoded_dfs:\n",
    "    columns_to_join = one_hot_encoded_df.sum()\n",
    "    columns_to_join = columns_to_join.loc[columns_to_join>=movies.shape[0]/100].index\n",
    "    feature_table = feature_table.join(one_hot_encoded_df[columns_to_join])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# join the historical rating of film crew, for the categories which are available for at least 1% of the movies\n",
    "\n",
    "categories_in_scope = title_principals[\"category\"].value_counts()\n",
    "categories_in_scope = categories_in_scope.loc[categories_in_scope>=movies.shape[0]/100].index\n",
    "rating_history_columns = [\"target_history_count_{}\".format(x) for x in categories_in_scope]\n",
    "rating_history_columns += [\"target_history_ratio_{}\".format(x) for x in categories_in_scope]\n",
    "feature_table = feature_table.join(rating_history_agg[rating_history_columns])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the list of features name\n",
    "features = [x for x in feature_table.columns if x!=target]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Filtering"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "na_values = feature_table.isna().sum()\n",
    "na_values.loc[na_values>0].sort_values(ascending = False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "historical rating features are not available for half of the movies\n",
    "it is due to missing records in the title_principals dataset\n",
    "the assumption here is that the data is avaialble but not updated yet, therefore I chose to keep the features and drop the missings, assuming the mssing title_principals data will become available"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_table = feature_table.dropna()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Train validation split"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "X = feature_table[features]\n",
    "y = feature_table[target]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# selecting 10% of the data as validation set\n",
    "\n",
    "X_train_test, X_val, y_train_test, y_val = train_test_split(X, y, test_size=0.1, random_state=0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Resampling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "value_counts = y_train_test.value_counts()\n",
    "(value_counts/value_counts.sum()).plot(kind=\"barh\")"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "as the dataset is imbalanced, we will resample it using a random over sample to 1:10"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "ros = RandomOverSampler(random_state=0, sampling_strategy=0.1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Modeling"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# using GradientBoostingRegressor for the classification as it performs well for tabular data\n",
    "clf = GradientBoostingRegressor()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Pipeline"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# building a pipeline including the over sampler and the classifier\n",
    "boost_model = Pipeline([\n",
    "        ('upsampling', ros),\n",
    "        ('clf', clf)\n",
    "    ])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Feature selection"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# getting ppscore for our features\n",
    "\n",
    "# resampling the data to 1:1 to get scores representing correct prediction of the two classes as ppscore uses decision trees\n",
    "ros_ppscore = RandomOverSampler(random_state=0, sampling_strategy=1)\n",
    "X_ppscore, y_ppscore = ros_ppscore.fit_resample(X_train_test, y_train_test)\n",
    "\n",
    "X_y_ppscore = X_ppscore.join(y_ppscore.astype(str))\n",
    "\n",
    "predictors_df = pps.predictors(X_y_ppscore, y=target, sample = None)\n",
    "predictors_df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# getting the number of features bringing predictive power according to pps\n",
    "n_feature_max = (predictors_df[\"ppscore\"]>0).sum()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# using recursive feature ranking to select which feature to keep in the model\n",
    "\n",
    "# creating a function to get the feature importance from the model in the pipeline\n",
    "get_feature_importance = lambda pipe: np.array(list(pipe['clf'].feature_importances_))\n",
    "\n",
    "\n",
    "# using average precision, or area under the precision curve, as scoring because it is based on the decision function and therefore allows to choose the prediction threshold based on the use case\n",
    "selector = RFECV(boost_model, step=1, cv=5, verbose=10, scoring='average_precision', n_jobs=5, importance_getter=get_feature_importance)\n",
    "\n",
    "if compute_rfe:\n",
    "    selector = selector.fit(X_train_test[predictors_df[\"x\"].iloc[:n_feature_max]], y_train_test)\n",
    "\n",
    "    pd.Series(selector.grid_scores_.mean(axis=1), index=range(1, n_feature_max+1)).plot(kind=\"barh\")\n",
    "\n",
    "    # selection the feature based on the recursive selection search\n",
    "    feature_list = predictors_df[\"x\"].iloc[:n_feature_max].loc[selector.support_]\n",
    "    \n",
    "else:\n",
    "    feature_list = ['age',\n",
    "     'duration',\n",
    "     'target_history_ratio_actor',\n",
    "     'target_history_ratio_director',\n",
    "     'desc_knn_mean_rating',\n",
    "     'Drama',\n",
    "     'English',\n",
    "     'target_history_ratio_composer',\n",
    "     'target_history_ratio_writer',\n",
    "     'target_history_count_producer',\n",
    "     'target_history_ratio_actress',\n",
    "     'target_history_count_director',\n",
    "     'target_history_ratio_cinematographer',\n",
    "     'target_history_count_actress',\n",
    "     'target_history_count_actor',\n",
    "     'India',\n",
    "     'target_history_count_writer',\n",
    "     'target_history_ratio_producer',\n",
    "     'target_history_count_cinematographer']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model tuning"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# tune regressor parameters using a grid search\n",
    "\n",
    "parameters = {\n",
    "    'clf__learning_rate': [0.1, 0.05], #explore multi feature pattern\n",
    "    'clf__max_depth': [2, 3, 5, 7, 9], #explore multi feature pattern\n",
    "    'clf__subsample': [0.6, 0.8, 1.0], #avoid overfit\n",
    "    'clf__n_estimators': [1000], #high number of tree but with an early stopping of 50\n",
    "    'clf__random_state': [2021],\n",
    "    'clf__n_iter_no_change': [50],\n",
    "    'clf__tol':[0.01],\n",
    "}\n",
    "\n",
    "fit_params={}\n",
    "\n",
    "\n",
    "clf_gs = GridSearchCV(boost_model, parameters, \n",
    "                      n_jobs=4,\n",
    "                   cv=5, \n",
    "                   scoring='average_precision',\n",
    "                   verbose=10, refit=True)\n",
    "\n",
    "if compute_grid_search:\n",
    "\n",
    "    clf_gs.fit(X_train_test[feature_list], y_train_test, **fit_params)\n",
    "\n",
    "    print(clf_gs.cv_results_[\"mean_test_score\"].max())\n",
    "    regr_params = clf_gs.best_params_\n",
    "    \n",
    "else:\n",
    "    regr_params = {\n",
    "        'clf__learning_rate': 0.1,\n",
    "        'clf__max_depth': 5,\n",
    "        'clf__n_estimators': 1000,\n",
    "        'clf__n_iter_no_change': 50,\n",
    "        'clf__random_state': 2021,\n",
    "        'clf__subsample': 0.6,\n",
    "        'clf__tol': 0.01\n",
    "    }"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model evaluation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# testing on the validation dataset\n",
    "\n",
    "boost_model.set_params(**regr_params)\n",
    "\n",
    "boost_model.fit(X_train_test[feature_list], y_train_test, **fit_params)\n",
    "\n",
    "y_score = boost_model.predict(X_val[feature_list])\n",
    "\n",
    "print(average_precision_score(y_val, y_score))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plotting the ROC curve\n",
    "\n",
    "f, ax = plt.subplots(figsize = (16,9))\n",
    "\n",
    "fpr, tpr, _ = roc_curve(y_val, y_score)\n",
    "roc_display = RocCurveDisplay(fpr=fpr, tpr=tpr).plot(ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#plotting the PR curve\n",
    "\n",
    "f, ax = plt.subplots(figsize = (16,9))\n",
    "\n",
    "prec, recall, _ = precision_recall_curve(y_val, y_score)\n",
    "pr_display = PrecisionRecallDisplay(precision=prec, recall=recall).plot(ax=ax)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Model insights"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# getting feature importance based on split gain\n",
    "\n",
    "f, ax = plt.subplots(figsize = (16,9))\n",
    "\n",
    "pd.Series(boost_model['clf'].feature_importances_, index=feature_list).sort_values().plot(kind=\"barh\", ax=ax)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# getting feature importance using shapley value\n",
    "\n",
    "# load JS visualization code to notebook\n",
    "shap.initjs()\n",
    "\n",
    "explainer = shap.TreeExplainer(boost_model[\"clf\"])\n",
    "shap_values = explainer.shap_values(X_train_test[feature_list])\n",
    "\n",
    "shap.summary_plot(shap_values, X_train_test[feature_list])\n",
    "\n",
    "f = plt.gcf()\n",
    "f.set_size_inches(16, 9)\n",
    "f.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "feature_importance_order = np.array(feature_list)[np.argsort(np.abs(shap_values).mean(0))][::-1].tolist()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# get the impact on the prediction for each feature\n",
    "\n",
    "for feature in feature_importance_order:\n",
    "    shap.dependence_plot(feature, shap_values, X_train_test[feature_list], show=False)\n",
    "    f = plt.gcf()\n",
    "    f.set_size_inches(16, 9)\n",
    "    f.show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# same as above but without the coloring for easier reading\n",
    "\n",
    "for feature in feature_importance_order:\n",
    "    shap.dependence_plot(feature, shap_values, X_train_test[feature_list], interaction_index=None, show=False)\n",
    "    f = plt.gcf()\n",
    "    f.set_size_inches(16, 9)\n",
    "    f.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.9"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}