Working Example on Hands-On-Recommendation-Systems-with-Python - Chapter6.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Recommended Systems Best Practices"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>age</th>\n",
       "      <th>sex</th>\n",
       "      <th>occupation</th>\n",
       "      <th>zip_code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>24</td>\n",
       "      <td>M</td>\n",
       "      <td>technician</td>\n",
       "      <td>85711</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>53</td>\n",
       "      <td>F</td>\n",
       "      <td>other</td>\n",
       "      <td>94043</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>23</td>\n",
       "      <td>M</td>\n",
       "      <td>writer</td>\n",
       "      <td>32067</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>24</td>\n",
       "      <td>M</td>\n",
       "      <td>technician</td>\n",
       "      <td>43537</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>33</td>\n",
       "      <td>F</td>\n",
       "      <td>other</td>\n",
       "      <td>15213</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  age sex  occupation zip_code\n",
       "0        1   24   M  technician    85711\n",
       "1        2   53   F       other    94043\n",
       "2        3   23   M      writer    32067\n",
       "3        4   24   M  technician    43537\n",
       "4        5   33   F       other    15213"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Load the u.user file into a dataframe\n",
    "u_cols = ['user_id', 'age', 'sex', 'occupation', 'zip_code']\n",
    "\n",
    "users = pd.read_csv('./data/u.user', sep='|', names=u_cols,\n",
    " encoding='latin-1')\n",
    "\n",
    "users.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>movie_id</th>\n",
       "      <th>title</th>\n",
       "      <th>release date</th>\n",
       "      <th>video release date</th>\n",
       "      <th>IMDb URL</th>\n",
       "      <th>unknown</th>\n",
       "      <th>Action</th>\n",
       "      <th>Adventure</th>\n",
       "      <th>Animation</th>\n",
       "      <th>Children's</th>\n",
       "      <th>...</th>\n",
       "      <th>Fantasy</th>\n",
       "      <th>Film-Noir</th>\n",
       "      <th>Horror</th>\n",
       "      <th>Musical</th>\n",
       "      <th>Mystery</th>\n",
       "      <th>Romance</th>\n",
       "      <th>Sci-Fi</th>\n",
       "      <th>Thriller</th>\n",
       "      <th>War</th>\n",
       "      <th>Western</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1</td>\n",
       "      <td>Toy Story (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Toy%20Story%2...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2</td>\n",
       "      <td>GoldenEye (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?GoldenEye%20(...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>3</td>\n",
       "      <td>Four Rooms (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Four%20Rooms%...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4</td>\n",
       "      <td>Get Shorty (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Get%20Shorty%...</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5</td>\n",
       "      <td>Copycat (1995)</td>\n",
       "      <td>01-Jan-1995</td>\n",
       "      <td>NaN</td>\n",
       "      <td>http://us.imdb.com/M/title-exact?Copycat%20(1995)</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 24 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   movie_id              title release date  video release date  \\\n",
       "0         1   Toy Story (1995)  01-Jan-1995                 NaN   \n",
       "1         2   GoldenEye (1995)  01-Jan-1995                 NaN   \n",
       "2         3  Four Rooms (1995)  01-Jan-1995                 NaN   \n",
       "3         4  Get Shorty (1995)  01-Jan-1995                 NaN   \n",
       "4         5     Copycat (1995)  01-Jan-1995                 NaN   \n",
       "\n",
       "                                            IMDb URL  unknown  Action  \\\n",
       "0  http://us.imdb.com/M/title-exact?Toy%20Story%2...        0       0   \n",
       "1  http://us.imdb.com/M/title-exact?GoldenEye%20(...        0       1   \n",
       "2  http://us.imdb.com/M/title-exact?Four%20Rooms%...        0       0   \n",
       "3  http://us.imdb.com/M/title-exact?Get%20Shorty%...        0       1   \n",
       "4  http://us.imdb.com/M/title-exact?Copycat%20(1995)        0       0   \n",
       "\n",
       "   Adventure  Animation  Children's  ...  Fantasy  Film-Noir  Horror  Musical  \\\n",
       "0          0          1           1  ...        0          0       0        0   \n",
       "1          1          0           0  ...        0          0       0        0   \n",
       "2          0          0           0  ...        0          0       0        0   \n",
       "3          0          0           0  ...        0          0       0        0   \n",
       "4          0          0           0  ...        0          0       0        0   \n",
       "\n",
       "   Mystery  Romance  Sci-Fi  Thriller  War  Western  \n",
       "0        0        0       0         0    0        0  \n",
       "1        0        0       0         1    0        0  \n",
       "2        0        0       0         1    0        0  \n",
       "3        0        0       0         0    0        0  \n",
       "4        0        0       0         1    0        0  \n",
       "\n",
       "[5 rows x 24 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "#Load the u.item file into a dataframe\n",
    "i_cols = ['movie_id', 'title' ,'release date','video release date', 'IMDb URL', 'unknown', 'Action', 'Adventure',\n",
    " 'Animation', 'Children\\'s', 'Comedy', 'Crime', 'Documentary', 'Drama', 'Fantasy',\n",
    " 'Film-Noir', 'Horror', 'Musical', 'Mystery', 'Romance', 'Sci-Fi', 'Thriller', 'War', 'Western']\n",
    "\n",
    "movies = pd.read_csv('./data//u.item', sep='|', names=i_cols, encoding='latin-1')\n",
    "\n",
    "movies.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Remove all information except Movie ID and title\n",
    "movies = movies[['movie_id', 'title']]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>user_id</th>\n",
       "      <th>movie_id</th>\n",
       "      <th>rating</th>\n",
       "      <th>timestamp</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>196</td>\n",
       "      <td>242</td>\n",
       "      <td>3</td>\n",
       "      <td>881250949</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>186</td>\n",
       "      <td>302</td>\n",
       "      <td>3</td>\n",
       "      <td>891717742</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>22</td>\n",
       "      <td>377</td>\n",
       "      <td>1</td>\n",
       "      <td>878887116</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>244</td>\n",
       "      <td>51</td>\n",
       "      <td>2</td>\n",
       "      <td>880606923</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>166</td>\n",
       "      <td>346</td>\n",
       "      <td>1</td>\n",
       "      <td>886397596</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "   user_id  movie_id  rating  timestamp\n",
       "0      196       242       3  881250949\n",
       "1      186       302       3  891717742\n",
       "2       22       377       1  878887116\n",
       "3      244        51       2  880606923\n",
       "4      166       346       1  886397596"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Load the u.data file into a dataframe\n",
    "r_cols = ['user_id', 'movie_id', 'rating', 'timestamp']\n",
    "\n",
    "ratings = pd.read_csv('./data/u.data', sep='\\t', names=r_cols,\n",
    " encoding='latin-1')\n",
    "\n",
    "ratings.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Drop the timestamp column\n",
    "ratings = ratings.drop('timestamp', axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "#Import the train_test_split function\n",
    "from sklearn.model_selection import train_test_split\n",
    "\n",
    "#Assign X as the original ratings dataframe and y as the user_id column of ratings.\n",
    "X = ratings.copy()\n",
    "y = ratings['user_id']\n",
    "\n",
    "#Split into training and test datasets, stratified along user_id\n",
    "X_train, X_test, y_train, y_test = train_test_split(X, y, test_size = 0.25, stratify=y, random_state=42)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Import the mean_squared_error function\n",
    "from sklearn.metrics import mean_squared_error\n",
    "\n",
    "#Function that computes the root mean squared error (or RMSE)\n",
    "def rmse(y_true, y_pred):\n",
    "    return np.sqrt(mean_squared_error(y_true, y_pred))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define the baseline model to always return 3.\n",
    "def baseline(user_id, movie_id):\n",
    "    return 3.0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Function to compute the RMSE score obtained on the testing set by a model\n",
    "def score(cf_model):\n",
    "    \n",
    "    #Construct a list of user-movie tuples from the testing dataset\n",
    "    id_pairs = zip(X_test['user_id'], X_test['movie_id'])\n",
    "    \n",
    "    #Predict the rating for every user-movie tuple\n",
    "    y_pred = np.array([cf_model(user, movie) for (user, movie) in id_pairs])\n",
    "    \n",
    "    #Extract the actual ratings given by the users in the test data\n",
    "    y_true = np.array(X_test['rating'])\n",
    "    \n",
    "    #Return the final RMSE score\n",
    "    return rmse(y_true, y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.2470926188539486"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score(baseline)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>movie_id</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>...</th>\n",
       "      <th>1669</th>\n",
       "      <th>1670</th>\n",
       "      <th>1671</th>\n",
       "      <th>1673</th>\n",
       "      <th>1674</th>\n",
       "      <th>1675</th>\n",
       "      <th>1676</th>\n",
       "      <th>1679</th>\n",
       "      <th>1681</th>\n",
       "      <th>1682</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>4.0</td>\n",
       "      <td>1.0</td>\n",
       "      <td>5.0</td>\n",
       "      <td>3.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>2.0</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>NaN</td>\n",
       "      <td>3.0</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "      <td>NaN</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1647 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "movie_id  1     2     3     4     5     6     7     8     9     10    ...  \\\n",
       "user_id                                                               ...   \n",
       "1          5.0   3.0   4.0   3.0   3.0   5.0   4.0   1.0   5.0   3.0  ...   \n",
       "2          NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   2.0  ...   \n",
       "3          NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "4          NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "5          NaN   3.0   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  ...   \n",
       "\n",
       "movie_id  1669  1670  1671  1673  1674  1675  1676  1679  1681  1682  \n",
       "user_id                                                               \n",
       "1          NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "2          NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "3          NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "4          NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "5          NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN   NaN  \n",
       "\n",
       "[5 rows x 1647 columns]"
      ]
     },
     "execution_count": 13,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Build the ratings matrix using pivot_table function\n",
    "r_matrix = X_train.pivot_table(values='rating', index='user_id', columns='movie_id')\n",
    "\n",
    "r_matrix.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "#User Based Collaborative Filter using Mean Ratings\n",
    "def cf_user_mean(user_id, movie_id):\n",
    "    \n",
    "    #Check if movie_id exists in r_matrix\n",
    "    if movie_id in r_matrix:\n",
    "        #Compute the mean of all the ratings given to the movie\n",
    "        mean_rating = r_matrix[movie_id].mean()\n",
    "    \n",
    "    else:\n",
    "        #Default to a rating of 3.0 in the absence of any information\n",
    "        mean_rating = 3.0\n",
    "    \n",
    "    return mean_rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0234701463131335"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Compute RMSE for the Mean model\n",
    "score(cf_user_mean)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Create a dummy ratings matrix with all null values imputed to 0\n",
    "r_matrix_dummy = r_matrix.copy().fillna(0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "# Import cosine_score \n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "#Compute the cosine similarity matrix using the dummy ratings matrix\n",
    "cosine_sim = cosine_similarity(r_matrix_dummy, r_matrix_dummy)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th>user_id</th>\n",
       "      <th>1</th>\n",
       "      <th>2</th>\n",
       "      <th>3</th>\n",
       "      <th>4</th>\n",
       "      <th>5</th>\n",
       "      <th>6</th>\n",
       "      <th>7</th>\n",
       "      <th>8</th>\n",
       "      <th>9</th>\n",
       "      <th>10</th>\n",
       "      <th>...</th>\n",
       "      <th>934</th>\n",
       "      <th>935</th>\n",
       "      <th>936</th>\n",
       "      <th>937</th>\n",
       "      <th>938</th>\n",
       "      <th>939</th>\n",
       "      <th>940</th>\n",
       "      <th>941</th>\n",
       "      <th>942</th>\n",
       "      <th>943</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>user_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.118076</td>\n",
       "      <td>0.029097</td>\n",
       "      <td>0.011628</td>\n",
       "      <td>0.264677</td>\n",
       "      <td>0.312419</td>\n",
       "      <td>0.308729</td>\n",
       "      <td>0.224269</td>\n",
       "      <td>0.026017</td>\n",
       "      <td>0.286411</td>\n",
       "      <td>...</td>\n",
       "      <td>0.308475</td>\n",
       "      <td>0.055872</td>\n",
       "      <td>0.197862</td>\n",
       "      <td>0.131367</td>\n",
       "      <td>0.152449</td>\n",
       "      <td>0.084456</td>\n",
       "      <td>0.293293</td>\n",
       "      <td>0.056765</td>\n",
       "      <td>0.103536</td>\n",
       "      <td>0.326491</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0.118076</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.099097</td>\n",
       "      <td>0.107680</td>\n",
       "      <td>0.034279</td>\n",
       "      <td>0.152789</td>\n",
       "      <td>0.086705</td>\n",
       "      <td>0.078864</td>\n",
       "      <td>0.068940</td>\n",
       "      <td>0.092399</td>\n",
       "      <td>...</td>\n",
       "      <td>0.086927</td>\n",
       "      <td>0.259636</td>\n",
       "      <td>0.289092</td>\n",
       "      <td>0.318824</td>\n",
       "      <td>0.149105</td>\n",
       "      <td>0.186347</td>\n",
       "      <td>0.168034</td>\n",
       "      <td>0.106748</td>\n",
       "      <td>0.136796</td>\n",
       "      <td>0.080358</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0.029097</td>\n",
       "      <td>0.099097</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.252131</td>\n",
       "      <td>0.026893</td>\n",
       "      <td>0.062539</td>\n",
       "      <td>0.039767</td>\n",
       "      <td>0.089474</td>\n",
       "      <td>0.078162</td>\n",
       "      <td>0.037670</td>\n",
       "      <td>...</td>\n",
       "      <td>0.040918</td>\n",
       "      <td>0.019031</td>\n",
       "      <td>0.065417</td>\n",
       "      <td>0.055373</td>\n",
       "      <td>0.086503</td>\n",
       "      <td>0.018418</td>\n",
       "      <td>0.096993</td>\n",
       "      <td>0.109631</td>\n",
       "      <td>0.092574</td>\n",
       "      <td>0.018987</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0.011628</td>\n",
       "      <td>0.107680</td>\n",
       "      <td>0.252131</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.045543</td>\n",
       "      <td>0.078812</td>\n",
       "      <td>0.095354</td>\n",
       "      <td>0.059498</td>\n",
       "      <td>0.053879</td>\n",
       "      <td>...</td>\n",
       "      <td>0.024226</td>\n",
       "      <td>0.050703</td>\n",
       "      <td>0.056561</td>\n",
       "      <td>0.107294</td>\n",
       "      <td>0.098892</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.132900</td>\n",
       "      <td>0.142798</td>\n",
       "      <td>0.097066</td>\n",
       "      <td>0.015176</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>0.264677</td>\n",
       "      <td>0.034279</td>\n",
       "      <td>0.026893</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.202843</td>\n",
       "      <td>0.299619</td>\n",
       "      <td>0.163724</td>\n",
       "      <td>0.038474</td>\n",
       "      <td>0.153021</td>\n",
       "      <td>...</td>\n",
       "      <td>0.262547</td>\n",
       "      <td>0.048524</td>\n",
       "      <td>0.048312</td>\n",
       "      <td>0.022202</td>\n",
       "      <td>0.091910</td>\n",
       "      <td>0.066000</td>\n",
       "      <td>0.156172</td>\n",
       "      <td>0.115842</td>\n",
       "      <td>0.124297</td>\n",
       "      <td>0.267574</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>0.312419</td>\n",
       "      <td>0.152789</td>\n",
       "      <td>0.062539</td>\n",
       "      <td>0.045543</td>\n",
       "      <td>0.202843</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.375963</td>\n",
       "      <td>0.131795</td>\n",
       "      <td>0.110944</td>\n",
       "      <td>0.400758</td>\n",
       "      <td>...</td>\n",
       "      <td>0.287549</td>\n",
       "      <td>0.080312</td>\n",
       "      <td>0.162988</td>\n",
       "      <td>0.182856</td>\n",
       "      <td>0.114262</td>\n",
       "      <td>0.092090</td>\n",
       "      <td>0.261859</td>\n",
       "      <td>0.097606</td>\n",
       "      <td>0.206104</td>\n",
       "      <td>0.187637</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>0.308729</td>\n",
       "      <td>0.086705</td>\n",
       "      <td>0.039767</td>\n",
       "      <td>0.078812</td>\n",
       "      <td>0.299619</td>\n",
       "      <td>0.375963</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.211282</td>\n",
       "      <td>0.107795</td>\n",
       "      <td>0.328923</td>\n",
       "      <td>...</td>\n",
       "      <td>0.290002</td>\n",
       "      <td>0.074170</td>\n",
       "      <td>0.094619</td>\n",
       "      <td>0.084235</td>\n",
       "      <td>0.115620</td>\n",
       "      <td>0.100625</td>\n",
       "      <td>0.233843</td>\n",
       "      <td>0.039199</td>\n",
       "      <td>0.224227</td>\n",
       "      <td>0.296332</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>0.224269</td>\n",
       "      <td>0.078864</td>\n",
       "      <td>0.089474</td>\n",
       "      <td>0.095354</td>\n",
       "      <td>0.163724</td>\n",
       "      <td>0.131795</td>\n",
       "      <td>0.211282</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.037040</td>\n",
       "      <td>0.183375</td>\n",
       "      <td>...</td>\n",
       "      <td>0.165008</td>\n",
       "      <td>0.066843</td>\n",
       "      <td>0.058766</td>\n",
       "      <td>0.068759</td>\n",
       "      <td>0.087159</td>\n",
       "      <td>0.129381</td>\n",
       "      <td>0.188662</td>\n",
       "      <td>0.121223</td>\n",
       "      <td>0.083910</td>\n",
       "      <td>0.273238</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>0.026017</td>\n",
       "      <td>0.068940</td>\n",
       "      <td>0.078162</td>\n",
       "      <td>0.059498</td>\n",
       "      <td>0.038474</td>\n",
       "      <td>0.110944</td>\n",
       "      <td>0.107795</td>\n",
       "      <td>0.037040</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>0.155435</td>\n",
       "      <td>...</td>\n",
       "      <td>0.011708</td>\n",
       "      <td>0.000000</td>\n",
       "      <td>0.101710</td>\n",
       "      <td>0.034568</td>\n",
       "      <td>0.045002</td>\n",
       "      <td>0.052699</td>\n",
       "      <td>0.107486</td>\n",
       "      <td>0.055766</td>\n",
       "      <td>0.070065</td>\n",
       "      <td>0.088281</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10</th>\n",
       "      <td>0.286411</td>\n",
       "      <td>0.092399</td>\n",
       "      <td>0.037670</td>\n",
       "      <td>0.053879</td>\n",
       "      <td>0.153021</td>\n",
       "      <td>0.400758</td>\n",
       "      <td>0.328923</td>\n",
       "      <td>0.183375</td>\n",
       "      <td>0.155435</td>\n",
       "      <td>1.000000</td>\n",
       "      <td>...</td>\n",
       "      <td>0.278558</td>\n",
       "      <td>0.049310</td>\n",
       "      <td>0.153506</td>\n",
       "      <td>0.065471</td>\n",
       "      <td>0.060088</td>\n",
       "      <td>0.033686</td>\n",
       "      <td>0.197107</td>\n",
       "      <td>0.085402</td>\n",
       "      <td>0.118945</td>\n",
       "      <td>0.162538</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10 rows × 943 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "user_id       1         2         3         4         5         6         7    \\\n",
       "user_id                                                                         \n",
       "1        1.000000  0.118076  0.029097  0.011628  0.264677  0.312419  0.308729   \n",
       "2        0.118076  1.000000  0.099097  0.107680  0.034279  0.152789  0.086705   \n",
       "3        0.029097  0.099097  1.000000  0.252131  0.026893  0.062539  0.039767   \n",
       "4        0.011628  0.107680  0.252131  1.000000  0.000000  0.045543  0.078812   \n",
       "5        0.264677  0.034279  0.026893  0.000000  1.000000  0.202843  0.299619   \n",
       "6        0.312419  0.152789  0.062539  0.045543  0.202843  1.000000  0.375963   \n",
       "7        0.308729  0.086705  0.039767  0.078812  0.299619  0.375963  1.000000   \n",
       "8        0.224269  0.078864  0.089474  0.095354  0.163724  0.131795  0.211282   \n",
       "9        0.026017  0.068940  0.078162  0.059498  0.038474  0.110944  0.107795   \n",
       "10       0.286411  0.092399  0.037670  0.053879  0.153021  0.400758  0.328923   \n",
       "\n",
       "user_id       8         9         10   ...       934       935       936  \\\n",
       "user_id                                ...                                 \n",
       "1        0.224269  0.026017  0.286411  ...  0.308475  0.055872  0.197862   \n",
       "2        0.078864  0.068940  0.092399  ...  0.086927  0.259636  0.289092   \n",
       "3        0.089474  0.078162  0.037670  ...  0.040918  0.019031  0.065417   \n",
       "4        0.095354  0.059498  0.053879  ...  0.024226  0.050703  0.056561   \n",
       "5        0.163724  0.038474  0.153021  ...  0.262547  0.048524  0.048312   \n",
       "6        0.131795  0.110944  0.400758  ...  0.287549  0.080312  0.162988   \n",
       "7        0.211282  0.107795  0.328923  ...  0.290002  0.074170  0.094619   \n",
       "8        1.000000  0.037040  0.183375  ...  0.165008  0.066843  0.058766   \n",
       "9        0.037040  1.000000  0.155435  ...  0.011708  0.000000  0.101710   \n",
       "10       0.183375  0.155435  1.000000  ...  0.278558  0.049310  0.153506   \n",
       "\n",
       "user_id       937       938       939       940       941       942       943  \n",
       "user_id                                                                        \n",
       "1        0.131367  0.152449  0.084456  0.293293  0.056765  0.103536  0.326491  \n",
       "2        0.318824  0.149105  0.186347  0.168034  0.106748  0.136796  0.080358  \n",
       "3        0.055373  0.086503  0.018418  0.096993  0.109631  0.092574  0.018987  \n",
       "4        0.107294  0.098892  0.000000  0.132900  0.142798  0.097066  0.015176  \n",
       "5        0.022202  0.091910  0.066000  0.156172  0.115842  0.124297  0.267574  \n",
       "6        0.182856  0.114262  0.092090  0.261859  0.097606  0.206104  0.187637  \n",
       "7        0.084235  0.115620  0.100625  0.233843  0.039199  0.224227  0.296332  \n",
       "8        0.068759  0.087159  0.129381  0.188662  0.121223  0.083910  0.273238  \n",
       "9        0.034568  0.045002  0.052699  0.107486  0.055766  0.070065  0.088281  \n",
       "10       0.065471  0.060088  0.033686  0.197107  0.085402  0.118945  0.162538  \n",
       "\n",
       "[10 rows x 943 columns]"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Convert into pandas dataframe \n",
    "cosine_sim = pd.DataFrame(cosine_sim, index=r_matrix.index, columns=r_matrix.index)\n",
    "\n",
    "cosine_sim.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "#User Based Collaborative Filter using Weighted Mean Ratings\n",
    "def cf_user_wmean(user_id, movie_id):\n",
    "    \n",
    "    #Check if movie_id exists in r_matrix\n",
    "    if movie_id in r_matrix:\n",
    "        \n",
    "        #Get the similarity scores for the user in question with every other user\n",
    "        sim_scores = cosine_sim[user_id]\n",
    "        \n",
    "        #Get the user ratings for the movie in question\n",
    "        m_ratings = r_matrix[movie_id]\n",
    "        \n",
    "        #Extract the indices containing NaN in the m_ratings series\n",
    "        idx = m_ratings[m_ratings.isnull()].index\n",
    "        \n",
    "        #Drop the NaN values from the m_ratings Series\n",
    "        m_ratings = m_ratings.dropna()\n",
    "        \n",
    "        #Drop the corresponding cosine scores from the sim_scores series\n",
    "        sim_scores = sim_scores.drop(idx)\n",
    "        \n",
    "        #Compute the final weighted mean\n",
    "        wmean_rating = np.dot(sim_scores, m_ratings)/ sim_scores.sum()\n",
    "    \n",
    "    else:\n",
    "        #Default to a rating of 3.0 in the absence of any information\n",
    "        wmean_rating = 3.0\n",
    "    \n",
    "    return wmean_rating"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "1.0174483808407588"
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "score(cf_user_wmean)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "- OK, results are matached with the book!\n",
    "- on this workbook, I made a starting error on data!"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}