MMAI 891 Tutorial 1 Embedding.ipynb

{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# MMAI 891 Tutorial on Word Embeddings"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "This tutorial includes 2 parts\n",
    "1. Learning an embedding using Keras\n",
    "2. Use a pre-trained word embeddings (GloVe)\n",
    "\n",
    "We will then compare the accuarcy of our classification model between using our own word embeddings verses a pre-trained one (GloVe)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 1 - Learning an Embedding in Keras"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Install packages if you haven't already\n",
    "#!pip install -q keras\n",
    "#!pip install -q tensorflow\n",
    "#!pip install -q pyyaml \n",
    "#!pip install -q seaborn"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "C:\\Users\\st50\\AppData\\Local\\Continuum\\anaconda3\\lib\\site-packages\\h5py\\__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.\n",
      "  from ._conv import register_converters as _register_converters\n",
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "import seaborn as sns\n",
    "from sklearn.model_selection import train_test_split\n",
    "from sklearn.preprocessing import LabelEncoder\n",
    "from keras.models import Model, Sequential\n",
    "from keras.layers import LSTM, Activation, Dense, Dropout, Input, Embedding, Bidirectional\n",
    "from keras.optimizers import RMSprop\n",
    "from keras.preprocessing.text import Tokenizer, one_hot\n",
    "from keras.preprocessing import sequence\n",
    "from keras.utils import to_categorical\n",
    "from keras.callbacks import EarlyStopping\n",
    "\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load the file\n",
    "df = pd.read_csv('data/sentiment_train.csv', delimiter=',',encoding='latin-1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Sentence</th>\n",
       "      <th>Polarity</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Wow... Loved this place.</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>Crust is not good.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>Not tasty and the texture was just nasty.</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Stopped by during the late May bank holiday of...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>The selection on the menu was great and so wer...</td>\n",
       "      <td>1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                            Sentence  Polarity\n",
       "0                           Wow... Loved this place.         1\n",
       "1                                 Crust is not good.         0\n",
       "2          Not tasty and the texture was just nasty.         0\n",
       "3  Stopped by during the late May bank holiday of...         1\n",
       "4  The selection on the menu was great and so wer...         1"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Preview the file\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Text(0.5, 0, 'Label')"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAZMAAAEPCAYAAACHuClZAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAFOFJREFUeJzt3X9MVff9x/HXwYtUh61K7xXCjMlcOxLsZPnadazu2i6roJTo7kzmj412W+vK1E63YSgwCCYGa4g0jaFZ+jX+QawZZQqO4aXL3MyQblOytMPSOBehFc3lomD5IQzuvd8/9u2dVIULH++90D4fidFz7rmeN8nJfXLP4R6sQCAQEAAABmKiPQAAYOYjJgAAY8QEAGCMmAAAjBETAIAxYgIAMEZMAADGiAkAwBgxAQAYIyYAAGPEBABgjJgAAIwREwCAMVu0Bwi3np4B+f3cGBkAQhETY2nBgs9N+nmf+pj4/QFiAgBhxmkuAIAxYgIAMEZMAADGiAkAwBgxAQAYIyYAAGPEBABg7FP/ORNT8+6/T/fFxUZ7DEwzQ8Mj6vtoKNpjANMGMZnAfXGx2rz7SLTHwDTzxv4t6hMxAT7GaS4AgDFiAgAwRkwAAMa4ZgLMUAsemC3b7Lhoj4FpZvTfw+q58e+I75eYADOUbXacWvY/F+0xMM38z+7/lRT5mHCaCwBgjJgAAIwREwCAMWICADBGTAAAxogJAMAYMQEAGCMmAABjxAQAYIyYAACMERMAgLGwxqS/v19PP/20Ll++LElqbm5Wdna2Vq9erYqKiuB2bW1tcrlcysjIUGFhoUZHRyVJV65c0ZYtW5SZmanc3FwNDAyEc1wAwBSFLSbvvPOONm3apPb2dknS0NCQCgoKVFlZqYaGBrW2tur06dOSpLy8PBUXF6uxsVGBQEDV1dWSpNLSUm3evFlut1vLli1TZWVluMYFABgIW0yqq6tVUlIih8MhSXr33Xe1ZMkSLV68WDabTdnZ2XK73ers7NTQ0JDS0tIkSS6XS263WyMjIzp79qwyMjLGrAcATD9huwX93r17xyx3dXXJbrcHlx0Ohzwez23r7Xa7PB6Penp6FB8fL5vNNmb9ZCUkxE/xKwDGZ7fPi/YIwB1F49iM2O8z8fv9siwruBwIBGRZ1l3Xf/z3rT65HIpr1/rl9wemPDcvGLgbr7cvqvvn2MTdmBybMTHWlL4Jj9hPcyUmJsrr9QaXvV6vHA7Hbeu7u7vlcDi0cOFC9fX1yefzjdkeADD9RCwmy5cv16VLl9TR0SGfz6f6+no5nU4lJycrLi5OLS0tkqS6ujo5nU7FxsZqxYoVamhokCTV1tbK6XRGalwAwCRE7DRXXFyc9u3bpx07dmh4eFirVq1SZmamJKm8vFxFRUXq7+9XamqqcnJyJEklJSXKz8/Xa6+9pqSkJB04cCBS4wIAJiHsMTl16lTw3+np6Tpx4sRt26SkpKimpua29cnJyaqqqgrrfAAAc3wCHgBgjJgAAIwREwCAMWICADBGTAAAxogJAMAYMQEAGCMmAABjxAQAYIyYAACMERMAgDFiAgAwRkwAAMaICQDAGDEBABgjJgAAY8QEAGCMmAAAjBETAIAxYgIAMEZMAADGiAkAwBgxAQAYIyYAAGPEBABgjJgAAIwREwCAMWICADAWlZjU1dUpKytLWVlZevnllyVJbW1tcrlcysjIUGFhoUZHRyVJV65c0ZYtW5SZmanc3FwNDAxEY2QAwDgiHpObN29q7969qqqqUl1dnc6dO6fm5mbl5eWpuLhYjY2NCgQCqq6uliSVlpZq8+bNcrvdWrZsmSorKyM9MgBgAhGPic/nk9/v182bNzU6OqrR0VHZbDYNDQ0pLS1NkuRyueR2uzUyMqKzZ88qIyNjzHoAwPRii/QO4+Pj9dOf/lRr1qzRnDlz9Oijjyo2NlZ2uz24jd1ul8fjUU9Pj+Lj42Wz2casn4yEhPh7Oj/wMbt9XrRHAO4oGsdmxGPy/vvv6ze/+Y3++Mc/at68efrFL36hM2fOyLKs4DaBQECWZQX/vtUnlydy7Vq//P7AlOflBQN34/X2RXX/HJu4G5NjMybGmtI34RE/zdXU1KT09HQlJCRo9uzZcrlc+utf/yqv1xvcpru7Ww6HQwsXLlRfX598Pp8kyev1yuFwRHpkAMAEIh6TlJQUNTc3a3BwUIFAQKdOndJXv/pVxcXFqaWlRdJ/ftrL6XQqNjZWK1asUENDgySptrZWTqcz0iMDACYQ8dNcK1eu1HvvvSeXy6XY2Fg98sgj2rp1q5566ikVFRWpv79fqampysnJkSSVlJQoPz9fr732mpKSknTgwIFIjwwAmEDEYyJJW7du1datW8esS0lJUU1NzW3bJicnq6qqKlKjAQCmgE/AAwCMERMAgDFiAgAwRkwAAMaICQDAGDEBABgjJgAAY8QEAGCMmAAAjBETAIAxYgIAMEZMAADGiAkAwBgxAQAYIyYAAGPEBABgjJgAAIwREwCAMWICADBGTAAAxogJAMAYMQEAGAspJh6P57Z1Fy9evOfDAABmpnFj0tvbq97eXj3//PO6ceNGcLm7u1vbt2+P1IwAgGnONt6DP//5z3XmzBlJ0mOPPfbfJ9lsysjICO9kAIAZY9yYHDp0SJL00ksvqaysLCIDAQBmnnFj8rGysjJ1dnbqxo0bCgQCwfWpqalhGwwAMHOEFJNXX31Vhw4dUkJCQnCdZVn6wx/+ELbBAAAzR0gxqa2t1VtvvaVFixbdk52eOnVKBw8e1M2bN/X444+rqKhIzc3NKisr0/DwsNasWaNdu3ZJktra2lRYWKiBgQGtWLFCpaWlstlCGhsAECEh/WhwUlLSPQvJhx9+qJKSElVWVurEiRN67733dPr0aRUUFKiyslINDQ1qbW3V6dOnJUl5eXkqLi5WY2OjAoGAqqur78kcAIB7J6SYpKena//+/WppadH58+eDf6bi97//vdauXavExETFxsaqoqJCc+bM0ZIlS7R48WLZbDZlZ2fL7Xars7NTQ0NDSktLkyS5XC653e4p7RcAED4hnS86duyYJI15IZ/qNZOOjg7FxsbqhRde0NWrV/XEE0/ooYcekt1uD27jcDjk8XjU1dU1Zr3dbr/jBygBANEVUkxOnTp1z3bo8/l07tw5VVVVae7cucrNzdV9990ny7KC2wQCAVmWJb/ff8f1k5GQEH/PZgduZbfPi/YIwB1F49gMKSaHDx++4/of/OAHk97hgw8+qPT0dC1cuFCS9K1vfUtut1uzZs0KbuP1euVwOJSYmCiv1xtc393dLYfDMan9XbvWL78/MPGGd8ELBu7G6+2L6v45NnE3JsdmTIw1pW/CQ7pmcuHCheCf1tZWHT58WO+///6kdyZJTz75pJqamvTRRx/J5/Ppz3/+szIzM3Xp0iV1dHTI5/Opvr5eTqdTycnJiouLU0tLiySprq5OTqdzSvsFAIRPyB9avJXH41FhYeGUdrh8+XI999xz2rx5s0ZGRvT4449r06ZN+sIXvqAdO3ZoeHhYq1atUmZmpiSpvLxcRUVF6u/vV2pqqnJycqa0XwBA+EzpAxuLFi1SZ2fnlHe6YcMGbdiwYcy69PR0nThx4rZtU1JSVFNTM+V9AQDCb9LXTAKBgFpbW8d8Gh4A8NkWUkwuXLgwZjkpKUm7d+8Oy0AAgJlnUtdMOjs7NTo6qiVLloR1KADAzBJSTDo6OvSTn/xEXV1d8vv9WrBggX71q19p6dKl4Z4PADADhPSjwXv27NFzzz2ns2fPqqWlRbm5uSotLQ33bACAGSKkmFy7dk3f/va3g8vf+c531NPTE7ahAAAzS0gx8fl86u3tDS5fv349bAMBAGaekK6ZfO9739N3v/tdrVmzRpZlqaGhQc8880y4ZwMAzBAhvTNZtWqVJGlkZET/+te/5PF49NRTT4V1MADAzBHSO5P8/Hxt2bJFOTk5Gh4e1tGjR1VQUKDXX3893PMBAGaAkN6Z9PT0BO+JFRcXp2effXbM3XwBAJ9tIV+Av/WXUnV3dysQmPpt3QEAny4hneZ69tlntX79en3jG9+QZVlqbm7mdioAgKCQYrJhwwYtW7ZMf/nLXzRr1iz96Ec/0sMPPxzu2QAAM0TIt6BPSUlRSkpKOGcBAMxQIV0zAQBgPMQEAGCMmAAAjBETAIAxYgIAMEZMAADGiAkAwBgxAQAYIyYAAGPEBABgjJgAAIwREwCAMWICADAWtZi8/PLLys/PlyS1tbXJ5XIpIyNDhYWFGh0dlSRduXJFW7ZsUWZmpnJzczUwMBCtcQEA44hKTN5++20dP348uJyXl6fi4mI1NjYqEAiourpaklRaWqrNmzfL7XZr2bJlqqysjMa4AIAJRDwmvb29qqio0AsvvCBJ6uzs1NDQkNLS0iRJLpdLbrdbIyMjOnv2rDIyMsasBwBMPxGPSXFxsXbt2qX7779fktTV1SW73R583G63y+PxqKenR/Hx8bLZbGPWAwCmn5B/0+K98OabbyopKUnp6ek6duyYJMnv98uyrOA2gUBAlmUF/77VJ5dDkZAQbzY0cBd2+7xojwDcUTSOzYjGpKGhQV6vV+vWrdONGzc0ODgoy7Lk9XqD23R3d8vhcGjhwoXq6+uTz+fTrFmz5PV65XA4Jr3Pa9f65fcHpjwzLxi4G6+3L6r759jE3ZgcmzEx1pS+CY/oaa7Dhw+rvr5edXV1evHFF/XNb35TZWVliouLU0tLiySprq5OTqdTsbGxWrFihRoaGiRJtbW1cjqdkRwXABCiafE5k/LycpWVlSkzM1ODg4PKycmRJJWUlKi6ulpr167VuXPntHPnzihPCgC4k4ie5rqVy+WSy+WSJKWkpKimpua2bZKTk1VVVRXp0QAAkzQt3pkAAGY2YgIAMEZMAADGiAkAwBgxAQAYIyYAAGPEBABgjJgAAIwREwCAMWICADBGTAAAxogJAMAYMQEAGCMmAABjxAQAYIyYAACMERMAgDFiAgAwRkwAAMaICQDAGDEBABgjJgAAY8QEAGCMmAAAjBETAIAxYgIAMEZMAADGiAkAwFhUYnLw4EFlZWUpKytL+/fvlyQ1NzcrOztbq1evVkVFRXDbtrY2uVwuZWRkqLCwUKOjo9EYGQAwjojHpLm5WU1NTTp+/Lhqa2t1/vx51dfXq6CgQJWVlWpoaFBra6tOnz4tScrLy1NxcbEaGxsVCARUXV0d6ZEBABOIeEzsdrvy8/M1e/ZsxcbGaunSpWpvb9eSJUu0ePFi2Ww2ZWdny+12q7OzU0NDQ0pLS5MkuVwuud3uSI8MAJhAxGPy0EMPBePQ3t6ukydPyrIs2e324DYOh0Mej0ddXV1j1tvtdnk8nkiPDACYgC1aO/7nP/+pH//4x9q9e7dmzZql9vb24GOBQECWZcnv98uyrNvWT0ZCQvy9GhkYw26fF+0RgDuKxrEZlZi0tLToxRdfVEFBgbKysvS3v/1NXq83+LjX65XD4VBiYuKY9d3d3XI4HJPa17Vr/fL7A1OelRcM3I3X2xfV/XNs4m5Mjs2YGGtK34RH/DTX1atXtW3bNpWXlysrK0uStHz5cl26dEkdHR3y+Xyqr6+X0+lUcnKy4uLi1NLSIkmqq6uT0+mM9MgAgAlE/J3JoUOHNDw8rH379gXXbdy4Ufv27dOOHTs0PDysVatWKTMzU5JUXl6uoqIi9ff3KzU1VTk5OZEeGQAwgYjHpKioSEVFRXd87MSJE7etS0lJUU1NTbjHAgAY4BPwAABjxAQAYIyYAACMERMAgDFiAgAwRkwAAMaICQDAGDEBABgjJgAAY8QEAGCMmAAAjBETAIAxYgIAMEZMAADGiAkAwBgxAQAYIyYAAGPEBABgjJgAAIwREwCAMWICADBGTAAAxogJAMAYMQEAGCMmAABjxAQAYIyYAACMERMAgDFiAgAwNiNi8tvf/lZr167V6tWrdeTIkWiPAwD4BFu0B5iIx+NRRUWFjh07ptmzZ2vjxo167LHH9MUvfjHaowEA/t+0j0lzc7O+9rWvaf78+ZKkjIwMud1ubd++PaTnx8RYxjM8uOBzxv8HPn3uxbFlavb9CdEeAdOQybE51edO+5h0dXXJbrcHlx0Oh959992Qn7/gHoTg1ZfWG/8f+PRJSIiP9gh65IWXoz0CpqFoHJvT/pqJ3++XZf23lIFAYMwyACD6pn1MEhMT5fV6g8ter1cOhyOKEwEAPmnax+TrX/+63n77bV2/fl03b97UW2+9JafTGe2xAAC3mPbXTBYtWqRdu3YpJydHIyMj2rBhg7785S9HeywAwC2sQCAQiPYQAICZbdqf5gIATH/EBABgjJgAAIwREwCAMWKCCXGjTUxn/f39evrpp3X58uVoj/KZRkwwro9vtPnGG2+otrZWv/71r3Xx4sVojwVIkt555x1t2rRJ7e3t0R7lM4+YYFy33mhz7ty5wRttAtNBdXW1SkpKuCvGNDDtP7SI6DK90SYQTnv37o32CPh/vDPBuLjRJoBQEBOMixttAggFMcG4uNEmgFBwzQTj4kabAELBjR4BAMY4zQUAMEZMAADGiAkAwBgxAQAYIyYAAGPEBDB0+fJlfeUrX5nUc770pS/p+vXrk3pOfn6+Dh06NKnnAJFCTAAAxvjQIhAmly5d0p49ezQwMCCv16uUlBS98soriouLkyS98sor+sc//iG/36+dO3fqySeflCS9+eabOnr0qPx+v+bPn69f/vKXWrp0aTS/FGBCxAQIk+rqaq1fv17r1q3TyMiIXC6X/vSnPykjI0OS9PnPf1579uzRhQsX9P3vf18nT57UxYsXVVtbqyNHjmjOnDlqamrS9u3bdfLkySh/NcD4iAkQJnl5eTpz5oxef/11tbe3q6urS4ODg8HHN23aJEl6+OGHtXTpUv39739XS0uLOjo6tHHjxuB2H330kXp7eyM+PzAZxAQIk5/97Gfy+Xxas2aNnnjiCV29elW33r0oJua/lyz9fr9sNpv8fr/WrVunvLy84Pquri498MADEZ8fmAwuwANh0tTUpG3btmnt2rWS/vMrZn0+X/Dx48ePS5LOnz+vDz74QMuXL9fKlSv1u9/9Tl1dXZKko0eP6plnnon88MAk8c4EuAcGBwdv+/HgnTt3atu2bZo7d67i4+P16KOP6oMPPgg+/uGHH2r9+vWyLEsHDhzQ/PnztXLlSj3//PP64Q9/KMuyFB8fr4MHD/ILyTDtcddgAIAxTnMBAIwREwCAMWICADBGTAAAxogJAMAYMQEAGCMmAABjxAQAYOz/AP37F/vP997aAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "#See the distribution of the label\n",
    "\n",
    "sns.set(style=\"darkgrid\")\n",
    "ax = sns.countplot(x=\"Polarity\", data=df)\n",
    "plt.xlabel('Label')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Define the documents and the labels\n",
    "X = df.Sentence\n",
    "y = df.Polarity\n",
    "\n",
    "le = LabelEncoder()\n",
    "y = le.fit_transform(y)\n",
    "y = y.reshape(-1,1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2202, 1)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([[1],\n",
       "       [0],\n",
       "       [0],\n",
       "       ...,\n",
       "       [0],\n",
       "       [1],\n",
       "       [1]], dtype=int64)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#check dimensions of the labels\n",
    "y.shape\n",
    "y"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2202,)"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "0                             Wow... Loved this place.\n",
       "1                                   Crust is not good.\n",
       "2            Not tasty and the texture was just nasty.\n",
       "3    Stopped by during the late May bank holiday of...\n",
       "4    The selection on the menu was great and so wer...\n",
       "5       Now I am getting angry and I want my damn pho.\n",
       "6                Honeslty it didn't taste THAT fresh.)\n",
       "7    The potatoes were like rubber and you could te...\n",
       "8                            The fries were great too.\n",
       "9                                       A great touch.\n",
       "Name: Sentence, dtype: object"
      ]
     },
     "execution_count": 8,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#check dimensions of the do`cuments\n",
    "X.shape\n",
    "X[0:10,]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 76,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0                                      wow loved place\n",
       "1                                       crust not good\n",
       "2                              not tasty texture nasty\n",
       "3    stopped late may bank holiday rick steve recom...\n",
       "4                           selection menu great price\n",
       "Name: Sentence, dtype: object"
      ]
     },
     "execution_count": 76,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from nltk.corpus import stopwords\n",
    "from nltk.stem import WordNetLemmatizer\n",
    "import re\n",
    "import unidecode\n",
    "\n",
    "stop_words = set(stopwords.words('english'))\n",
    "stop_words.remove(\"not\")\n",
    "\n",
    "lemmer = WordNetLemmatizer()\n",
    "\n",
    "def preprocess(x):\n",
    "    x = x.lower()\n",
    "    \n",
    "    x = re.sub(r'[^\\w\\s]', '', x)\n",
    "    \n",
    "    x = unidecode.unidecode(x)\n",
    "    \n",
    "    x = re.sub(r'\\d+', '', x)\n",
    "    \n",
    "    x = [lemmer.lemmatize(w) for w in x.split() if w not in stop_words]\n",
    "\n",
    "    return ' '.join(x)\n",
    "\n",
    "X_bak = X.copy()\n",
    "X = X.apply(preprocess)\n",
    "X.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 77,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Split data for test and train\n",
    "X_train,X_test,y_train, y_test = train_test_split (X, y,  test_size=0.15, random_state=99) #remember to set a random_state for the split"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now that we have the document split between test and train, we can start building our word embedding"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 78,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Start with creating the sequence matrix\n",
    "\n",
    "max_words = 1000 #to define vocab size for max num of words to keep based on word freq, here we are only keeping the 1000-1 most common words\n",
    "max_len = 150 #to define fixed sequence length, here we are padding the input sequence to have the same length of 150\n",
    "\n",
    "#vectorize the corpus\n",
    "tok = Tokenizer(num_words=max_words) \n",
    "tok.fit_on_texts(X_train) #Updates internal vocabulary based on a list of texts. This method creates the vocabulary index based on word frequency\n",
    "sequences = tok.texts_to_sequences(X_train) #Transforms each text in texts to a sequence of integers.\n",
    "sequences_matrix = sequence.pad_sequences(sequences, maxlen=max_len) #pad the vector so they are all the same length of 150\n",
    "\n",
    "#we will use this sequences_matrix when we train the model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 79,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1871, 150)"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0, 424, 234, 274,   2, 890,  18])"
      ]
     },
     "execution_count": 79,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#check the dimension of our vectorized training data\n",
    "sequences_matrix.shape \n",
    "sequences_matrix[0,:] #check if the array has the same elements as defined max_len of 150.  Yes? GOOD"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 80,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Show the 1000 most frequent words included in the matrix\n",
    "word_index = tok.word_index"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 81,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "82"
      ]
     },
     "execution_count": 81,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#Find the index for 'happy'\n",
    "word_index['happy']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "sequences :  [250, 703, 104, 24, 93] \n",
      "\n"
     ]
    }
   ],
   "source": [
    "# print the ids of the words in document 159\n",
    "print (\"sequences : \", sequences[159], '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'id advise anyone go see'"
      ]
     },
     "execution_count": 83,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X_train.iloc[159]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 85,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['id', 'advise', 'anyone', 'go', 'see']"
      ]
     },
     "execution_count": 85,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "[tok.index_word[i] for i in sequences[159]]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "_________________________________________________________________\n",
      "Layer (type)                 Output Shape              Param #   \n",
      "=================================================================\n",
      "embedding_2 (Embedding)      (None, 150, 32)           32000     \n",
      "_________________________________________________________________\n",
      "lstm_2 (LSTM)                (None, 64)                24832     \n",
      "_________________________________________________________________\n",
      "out_layer (Dense)            (None, 1)                 65        \n",
      "=================================================================\n",
      "Total params: 56,897\n",
      "Trainable params: 56,897\n",
      "Non-trainable params: 0\n",
      "_________________________________________________________________\n"
     ]
    }
   ],
   "source": [
    "#define the Embedding layer as part of our neural network model\n",
    "model1 = Sequential()\n",
    "\n",
    "model1.add(Embedding(max_words, 32, input_length=max_len, mask_zero=False))\n",
    "model1.add((LSTM(64, dropout=0.2, recurrent_dropout=0.2)))\n",
    "model1.add(Dense(1, name='out_layer', activation='sigmoid'))\n",
    "\n",
    "model1.summary()\n",
    "model1.compile(loss='binary_crossentropy',optimizer=RMSprop(), metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 87,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Train on 1496 samples, validate on 375 samples\n",
      "Epoch 1/10\n",
      "1496/1496 [==============================] - 4s 3ms/step - loss: 0.6907 - acc: 0.5541 - val_loss: 0.6868 - val_acc: 0.5760\n",
      "Epoch 2/10\n",
      "1496/1496 [==============================] - 2s 2ms/step - loss: 0.6763 - acc: 0.6771 - val_loss: 0.6677 - val_acc: 0.6667\n",
      "Epoch 3/10\n",
      "1496/1496 [==============================] - 2s 2ms/step - loss: 0.6420 - acc: 0.7380 - val_loss: 0.6245 - val_acc: 0.7787\n",
      "Epoch 4/10\n",
      "1496/1496 [==============================] - 2s 2ms/step - loss: 0.5974 - acc: 0.7914 - val_loss: 0.5801 - val_acc: 0.7840\n",
      "Epoch 5/10\n",
      "1496/1496 [==============================] - 2s 2ms/step - loss: 0.5287 - acc: 0.8235 - val_loss: 0.5302 - val_acc: 0.7840\n",
      "Epoch 6/10\n",
      "1496/1496 [==============================] - 2s 2ms/step - loss: 0.4913 - acc: 0.8342 - val_loss: 0.4946 - val_acc: 0.8027\n",
      "Epoch 7/10\n",
      "1496/1496 [==============================] - 2s 1ms/step - loss: 0.4310 - acc: 0.8590 - val_loss: 0.4570 - val_acc: 0.8347\n",
      "Epoch 8/10\n",
      "1496/1496 [==============================] - 2s 1ms/step - loss: 0.3831 - acc: 0.8790 - val_loss: 0.4261 - val_acc: 0.8320\n",
      "Epoch 9/10\n",
      "1496/1496 [==============================] - 2s 1ms/step - loss: 0.3442 - acc: 0.8890 - val_loss: 0.4005 - val_acc: 0.8320\n",
      "Epoch 10/10\n",
      "1496/1496 [==============================] - 2s 1ms/step - loss: 0.3034 - acc: 0.8930 - val_loss: 0.3946 - val_acc: 0.8347\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "<keras.callbacks.History at 0x1fa393c95c0>"
      ]
     },
     "execution_count": 87,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#train the model\n",
    "model1.fit(sequences_matrix, y_train, batch_size=128, epochs=10,\n",
    "          validation_split=0.2, callbacks=[EarlyStopping(monitor='val_loss',min_delta=0.0001)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 88,
   "metadata": {},
   "outputs": [],
   "source": [
    "#set up the test set\n",
    "test_sequences = tok.texts_to_sequences(X_test)\n",
    "test_sequences_matrix = sequence.pad_sequences(test_sequences, maxlen=max_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 89,
   "metadata": {},
   "outputs": [],
   "source": [
    "#show prediction: value closer to 1 is strong positive sentiment and value closer to 0 is a strong negative\n",
    "predict_1 = model1.predict(x=test_sequences_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 90,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(331, 1)"
      ]
     },
     "execution_count": 90,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "predict_1.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[0.15192512],\n",
       "       [0.53643584],\n",
       "       [0.7025749 ],\n",
       "       [0.19202891],\n",
       "       [0.9713885 ],\n",
       "       [0.42433485],\n",
       "       [0.89569753],\n",
       "       [0.8787063 ],\n",
       "       [0.64814806],\n",
       "       [0.10122192]], dtype=float32)"
      ]
     },
     "execution_count": 91,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#see the first 10 prediction\n",
    "predict_1[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 92,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "331/331 [==============================] - 0s 690us/step\n",
      "Test set\n",
      "  Loss: 0.452\n",
      "  Accuracy: 0.758\n"
     ]
    }
   ],
   "source": [
    "#evaluate with test set\n",
    "accr_1 = model1.evaluate(test_sequences_matrix, y_test)\n",
    "\n",
    "print('Test set\\n  Loss: {:0.3f}\\n  Accuracy: {:0.3f}'.format(accr_1[0],accr_1[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 93,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1871, 150)"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([  0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,   0,\n",
       "         0, 424, 234, 274,   2, 890,  18])"
      ]
     },
     "execution_count": 93,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sequences_matrix.shape\n",
    "sequences_matrix[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 94,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "keras_preprocessing.text.Tokenizer"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "1871"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "'not'"
      ]
     },
     "execution_count": 94,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "type(tok)\n",
    "tok.document_count\n",
    "tok.index_word[1]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 95,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000, 32)"
      ]
     },
     "execution_count": 95,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#show the learned embeddings \n",
    "embeds = model1.get_layer(index=0).get_weights()[0]\n",
    "embeds.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 96,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.03623868, -0.03221986, -0.04394674,  0.01576995, -0.02581737,\n",
       "        0.00620955,  0.00877435,  0.03963459,  0.00567459,  0.00242022,\n",
       "       -0.02872797, -0.01289139,  0.04647105, -0.03794513, -0.0528491 ,\n",
       "        0.0020562 , -0.02260046, -0.0381529 ,  0.01511889,  0.00238416,\n",
       "        0.02426012,  0.03637074,  0.02622405, -0.02957919,  0.02037595,\n",
       "        0.04848439,  0.0317261 , -0.02074072, -0.00067305, -0.00518522,\n",
       "        0.03326844, -0.04055934], dtype=float32)"
      ]
     },
     "execution_count": 96,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#embedding for word 1\n",
    "\n",
    "embeds[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 97,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.00630081, -0.05250498, -0.00825288, -0.05047157, -0.06607118,\n",
       "       -0.01574466, -0.03132604, -0.03279191, -0.00745375,  0.04764229,\n",
       "       -0.06242871,  0.01799775,  0.03421138,  0.04111866,  0.05978788,\n",
       "       -0.03788604,  0.00065762,  0.00821518,  0.07058298, -0.06028337,\n",
       "       -0.03565179, -0.02640403, -0.03452161, -0.01225196, -0.02872601,\n",
       "        0.03365713, -0.05572059,  0.01647411, -0.00650643, -0.00622066,\n",
       "        0.02364117,  0.01745653], dtype=float32)"
      ]
     },
     "execution_count": 97,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#embedding for word 2\n",
    "embeds[100]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 106,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(1000, 1000)"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "numpy.ndarray"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([ 92,  67,  31, 216, 236], dtype=int64)"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "array([0.8882027 , 0.8912338 , 0.90777194, 0.8996068 , 0.9999999 ],\n",
       "      dtype=float32)"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "['u', 'fit', 'love', 'crap', 'jabra']"
      ]
     },
     "execution_count": 106,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "sims =  cosine_similarity(embeds)\n",
    "sims.shape\n",
    "type(sims)\n",
    "\n",
    "wid = tok.word_index['lunch']\n",
    "\n",
    "ind = np.argpartition(sims[wid-1], -5)[-5:]\n",
    "ind\n",
    "sims[wid-1][ind]\n",
    "[tok.index_word[i-1] for i in ind]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 102,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "OrderedDict([('sweet', 5),\n",
       "             ('potato', 9),\n",
       "             ('fry', 8),\n",
       "             ('good', 162),\n",
       "             ('seasoned', 2),\n",
       "             ('well', 58),\n",
       "             ('ambience', 3),\n",
       "             ('perfect', 12),\n",
       "             ('ive', 51),\n",
       "             ('never', 38),\n",
       "             ('treated', 2),\n",
       "             ('bad', 43),\n",
       "             ('show', 5),\n",
       "             ('samsung', 5),\n",
       "             ('warm', 6),\n",
       "             ('beer', 7),\n",
       "             ('didnt', 26),\n",
       "             ('help', 5),\n",
       "             ('comfortable', 16),\n",
       "             ('cumbersome', 2),\n",
       "             ('design', 11),\n",
       "             ('nice', 37),\n",
       "             ('headphone', 6),\n",
       "             ('price', 36),\n",
       "             ('work', 80),\n",
       "             ('great', 153),\n",
       "             ('best', 58),\n",
       "             ('headset', 50),\n",
       "             ('ever', 53),\n",
       "             ('dont', 60),\n",
       "             ('waste', 20),\n",
       "             ('one', 85),\n",
       "             ('yet', 7),\n",
       "             ('plantronincs', 1),\n",
       "             ('continues', 1),\n",
       "             ('use', 41),\n",
       "             ('flawed', 2),\n",
       "             ('charger', 17),\n",
       "             ('cool', 12),\n",
       "             ('movie', 77),\n",
       "             ('suffered', 1),\n",
       "             ('writing', 7),\n",
       "             ('needed', 7),\n",
       "             ('suspense', 3),\n",
       "             ('waited', 8),\n",
       "             ('thirty', 1),\n",
       "             ('minute', 25),\n",
       "             ('seated', 5),\n",
       "             ('although', 6),\n",
       "             ('vacant', 1),\n",
       "             ('table', 12),\n",
       "             ('folk', 4),\n",
       "             ('waiting', 7),\n",
       "             ('product', 52),\n",
       "             ('food', 99),\n",
       "             ('gooodd', 1),\n",
       "             ('sure', 15),\n",
       "             ('beat', 5),\n",
       "             ('using', 10),\n",
       "             ('finger', 1),\n",
       "             ('would', 64),\n",
       "             ('definitely', 25),\n",
       "             ('recommend', 37),\n",
       "             ('jabra', 9),\n",
       "             ('btv', 1),\n",
       "             ('looking', 8),\n",
       "             ('comfort', 3),\n",
       "             ('clarity', 5),\n",
       "             ('bar', 13),\n",
       "             ('vega', 17),\n",
       "             ('edge', 3),\n",
       "             ('seat', 3),\n",
       "             ('made', 36),\n",
       "             ('somewhat', 3),\n",
       "             ('afraid', 1),\n",
       "             ('go', 51),\n",
       "             ('car', 17),\n",
       "             ('end', 8),\n",
       "             ('night', 14),\n",
       "             ('music', 8),\n",
       "             ('film', 55),\n",
       "             ('really', 62),\n",
       "             ('sadly', 1),\n",
       "             ('gordon', 1),\n",
       "             ('ramseys', 1),\n",
       "             ('steak', 16),\n",
       "             ('place', 99),\n",
       "             ('shall', 1),\n",
       "             ('sharply', 1),\n",
       "             ('avoid', 6),\n",
       "             ('next', 10),\n",
       "             ('trip', 5),\n",
       "             ('lighting', 1),\n",
       "             ('dark', 2),\n",
       "             ('enough', 23),\n",
       "             ('set', 6),\n",
       "             ('mood', 2),\n",
       "             ('authentic', 5),\n",
       "             ('leather', 4),\n",
       "             ('shine', 1),\n",
       "             ('case', 35),\n",
       "             ('tried', 12),\n",
       "             ('talking', 2),\n",
       "             ('real', 17),\n",
       "             ('loud', 5),\n",
       "             ('shouting', 1),\n",
       "             ('telephone', 1),\n",
       "             ('get', 50),\n",
       "             ('old', 9),\n",
       "             ('still', 25),\n",
       "             ('told', 6),\n",
       "             ('wasnt', 16),\n",
       "             ('strong', 6),\n",
       "             ('rubberpetroleum', 1),\n",
       "             ('smell', 3),\n",
       "             ('unbearable', 1),\n",
       "             ('caused', 2),\n",
       "             ('return', 8),\n",
       "             ('fit', 24),\n",
       "             ('comfortably', 4),\n",
       "             ('came', 22),\n",
       "             ('three', 11),\n",
       "             ('size', 5),\n",
       "             ('earbud', 3),\n",
       "             ('tip', 4),\n",
       "             ('paying', 1),\n",
       "             ('hot', 7),\n",
       "             ('dog', 3),\n",
       "             ('look', 21),\n",
       "             ('like', 76),\n",
       "             ('kid', 6),\n",
       "             ('meal', 12),\n",
       "             ('wienerschnitzel', 1),\n",
       "             ('not', 219),\n",
       "             ('idea', 2),\n",
       "             ('today', 7),\n",
       "             ('second', 7),\n",
       "             ('time', 84),\n",
       "             ('lunch', 9),\n",
       "             ('buffet', 10),\n",
       "             ('pretty', 29),\n",
       "             ('going', 25),\n",
       "             ('back', 61),\n",
       "             ('plastic', 6),\n",
       "             ('break', 4),\n",
       "             ('easy', 15),\n",
       "             ('clip', 4),\n",
       "             ('thing', 36),\n",
       "             ('hate', 6),\n",
       "             ('mode', 1),\n",
       "             ('button', 10),\n",
       "             ('side', 15),\n",
       "             ('igo', 2),\n",
       "             ('anyone', 17),\n",
       "             ('different', 9),\n",
       "             ('brand', 2),\n",
       "             ('cell', 12),\n",
       "             ('phonesmp', 1),\n",
       "             ('player', 3),\n",
       "             ('family', 15),\n",
       "             ('continue', 2),\n",
       "             ('pairing', 3),\n",
       "             ('periodically', 1),\n",
       "             ('since', 16),\n",
       "             ('somehow', 2),\n",
       "             ('kept', 9),\n",
       "             ('dropping', 1),\n",
       "             ('battery', 40),\n",
       "             ('life', 16),\n",
       "             ('forty', 1),\n",
       "             ('five', 3),\n",
       "             ('vain', 1),\n",
       "             ('eat', 17),\n",
       "             ('money', 24),\n",
       "             ('phone', 144),\n",
       "             ('salad', 13),\n",
       "             ('realized', 2),\n",
       "             ('coming', 11),\n",
       "             ('soon', 11),\n",
       "             ('circumstance', 1),\n",
       "             ('top', 4),\n",
       "             ('list', 3),\n",
       "             ('started', 5),\n",
       "             ('tuna', 1),\n",
       "             ('sashimi', 3),\n",
       "             ('brownish', 1),\n",
       "             ('color', 8),\n",
       "             ('obviously', 7),\n",
       "             ('fresh', 13),\n",
       "             ('baby', 3),\n",
       "             ('owl', 1),\n",
       "             ('adorable', 3),\n",
       "             ('need', 11),\n",
       "             ('two', 23),\n",
       "             ('hand', 9),\n",
       "             ('operate', 1),\n",
       "             ('screenthis', 1),\n",
       "             ('software', 6),\n",
       "             ('interface', 1),\n",
       "             ('decade', 1),\n",
       "             ('cannot', 7),\n",
       "             ('compete', 1),\n",
       "             ('new', 25),\n",
       "             ('left', 10),\n",
       "             ('frustrated', 1),\n",
       "             ('fun', 7),\n",
       "             ('chef', 3),\n",
       "             ('star', 20),\n",
       "             ('plus', 7),\n",
       "             ('find', 13),\n",
       "             ('unit', 9),\n",
       "             ('come', 15),\n",
       "             ('earpad', 1),\n",
       "             ('onlyi', 1),\n",
       "             ('wont', 21),\n",
       "             ('returning', 2),\n",
       "             ('love', 44),\n",
       "             ('server', 17),\n",
       "             ('negligent', 1),\n",
       "             ('u', 19),\n",
       "             ('feel', 20),\n",
       "             ('unwelcome', 1),\n",
       "             ('suggest', 1),\n",
       "             ('completely', 10),\n",
       "             ('grossed', 2),\n",
       "             ('sat', 5),\n",
       "             ('another', 17),\n",
       "             ('ten', 2),\n",
       "             ('finally', 5),\n",
       "             ('gave', 8),\n",
       "             ('took', 10),\n",
       "             ('hour', 20),\n",
       "             ('restaurant', 26),\n",
       "             ('luke', 1),\n",
       "             ('sever', 1),\n",
       "             ('running', 5),\n",
       "             ('around', 14),\n",
       "             ('totally', 11),\n",
       "             ('overwhelmed', 1),\n",
       "             ('bother', 3),\n",
       "             ('file', 1),\n",
       "             ('browser', 1),\n",
       "             ('offer', 4),\n",
       "             ('option', 6),\n",
       "             ('needshandsfree', 1),\n",
       "             ('attached', 1),\n",
       "             ('gas', 1),\n",
       "             ('station', 4),\n",
       "             ('rarely', 1),\n",
       "             ('sign', 2),\n",
       "             ('directing', 3),\n",
       "             ('seems', 6),\n",
       "             ('pretentious', 4),\n",
       "             ('everything', 25),\n",
       "             ('acting', 11),\n",
       "             ('cinematography', 4),\n",
       "             ('solid', 3),\n",
       "             ('jalapeno', 1),\n",
       "             ('bacon', 5),\n",
       "             ('soooo', 2),\n",
       "             ('blue', 4),\n",
       "             ('ant', 1),\n",
       "             ('chicken', 13),\n",
       "             ('dish', 14),\n",
       "             ('ok', 5),\n",
       "             ('beef', 5),\n",
       "             ('shoe', 1),\n",
       "             ('par', 1),\n",
       "             ('dennys', 1),\n",
       "             ('say', 26),\n",
       "             ('dropped', 8),\n",
       "             ('even', 42),\n",
       "             ('concrete', 1),\n",
       "             ('knock', 1),\n",
       "             ('wood', 1),\n",
       "             ('also', 45),\n",
       "             ('problem', 21),\n",
       "             ('reading', 3),\n",
       "             ('memory', 4),\n",
       "             ('card', 1),\n",
       "             ('always', 17),\n",
       "             ('turn', 9),\n",
       "             ('deliciously', 1),\n",
       "             ('outside', 4),\n",
       "             ('moist', 3),\n",
       "             ('inside', 5),\n",
       "             ('disgusting', 3),\n",
       "             ('highly', 13),\n",
       "             ('utter', 2),\n",
       "             ('crap', 10),\n",
       "             ('sound', 45),\n",
       "             ('quality', 60),\n",
       "             ('terrible', 19),\n",
       "             ('menu', 12),\n",
       "             ('terrific', 4),\n",
       "             ('thrilled', 2),\n",
       "             ('amazing', 25),\n",
       "             ('accommodation', 1),\n",
       "             ('vegetarian', 3),\n",
       "             ('daughter', 2),\n",
       "             ('happy', 21),\n",
       "             ('served', 6),\n",
       "             ('bread', 5),\n",
       "             ('butter', 2),\n",
       "             ('home', 6),\n",
       "             ('chip', 7),\n",
       "             ('bit', 13),\n",
       "             ('topvery', 1),\n",
       "             ('original', 5),\n",
       "             ('brunch', 3),\n",
       "             ('spot', 8),\n",
       "             ('muddy', 1),\n",
       "             ('low', 10),\n",
       "             ('casing', 1),\n",
       "             ('wire', 1),\n",
       "             ('insert', 1),\n",
       "             ('poorly', 4),\n",
       "             ('super', 9),\n",
       "             ('glued', 1),\n",
       "             ('slid', 1),\n",
       "             ('needle', 3),\n",
       "             ('wasted', 7),\n",
       "             ('earpiece', 6),\n",
       "             ('right', 22),\n",
       "             ('stay', 8),\n",
       "             ('ear', 35),\n",
       "             ('first', 33),\n",
       "             ('given', 7),\n",
       "             ('year', 23),\n",
       "             ('item', 16),\n",
       "             ('story', 11),\n",
       "             ('beyond', 4),\n",
       "             ('stupid', 9),\n",
       "             ('fast', 7),\n",
       "             ('service', 88),\n",
       "             ('broke', 8),\n",
       "             ('within', 5),\n",
       "             ('month', 11),\n",
       "             ('horrible', 15),\n",
       "             ('attitude', 2),\n",
       "             ('towards', 2),\n",
       "             ('customer', 14),\n",
       "             ('talk', 12),\n",
       "             ('enjoy', 8),\n",
       "             ('thats', 11),\n",
       "             ('rightthe', 1),\n",
       "             ('red', 4),\n",
       "             ('velvet', 1),\n",
       "             ('cakeohhh', 1),\n",
       "             ('stuff', 3),\n",
       "             ('trying', 7),\n",
       "             ('many', 15),\n",
       "             ('handsfree', 3),\n",
       "             ('gadget', 2),\n",
       "             ('breakfast', 5),\n",
       "             ('damn', 3),\n",
       "             ('attempting', 1),\n",
       "             ('artiness', 1),\n",
       "             ('black', 11),\n",
       "             ('white', 6),\n",
       "             ('clever', 3),\n",
       "             ('camera', 14),\n",
       "             ('angle', 2),\n",
       "             ('disappointed', 27),\n",
       "             ('became', 1),\n",
       "             ('ridiculous', 3),\n",
       "             ('poor', 22),\n",
       "             ('plot', 10),\n",
       "             ('line', 12),\n",
       "             ('almost', 7),\n",
       "             ('nonexistent', 1),\n",
       "             ('went', 16),\n",
       "             ('tigerlilly', 1),\n",
       "             ('fantastic', 13),\n",
       "             ('afternoon', 1),\n",
       "             ('cheese', 3),\n",
       "             ('crisp', 3),\n",
       "             ('town', 7),\n",
       "             ('hear', 10),\n",
       "             ('im', 46),\n",
       "             ('driving', 3),\n",
       "             ('usually', 2),\n",
       "             ('put', 9),\n",
       "             ('loudest', 1),\n",
       "             ('setting', 4),\n",
       "             ('absolutel', 1),\n",
       "             ('junk', 7),\n",
       "             ('inhouse', 1),\n",
       "             ('cut', 5),\n",
       "             ('piece', 20),\n",
       "             ('day', 23),\n",
       "             ('wonderful', 10),\n",
       "             ('tender', 4),\n",
       "             ('flavored', 1),\n",
       "             ('however', 13),\n",
       "             ('better', 38),\n",
       "             ('instruction', 4),\n",
       "             ('ordered', 13),\n",
       "             ('double', 2),\n",
       "             ('cheeseburger', 2),\n",
       "             ('got', 26),\n",
       "             ('single', 4),\n",
       "             ('patty', 1),\n",
       "             ('falling', 1),\n",
       "             ('apart', 2),\n",
       "             ('picture', 14),\n",
       "             ('uploaded', 1),\n",
       "             ('yeah', 2),\n",
       "             ('suck', 8),\n",
       "             ('worst', 32),\n",
       "             ('plugged', 3),\n",
       "             ('darn', 1),\n",
       "             ('worked', 16),\n",
       "             ('delivers', 2),\n",
       "             ('face', 7),\n",
       "             ('dissapointing', 1),\n",
       "             ('performance', 11),\n",
       "             ('margarita', 2),\n",
       "             ('expect', 8),\n",
       "             ('tapa', 2),\n",
       "             ('delicious', 20),\n",
       "             ('awsome', 1),\n",
       "             ('device', 12),\n",
       "             ('sour', 2),\n",
       "             ('egg', 4),\n",
       "             ('flower', 2),\n",
       "             ('soup', 5),\n",
       "             ('absolutely', 15),\n",
       "             ('joy', 3),\n",
       "             ('open', 2),\n",
       "             ('connection', 5),\n",
       "             ('broken', 1),\n",
       "             ('turned', 2),\n",
       "             ('def', 1),\n",
       "             ('bowl', 2),\n",
       "             ('appetite', 1),\n",
       "             ('instantly', 1),\n",
       "             ('gone', 4),\n",
       "             ('treat', 3),\n",
       "             ('see', 19),\n",
       "             ('anthony', 1),\n",
       "             ('quinn', 1),\n",
       "             ('playing', 4),\n",
       "             ('crazy', 3),\n",
       "             ('horse', 1),\n",
       "             ('servicecheck', 1),\n",
       "             ('stretch', 1),\n",
       "             ('imagination', 4),\n",
       "             ('experience', 23),\n",
       "             ('spend', 1),\n",
       "             ('elsewhere', 4),\n",
       "             ('anyway', 3),\n",
       "             ('f', 1),\n",
       "             ('breakfastlunch', 1),\n",
       "             ('lacking', 4),\n",
       "             ('passed', 2),\n",
       "             ('mark', 1),\n",
       "             ('wear', 9),\n",
       "             ('functional', 1),\n",
       "             ('average', 8),\n",
       "             ('rent', 1),\n",
       "             ('bite', 3),\n",
       "             ('refused', 2),\n",
       "             ('anymore', 1),\n",
       "             ('weve', 2),\n",
       "             ('gotten', 4),\n",
       "             ('much', 35),\n",
       "             ('pizza', 16),\n",
       "             ('door', 3),\n",
       "             ('received', 10),\n",
       "             ('awesome', 16),\n",
       "             ('bought', 16),\n",
       "             ('ebay', 1),\n",
       "             ('additional', 2),\n",
       "             ('gel', 2),\n",
       "             ('provided', 5),\n",
       "             ('whatsoever', 5),\n",
       "             ('block', 1),\n",
       "             ('hoping', 3),\n",
       "             ('could', 39),\n",
       "             ('make', 35),\n",
       "             ('bluetooth', 16),\n",
       "             ('impossible', 1),\n",
       "             ('cute', 2),\n",
       "             ('week', 9),\n",
       "             ('later', 6),\n",
       "             ('activated', 1),\n",
       "             ('suddenly', 1),\n",
       "             ('died', 2),\n",
       "             ('bathroom', 5),\n",
       "             ('location', 5),\n",
       "             ('dirty', 4),\n",
       "             ('cover', 7),\n",
       "             ('replenished', 1),\n",
       "             ('plain', 3),\n",
       "             ('yucky', 1),\n",
       "             ('full', 8),\n",
       "             ('petty', 1),\n",
       "             ('several', 9),\n",
       "             ('past', 1),\n",
       "             ('charging', 5),\n",
       "             ('overnight', 1),\n",
       "             ('purchased', 4),\n",
       "             ('longer', 5),\n",
       "             ('working', 10),\n",
       "             ('giving', 2),\n",
       "             ('soyo', 1),\n",
       "             ('technology', 2),\n",
       "             ('hold', 9),\n",
       "             ('particular', 2),\n",
       "             ('party', 6),\n",
       "             ('clearly', 3),\n",
       "             ('want', 19),\n",
       "             ('group', 2),\n",
       "             ('claimed', 1),\n",
       "             ('handled', 2),\n",
       "             ('beautifully', 1),\n",
       "             ('others', 4),\n",
       "             ('weird', 2),\n",
       "             ('vibe', 3),\n",
       "             ('owner', 6),\n",
       "             ('relocated', 1),\n",
       "             ('impressed', 17),\n",
       "             ('found', 16),\n",
       "             ('stranger', 1),\n",
       "             ('hair', 3),\n",
       "             ('must', 10),\n",
       "             ('state', 1),\n",
       "             ('allow', 1),\n",
       "             ('usage', 1),\n",
       "             ('someone', 7),\n",
       "             ('order', 13),\n",
       "             ('taco', 5),\n",
       "             ('think', 31),\n",
       "             ('may', 6),\n",
       "             ('part', 14),\n",
       "             ('ask', 8),\n",
       "             ('combo', 1),\n",
       "             ('ala', 1),\n",
       "             ('cart', 1),\n",
       "             ('concept', 1),\n",
       "             ('hooked', 1),\n",
       "             ('explain', 1),\n",
       "             ('microphone', 3),\n",
       "             ('jack', 1),\n",
       "             ('used', 22),\n",
       "             ('reason', 4),\n",
       "             ('fill', 2),\n",
       "             ('binge', 1),\n",
       "             ('drinking', 1),\n",
       "             ('carbs', 1),\n",
       "             ('stomach', 3),\n",
       "             ('sexy', 1),\n",
       "             ('mouth', 4),\n",
       "             ('youre', 3),\n",
       "             ('outrageously', 1),\n",
       "             ('flirting', 1),\n",
       "             ('hottest', 1),\n",
       "             ('person', 3),\n",
       "             ('resolution', 2),\n",
       "             ('far', 17),\n",
       "             ('comparablypriced', 1),\n",
       "             ('offering', 1),\n",
       "             ('id', 9),\n",
       "             ('advise', 3),\n",
       "             ('waitress', 9),\n",
       "             ('little', 22),\n",
       "             ('slow', 11),\n",
       "             ('cant', 19),\n",
       "             ('wrong', 7),\n",
       "             ('eargels', 2),\n",
       "             ('plethora', 1),\n",
       "             ('sandwich', 8),\n",
       "             ('seal', 1),\n",
       "             ('approval', 1),\n",
       "             ('drive', 5),\n",
       "             ('thru', 1),\n",
       "             ('mean', 5),\n",
       "             ('wait', 14),\n",
       "             ('half', 7),\n",
       "             ('seafood', 5),\n",
       "             ('generous', 1),\n",
       "             ('portion', 5),\n",
       "             ('worth', 17),\n",
       "             ('amazon', 6),\n",
       "             ('excellent', 36),\n",
       "             ('way', 25),\n",
       "             ('big', 12),\n",
       "             ('call', 22),\n",
       "             ('wife', 6),\n",
       "             ('martini', 1),\n",
       "             ('definitly', 1),\n",
       "             ('nyc', 2),\n",
       "             ('bagel', 2),\n",
       "             ('selection', 10),\n",
       "             ('cream', 4),\n",
       "             ('lox', 1),\n",
       "             ('caper', 1),\n",
       "             ('wordofmouth', 1),\n",
       "             ('promote', 1),\n",
       "             ('easier', 4),\n",
       "             ('consider', 3),\n",
       "             ('theft', 1),\n",
       "             ('greatest', 3),\n",
       "             ('salmon', 4),\n",
       "             ('promise', 1),\n",
       "             ('disappoint', 3),\n",
       "             ('salsa', 2),\n",
       "             ('tell', 8),\n",
       "             ('infuriating', 1),\n",
       "             ('shrimp', 5),\n",
       "             ('opinion', 1),\n",
       "             ('entree', 1),\n",
       "             ('gc', 1),\n",
       "             ('disappointment', 10),\n",
       "             ('waiter', 6),\n",
       "             ('building', 2),\n",
       "             ('neat', 2),\n",
       "             ('trippy', 1),\n",
       "             ('wouldnt', 6),\n",
       "             ('clear', 11),\n",
       "             ('job', 10),\n",
       "             ('motorola', 8),\n",
       "             ('sturdiness', 1),\n",
       "             ('scene', 14),\n",
       "             ('ohsomature', 1),\n",
       "             ('neighbourgirl', 1),\n",
       "             ('misplace', 1),\n",
       "             ('walk', 1),\n",
       "             ('theatre', 1),\n",
       "             ('relief', 1),\n",
       "             ('lasted', 1),\n",
       "             ('blew', 1),\n",
       "             ('curry', 2),\n",
       "             ('bamboo', 1),\n",
       "             ('shoot', 1),\n",
       "             ('tasty', 7),\n",
       "             ('husband', 4),\n",
       "             ('said', 14),\n",
       "             ('rude', 7),\n",
       "             ('apologize', 1),\n",
       "             ('anything', 10),\n",
       "             ('otherwise', 1),\n",
       "             ('install', 2),\n",
       "             ('mortified', 1),\n",
       "             ('burger', 12),\n",
       "             ('flavor', 11),\n",
       "             ('meat', 9),\n",
       "             ('bland', 11),\n",
       "             ('overcooked', 2),\n",
       "             ('charcoal', 2),\n",
       "             ('cable', 4),\n",
       "             ('allows', 1),\n",
       "             ('connect', 1),\n",
       "             ('miniusb', 1),\n",
       "             ('pc', 3),\n",
       "             ('mic', 4),\n",
       "             ('manager', 4),\n",
       "             ('razr', 4),\n",
       "             ('buy', 18),\n",
       "             ('outshining', 1),\n",
       "             ('halibut', 1),\n",
       "             ('felt', 10),\n",
       "             ('light', 11),\n",
       "             ('tinny', 3),\n",
       "             ('missed', 3),\n",
       "             ('numerous', 2),\n",
       "             ('fails', 3),\n",
       "             ('mobile', 2),\n",
       "             ('tool', 4),\n",
       "             ('glass', 3),\n",
       "             ('fine', 13),\n",
       "             ('disgusted', 1),\n",
       "             ('human', 3),\n",
       "             ('roll', 4),\n",
       "             ('saw', 4),\n",
       "             ('mirrormask', 1),\n",
       "             ('last', 16),\n",
       "             ('unsatisfactory', 1),\n",
       "             ('practically', 1),\n",
       "             ('true', 2),\n",
       "             ('masterpiece', 2),\n",
       "             ('sea', 1),\n",
       "             ('faux', 1),\n",
       "             ('firehouse', 1),\n",
       "             ('seriously', 8),\n",
       "             ('believe', 5),\n",
       "             ('steep', 1),\n",
       "             ('point', 2),\n",
       "             ('fact', 6),\n",
       "             ('hard', 13),\n",
       "             ('remember', 2),\n",
       "             ('ray', 4),\n",
       "             ('charles', 4),\n",
       "             ('acted', 2),\n",
       "             ('played', 3),\n",
       "             ('man', 5),\n",
       "             ('legendary', 1),\n",
       "             ('biographical', 1),\n",
       "             ('material', 1),\n",
       "             ('musician', 1),\n",
       "             ('hitchcock', 3),\n",
       "             ('director', 8),\n",
       "             ('ironically', 1),\n",
       "             ('mostly', 2),\n",
       "             ('total', 5),\n",
       "             ('watch', 9),\n",
       "             ('secondly', 1),\n",
       "             ('perfected', 1),\n",
       "             ('thriller', 3),\n",
       "             ('chase', 1),\n",
       "             ('pandering', 1),\n",
       "             ('audience', 3),\n",
       "             ('sabotage', 1),\n",
       "             ('hence', 1),\n",
       "             ('whole', 9),\n",
       "             ('lack', 3),\n",
       "             ('certain', 2),\n",
       "             ('energy', 2),\n",
       "             ('simply', 10),\n",
       "             ('rumble', 1),\n",
       "             ('machine', 2),\n",
       "             ('desperately', 1),\n",
       "             ('depending', 1),\n",
       "             ('addition', 3),\n",
       "             ('usual', 2),\n",
       "             ('logic', 1),\n",
       "             ('flaw', 4),\n",
       "             ('mishima', 1),\n",
       "             ('extremely', 9),\n",
       "             ('uninteresting', 1),\n",
       "             ('chilly', 2),\n",
       "             ('unremarkable', 1),\n",
       "             ('author', 1),\n",
       "             ('livingworking', 1),\n",
       "             ('abstruse', 1),\n",
       "             ('culture', 1),\n",
       "             ('flat', 4),\n",
       "             ('reenactment', 1),\n",
       "             ('attention', 3),\n",
       "             ('emotionally', 1),\n",
       "             ('adrift', 1),\n",
       "             ('stagy', 1),\n",
       "             ('rest', 4),\n",
       "             ('sits', 1),\n",
       "             ('awful', 9),\n",
       "             ('soldier', 1),\n",
       "             ('singing', 1),\n",
       "             ('song', 7),\n",
       "             ('masculinity', 1),\n",
       "             ('pledge', 1),\n",
       "             ('hairsplitting', 1),\n",
       "             ('purity', 1),\n",
       "             ('admiration', 1),\n",
       "             ('sword', 1),\n",
       "             ('etc', 4),\n",
       "             ('bore', 1),\n",
       "             ('kill', 1),\n",
       "             ('momentum', 1),\n",
       "             ('quicker', 1),\n",
       "             ('else', 4),\n",
       "             ('schrader', 1),\n",
       "             ('resume', 1),\n",
       "             ('lousy', 3),\n",
       "             ('amateurish', 1),\n",
       "             ('watched', 4),\n",
       "             ('loved', 13),\n",
       "             ('fascinated', 1),\n",
       "             ('dancing', 2),\n",
       "             ('recently', 4),\n",
       "             ('dvd', 1),\n",
       "             ('struck', 2),\n",
       "             ('storyline', 2),\n",
       "             ('contained', 2),\n",
       "             ('hole', 1),\n",
       "             ('inconsistency', 1),\n",
       "             ('frankly', 2),\n",
       "             ('lot', 17),\n",
       "             ('horrid', 1),\n",
       "             ('realistic', 1),\n",
       "             ('world', 6),\n",
       "             ('ballet', 1),\n",
       "             ('repertory', 1),\n",
       "             ('quite', 19),\n",
       "             ('pathetic', 4),\n",
       "             ('character', 17),\n",
       "             ('development', 2),\n",
       "             ('lacked', 3),\n",
       "             ('depth', 3),\n",
       "             ('woa', 1),\n",
       "             ('sappiest', 1),\n",
       "             ('dialogue', 5),\n",
       "             ('unwatchable', 1),\n",
       "             ('direction', 4),\n",
       "             ('actor', 7),\n",
       "             ('talent', 2),\n",
       "             ('speak', 2),\n",
       "             ('action', 2),\n",
       "             ('check', 6),\n",
       "             ('filmography', 1),\n",
       "             ('site', 1),\n",
       "             ('chance', 1),\n",
       "             ('tv', 5),\n",
       "             ('flick', 3),\n",
       "             ('intention', 1),\n",
       "             ('might', 5),\n",
       "             ('master', 1),\n",
       "             ('significant', 3),\n",
       "             ('themeat', 1),\n",
       "             ('least', 6),\n",
       "             ('wouldbe', 1),\n",
       "             ('theme', 2),\n",
       "             ('undertone', 1),\n",
       "             ('fifty', 1),\n",
       "             ('existential', 1),\n",
       "             ('worldweariness', 1),\n",
       "             ('aerial', 1),\n",
       "             ('ought', 1),\n",
       "             ('sens', 1),\n",
       "             ('deeply', 2),\n",
       "             ('care', 7),\n",
       "             ('regrettably', 1),\n",
       "             ('visual', 1),\n",
       "             ('interest', 1),\n",
       "             ('drama', 2),\n",
       "             ('expression', 1),\n",
       "             ('feeling', 6),\n",
       "             ('celebration', 1),\n",
       "             ('patriotism', 1),\n",
       "             ('underline', 1),\n",
       "             ('narrative', 1),\n",
       "             ('actress', 1),\n",
       "             ('worse', 7),\n",
       "             ('june', 1),\n",
       "             ('allison', 1),\n",
       "             ('watching', 7),\n",
       "             ('unfortunately', 9),\n",
       "             ('script', 2),\n",
       "             ('sucked', 9),\n",
       "             ('cinematographyif', 1),\n",
       "             ('called', 2),\n",
       "             ('thatsucked', 1),\n",
       "             ('soundtrack', 2),\n",
       "             ('concert', 1),\n",
       "             ('sequence', 1),\n",
       "             ('funny', 10),\n",
       "             ('overall', 12),\n",
       "             ('cheap', 12),\n",
       "             ('trash', 2),\n",
       "             ('considering', 5),\n",
       "             ('ridiculousness', 1),\n",
       "             ('angry', 2),\n",
       "             ('spoiler', 2),\n",
       "             ('whatever', 3),\n",
       "             ('surface', 1),\n",
       "             ('superbly', 2),\n",
       "             ('crafted', 2),\n",
       "             ('stunning', 2),\n",
       "             ('fx', 2),\n",
       "             ('nothing', 18),\n",
       "             ('stateoftheart', 1),\n",
       "             ('conceptually', 1),\n",
       "             ('everybody', 1),\n",
       "             ('parent', 2),\n",
       "             ('fantasy', 1),\n",
       "             ('andor', 1),\n",
       "             ('fan', 3),\n",
       "             ('note', 4),\n",
       "             ('actingwise', 1),\n",
       "             ('either', 9),\n",
       "             ('surprisingly', 1),\n",
       "             ('casting', 3),\n",
       "             ('considered', 1),\n",
       "             ('done', 11),\n",
       "             ('thanks', 3),\n",
       "             ('released', 1),\n",
       "             ('mexican', 3),\n",
       "             ('le', 6),\n",
       "             ('understood', 1),\n",
       "             ('matter', 1),\n",
       "             ('identified', 1),\n",
       "             ('rank', 1),\n",
       "             ('noircrimedrama', 1),\n",
       "             ('incredible', 8),\n",
       "             ('belmondo', 1),\n",
       "             ('lino', 1),\n",
       "             ('ventura', 1),\n",
       "             ('every', 25),\n",
       "             ('complex', 1),\n",
       "             ('psychological', 1),\n",
       "             ('portrayal', 3),\n",
       "             ('detailing', 1),\n",
       "             ('loyalty', 1),\n",
       "             ('treachery', 1),\n",
       "             ('hope', 5),\n",
       "             ('tremendous', 3),\n",
       "             ('melville', 1),\n",
       "             ('truly', 4),\n",
       "             ('take', 14),\n",
       "             ('journey', 1),\n",
       "             ('eye', 4),\n",
       "             ('soul', 1),\n",
       "             ('child', 2),\n",
       "             ('water', 4),\n",
       "             ('manages', 1),\n",
       "             ('transcend', 1),\n",
       "             ('limitation', 1),\n",
       "             ('indie', 1),\n",
       "             ('continually', 1),\n",
       "             ('subverting', 1),\n",
       "             ('expectation', 3),\n",
       "             ('emerge', 1),\n",
       "             ('intense', 1),\n",
       "             ('gripping', 1),\n",
       "             ('crocdodile', 1),\n",
       "             ('indeed', 1),\n",
       "             ('website', 5),\n",
       "             ('believed', 1),\n",
       "             ('crocs', 1),\n",
       "             ('swamp', 1),\n",
       "             ('fabulous', 3),\n",
       "             ('thoroughly', 2),\n",
       "             ('enjoyed', 6),\n",
       "             ('christopher', 1),\n",
       "             ('eccleston', 1),\n",
       "             ('control', 3),\n",
       "             ('tardis', 1),\n",
       "             ('continuation', 1),\n",
       "             ('series', 4),\n",
       "             ('disturbing', 1),\n",
       "             ('guess', 6),\n",
       "             ('succeeded', 1),\n",
       "             ('beautiful', 9),\n",
       "             ('forced', 3),\n",
       "             ('lame', 3),\n",
       "             ('here', 1),\n",
       "             ('pied', 1),\n",
       "             ('jerky', 2),\n",
       "             ('camerawork', 3),\n",
       "             ('theater', 2),\n",
       "             ('thought', 9),\n",
       "             ('sick', 5),\n",
       "             ('summary', 2),\n",
       "             ('witticism', 1),\n",
       "             ('werent', 3),\n",
       "             ('witty', 1),\n",
       "             ('let', 7),\n",
       "             ('billy', 1),\n",
       "             ('bob', 1),\n",
       "             ('couldnt', 16),\n",
       "             ('rise', 1),\n",
       "             ('rating', 4),\n",
       "             ('finale', 1),\n",
       "             ('possibly', 1),\n",
       "             ('trilogy', 3),\n",
       "             ('kieslowski', 1),\n",
       "             ('cease', 1),\n",
       "             ('amaze', 1),\n",
       "             ('favourite', 1),\n",
       "             ('talented', 2),\n",
       "             ('history', 4),\n",
       "             ('cinema', 3),\n",
       "             ('colour', 1),\n",
       "             ('french', 2),\n",
       "             ('flag', 1),\n",
       "             ('short', 4),\n",
       "             ('shot', 5),\n",
       "             ('art', 6),\n",
       "             ('visually', 1),\n",
       "             ('appealing', 3),\n",
       "             ('seen', 8),\n",
       "             ('subtle', 1),\n",
       "             ('mention', 3),\n",
       "             ('huge', 8),\n",
       "             ('ending', 1),\n",
       "             ('remaining', 1),\n",
       "             ('survivor', 1),\n",
       "             ('ferry', 1),\n",
       "             ('disaster', 3),\n",
       "             ('valentine', 1),\n",
       "             ('young', 2),\n",
       "             ('judge', 2),\n",
       "             ('together', 5),\n",
       "             ('solidifying', 1),\n",
       "             ('happiness', 1),\n",
       "             ('suffering', 1),\n",
       "             ('dealt', 1),\n",
       "             ('smile', 1),\n",
       "             ('wrap', 2),\n",
       "             ('everyone', 7),\n",
       "             ('anythinga', 1),\n",
       "             ('literally', 7),\n",
       "             ('vomited', 2),\n",
       "             ('people', 16),\n",
       "             ('roth', 1),\n",
       "             ('pearl', 1),\n",
       "             ('awarded', 1),\n",
       "             ('eloquently', 1),\n",
       "             ('francis', 1),\n",
       "             ('ford', 1),\n",
       "             ...])"
      ]
     },
     "execution_count": 102,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "tok.word_counts"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### A quick aside: cosine similarity "
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Using TF-IDF Vectorizer in Sklearn, calculate the cosine similarity between each record"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#using the first 5 sentences as an example\n",
    "documents = (X[0:5,])\n",
    "documents"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.feature_extraction.text import TfidfVectorizer\n",
    "\n",
    "#transform documents into TF-IDF matrix\n",
    "\n",
    "tfidf_vectorizer = TfidfVectorizer()\n",
    "tfidf_matrix = tfidf_vectorizer.fit_transform(documents)\n",
    "\n",
    "#show the dimension of the tfidf_matrix\n",
    "print (tfidf_matrix.shape)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Take a look at the vector for the 1st sentense\n",
    "print(tfidf_matrix[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "from scipy.spatial.distance import cosine\n",
    "\n",
    "\n",
    "e1 = \n",
    "\n",
    "cosine_similarity(tfidf_matrix[0], tfidf_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#compare how similar these 5 sentences are\n",
    "cosine_similarity(tfidf_matrix, tfidf_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "######IGNORE#####\n",
    "\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "\n",
    "print(\"sequences_matrix for happy : \" ,'\\n', sequences_matrix[word_index['happy'],:], '\\n')\n",
    "print(\"sequences_matrix for sad : \",'\\n', sequences_matrix[word_index['sad'],:], '\\n')\n",
    "\n",
    "x=(sequences_matrix[word_index['happy'],:]).reshape(1, -1)\n",
    "y=sequences_matrix[word_index['sad'],:].reshape(1,-1)\n",
    "\n",
    "cosine_similarity(x,y)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Part 2 - Using pre-trained Embedding - GloVe"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "GloVe stands for Global Vectors for Word Representation\n",
    "\n",
    "Download pre-trained word embeddings from https://nlp.stanford.edu/projects/glove/"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Same as before\n",
    "#Start with creating the sequence matrix\n",
    "\n",
    "max_words = 1000 #to define vocab size for max num of words to keep based on word freq, here we are only keeping the 1000-1 most common words\n",
    "max_len = 150 #to define fixed sequence length, here we are padding the input sequence to have the same length of 150\n",
    "\n",
    "#vectorize the corpus\n",
    "tok = Tokenizer(num_words=max_words) \n",
    "tok.fit_on_texts(X_train) #Updates internal vocabulary based on a list of texts. This method creates the vocabulary index based on word frequency\n",
    "sequences = tok.texts_to_sequences(X_train) #Transforms each text in texts to a sequence of integers.\n",
    "sequences_matrix = sequence.pad_sequences(sequences, maxlen=max_len) #pad the vector so they are all the same length of 150"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "sequences_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "len(tok.word_index)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "tok.word_index['happy']"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#load the GloVe word embedding into memory\n",
    "embeddings_index = dict()\n",
    "f = open ('glove.6B.100d.txt')\n",
    "for line in f:\n",
    "    values = line.split()\n",
    "    word = values[0]\n",
    "    coefs = np.asarray(values[1:], dtype='float32')\n",
    "    embeddings_index[word] = coefs\n",
    "f.close()\n",
    "\n",
    "print('Loaded %s word vectors.' % len(embeddings_index))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#See the 100 dimensions vector for \"happy\" and \"the\" in GloVe\n",
    "embeddings_index.get(\"happy\")\n",
    "embeddings_index.get(\"the\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "word_index.items()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#build the embedding matrix for words in training set\n",
    "\n",
    "embedding_matrix = np.zeros((len(word_index) + 1, 100), dtype='float32')\n",
    "for word, i in word_index.items():\n",
    "    embedding_vector = embeddings_index.get(word)\n",
    "    if embedding_vector is not None:\n",
    "        # words not found in embedding index will be all-zeros.\n",
    "        embedding_matrix[i] = embedding_vector"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "embedding_matrix"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#check dimension of the embedding_matrix : should be 1000 x 100 (1000 most frequent words and 100D vector from Glove)\n",
    "embedding_matrix.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#build the neural network model\n",
    "model2 = Sequential()\n",
    "\n",
    "#the only difference here is we use the embedding_matrix from previous step for the input layer\n",
    "model2.add(Embedding(len(word_index)+1, \n",
    "                     100, \n",
    "                     weights=[embedding_matrix], \n",
    "                     input_length=max_len, \n",
    "                     trainable=False))\n",
    "model2.add((LSTM(64, dropout=0.2, recurrent_dropout=0.2)))\n",
    "model2.add(Dense(1, name='out_layer', activation='sigmoid'))\n",
    "\n",
    "model2.summary()\n",
    "model2.compile(loss='binary_crossentropy',optimizer=RMSprop(), metrics=['accuracy'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#train the model\n",
    "model2.fit(sequences_matrix, y_train, batch_size=128, epochs=10,\n",
    "          validation_split=0.2, callbacks=[EarlyStopping(monitor='val_loss',min_delta=0.0001)])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#set up the test set\n",
    "test_sequences = tok.texts_to_sequences(X_test)\n",
    "test_sequences_matrix = sequence.pad_sequences(test_sequences, maxlen=max_len)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#show prediction: value closer to 1 is strong positive sentiment and value closer to 0 is a strong negative\n",
    "predict_2 = model2.predict(x=test_sequences_matrix)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "predict_2.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#see the first 10 predictions\n",
    "predict_2[0:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Let's compare the prediction with the actual label\n",
    "print (X_test[0:10])\n",
    "\n",
    "for i in range(10):\n",
    "    print ('Model predicted: ', predict_2[i],  'Actual score: ', y_test[i], '\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#evaluate with test set\n",
    "accr_2 = model2.evaluate(test_sequences_matrix, y_test)\n",
    "\n",
    "print('Test set\\n  Loss: {:0.3f}\\n  Accuracy: {:0.3f}'.format(accr_2[0],accr_2[1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "#Recall accuracy from previous model using our own embeddings \n",
    "\n",
    "accr_1 = model1.evaluate(test_sequences_matrix, y_test)\n",
    "\n",
    "print('Test set\\n  Loss: {:0.3f}\\n  Accuracy: {:0.3f}'.format(accr_1[0],accr_1[1]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we can see that the model using the pre-trained embeddings does not do as well as the one we built as part of our neural network."
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}