Assignment_NLP.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": []
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "code",
      "source": [
        "##installing and importing libraries"
      ],
      "metadata": {
        "id": "iBqyzSYKigwX"
      },
      "execution_count": 1,
      "outputs": []
    },
    {
      "cell_type": "code",
      "execution_count": 2,
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "ktzp2D62g8Li",
        "outputId": "87916b4e-7402-432f-a24e-c6ae4e630c62"
      },
      "outputs": [
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: nltk in /usr/local/lib/python3.10/dist-packages (3.8.1)\n",
            "Requirement already satisfied: click in /usr/local/lib/python3.10/dist-packages (from nltk) (8.1.3)\n",
            "Requirement already satisfied: joblib in /usr/local/lib/python3.10/dist-packages (from nltk) (1.2.0)\n",
            "Requirement already satisfied: regex>=2021.8.3 in /usr/local/lib/python3.10/dist-packages (from nltk) (2022.10.31)\n",
            "Requirement already satisfied: tqdm in /usr/local/lib/python3.10/dist-packages (from nltk) (4.65.0)\n",
            "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
            "Requirement already satisfied: spacy in /usr/local/lib/python3.10/dist-packages (3.5.2)\n",
            "Requirement already satisfied: spacy-legacy<3.1.0,>=3.0.11 in /usr/local/lib/python3.10/dist-packages (from spacy) (3.0.12)\n",
            "Requirement already satisfied: spacy-loggers<2.0.0,>=1.0.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (1.0.4)\n",
            "Requirement already satisfied: murmurhash<1.1.0,>=0.28.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (1.0.9)\n",
            "Requirement already satisfied: cymem<2.1.0,>=2.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy) (2.0.7)\n",
            "Requirement already satisfied: preshed<3.1.0,>=3.0.2 in /usr/local/lib/python3.10/dist-packages (from spacy) (3.0.8)\n",
            "Requirement already satisfied: thinc<8.2.0,>=8.1.8 in /usr/local/lib/python3.10/dist-packages (from spacy) (8.1.9)\n",
            "Requirement already satisfied: wasabi<1.2.0,>=0.9.1 in /usr/local/lib/python3.10/dist-packages (from spacy) (1.1.1)\n",
            "Requirement already satisfied: srsly<3.0.0,>=2.4.3 in /usr/local/lib/python3.10/dist-packages (from spacy) (2.4.6)\n",
            "Requirement already satisfied: catalogue<2.1.0,>=2.0.6 in /usr/local/lib/python3.10/dist-packages (from spacy) (2.0.8)\n",
            "Requirement already satisfied: typer<0.8.0,>=0.3.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (0.7.0)\n",
            "Requirement already satisfied: pathy>=0.10.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (0.10.1)\n",
            "Requirement already satisfied: smart-open<7.0.0,>=5.2.1 in /usr/local/lib/python3.10/dist-packages (from spacy) (6.3.0)\n",
            "Requirement already satisfied: tqdm<5.0.0,>=4.38.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (4.65.0)\n",
            "Requirement already satisfied: numpy>=1.15.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (1.22.4)\n",
            "Requirement already satisfied: requests<3.0.0,>=2.13.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (2.27.1)\n",
            "Requirement already satisfied: pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4 in /usr/local/lib/python3.10/dist-packages (from spacy) (1.10.7)\n",
            "Requirement already satisfied: jinja2 in /usr/local/lib/python3.10/dist-packages (from spacy) (3.1.2)\n",
            "Requirement already satisfied: setuptools in /usr/local/lib/python3.10/dist-packages (from spacy) (67.7.2)\n",
            "Requirement already satisfied: packaging>=20.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (23.1)\n",
            "Requirement already satisfied: langcodes<4.0.0,>=3.2.0 in /usr/local/lib/python3.10/dist-packages (from spacy) (3.3.0)\n",
            "Requirement already satisfied: typing-extensions>=4.2.0 in /usr/local/lib/python3.10/dist-packages (from pydantic!=1.8,!=1.8.1,<1.11.0,>=1.7.4->spacy) (4.5.0)\n",
            "Requirement already satisfied: urllib3<1.27,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (1.26.15)\n",
            "Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2022.12.7)\n",
            "Requirement already satisfied: charset-normalizer~=2.0.0 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (2.0.12)\n",
            "Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3.0.0,>=2.13.0->spacy) (3.4)\n",
            "Requirement already satisfied: blis<0.8.0,>=0.7.8 in /usr/local/lib/python3.10/dist-packages (from thinc<8.2.0,>=8.1.8->spacy) (0.7.9)\n",
            "Requirement already satisfied: confection<1.0.0,>=0.0.1 in /usr/local/lib/python3.10/dist-packages (from thinc<8.2.0,>=8.1.8->spacy) (0.0.4)\n",
            "Requirement already satisfied: click<9.0.0,>=7.1.1 in /usr/local/lib/python3.10/dist-packages (from typer<0.8.0,>=0.3.0->spacy) (8.1.3)\n",
            "Requirement already satisfied: MarkupSafe>=2.0 in /usr/local/lib/python3.10/dist-packages (from jinja2->spacy) (2.1.2)\n"
          ]
        }
      ],
      "source": [
        "\n",
        "!pip install nltk\n",
        "!pip install spacy"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "import numpy as np\n",
        "import pandas as pd\n",
        "import re\n",
        "import nltk\n",
        "import spacy\n",
        "import string"
      ],
      "metadata": {
        "id": "gPauMzZch5AR"
      },
      "execution_count": 3,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "##Loading the test data\n",
        "full_data = pd.read_csv('/content/1429_1.csv', nrows=5000)\n",
        "full_data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 820
        },
        "id": "kNaY7vcah9zV",
        "outputId": "42a3e7b4-d03b-42e0-f74f-68c2c636a8a9"
      },
      "execution_count": 4,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                     id                                               name  \\\n",
              "0  AVqkIhwDv8e3D1O-lebb  All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...   \n",
              "1  AVqkIhwDv8e3D1O-lebb  All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...   \n",
              "2  AVqkIhwDv8e3D1O-lebb  All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...   \n",
              "3  AVqkIhwDv8e3D1O-lebb  All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...   \n",
              "4  AVqkIhwDv8e3D1O-lebb  All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...   \n",
              "\n",
              "        asins   brand                                         categories  \\\n",
              "0  B01AHB9CN2  Amazon  Electronics,iPad & Tablets,All Tablets,Fire Ta...   \n",
              "1  B01AHB9CN2  Amazon  Electronics,iPad & Tablets,All Tablets,Fire Ta...   \n",
              "2  B01AHB9CN2  Amazon  Electronics,iPad & Tablets,All Tablets,Fire Ta...   \n",
              "3  B01AHB9CN2  Amazon  Electronics,iPad & Tablets,All Tablets,Fire Ta...   \n",
              "4  B01AHB9CN2  Amazon  Electronics,iPad & Tablets,All Tablets,Fire Ta...   \n",
              "\n",
              "                                                keys manufacturer  \\\n",
              "0  841667104676,amazon/53004484,amazon/b01ahb9cn2...       Amazon   \n",
              "1  841667104676,amazon/53004484,amazon/b01ahb9cn2...       Amazon   \n",
              "2  841667104676,amazon/53004484,amazon/b01ahb9cn2...       Amazon   \n",
              "3  841667104676,amazon/53004484,amazon/b01ahb9cn2...       Amazon   \n",
              "4  841667104676,amazon/53004484,amazon/b01ahb9cn2...       Amazon   \n",
              "\n",
              "               reviews.date     reviews.dateAdded  \\\n",
              "0  2017-01-13T00:00:00.000Z  2017-07-03T23:33:15Z   \n",
              "1  2017-01-13T00:00:00.000Z  2017-07-03T23:33:15Z   \n",
              "2  2017-01-13T00:00:00.000Z  2017-07-03T23:33:15Z   \n",
              "3  2017-01-13T00:00:00.000Z  2017-07-03T23:33:15Z   \n",
              "4  2017-01-12T00:00:00.000Z  2017-07-03T23:33:15Z   \n",
              "\n",
              "                                    reviews.dateSeen  ...  \\\n",
              "0  2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z  ...   \n",
              "1  2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z  ...   \n",
              "2  2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z  ...   \n",
              "3  2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z  ...   \n",
              "4  2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z  ...   \n",
              "\n",
              "   reviews.doRecommend reviews.id  reviews.numHelpful  reviews.rating  \\\n",
              "0                 True        NaN                 0.0             5.0   \n",
              "1                 True        NaN                 0.0             5.0   \n",
              "2                 True        NaN                 0.0             5.0   \n",
              "3                 True        NaN                 0.0             4.0   \n",
              "4                 True        NaN                 0.0             5.0   \n",
              "\n",
              "                                  reviews.sourceURLs  \\\n",
              "0  http://reviews.bestbuy.com/3545/5620406/review...   \n",
              "1  http://reviews.bestbuy.com/3545/5620406/review...   \n",
              "2  http://reviews.bestbuy.com/3545/5620406/review...   \n",
              "3  http://reviews.bestbuy.com/3545/5620406/review...   \n",
              "4  http://reviews.bestbuy.com/3545/5620406/review...   \n",
              "\n",
              "                                        reviews.text  \\\n",
              "0  This product so far has not disappointed. My c...   \n",
              "1  great for beginner or experienced person. Boug...   \n",
              "2  Inexpensive tablet for him to use and learn on...   \n",
              "3  I've had my Fire HD 8 two weeks now and I love...   \n",
              "4  I bought this for my grand daughter when she c...   \n",
              "\n",
              "                             reviews.title reviews.userCity  \\\n",
              "0                                   Kindle              NaN   \n",
              "1                                very fast              NaN   \n",
              "2  Beginner tablet for our 9 year old son.              NaN   \n",
              "3                                  Good!!!              NaN   \n",
              "4                Fantastic Tablet for kids              NaN   \n",
              "\n",
              "   reviews.userProvince  reviews.username  \n",
              "0                   NaN           Adapter  \n",
              "1                   NaN            truman  \n",
              "2                   NaN             DaveZ  \n",
              "3                   NaN            Shacks  \n",
              "4                   NaN         explore42  \n",
              "\n",
              "[5 rows x 21 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e880e67a-01b9-4071-82ce-902aa23fa493\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>name</th>\n",
              "      <th>asins</th>\n",
              "      <th>brand</th>\n",
              "      <th>categories</th>\n",
              "      <th>keys</th>\n",
              "      <th>manufacturer</th>\n",
              "      <th>reviews.date</th>\n",
              "      <th>reviews.dateAdded</th>\n",
              "      <th>reviews.dateSeen</th>\n",
              "      <th>...</th>\n",
              "      <th>reviews.doRecommend</th>\n",
              "      <th>reviews.id</th>\n",
              "      <th>reviews.numHelpful</th>\n",
              "      <th>reviews.rating</th>\n",
              "      <th>reviews.sourceURLs</th>\n",
              "      <th>reviews.text</th>\n",
              "      <th>reviews.title</th>\n",
              "      <th>reviews.userCity</th>\n",
              "      <th>reviews.userProvince</th>\n",
              "      <th>reviews.username</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>AVqkIhwDv8e3D1O-lebb</td>\n",
              "      <td>All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...</td>\n",
              "      <td>B01AHB9CN2</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>Electronics,iPad &amp; Tablets,All Tablets,Fire Ta...</td>\n",
              "      <td>841667104676,amazon/53004484,amazon/b01ahb9cn2...</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>2017-01-13T00:00:00.000Z</td>\n",
              "      <td>2017-07-03T23:33:15Z</td>\n",
              "      <td>2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z</td>\n",
              "      <td>...</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>http://reviews.bestbuy.com/3545/5620406/review...</td>\n",
              "      <td>This product so far has not disappointed. My c...</td>\n",
              "      <td>Kindle</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Adapter</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>AVqkIhwDv8e3D1O-lebb</td>\n",
              "      <td>All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...</td>\n",
              "      <td>B01AHB9CN2</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>Electronics,iPad &amp; Tablets,All Tablets,Fire Ta...</td>\n",
              "      <td>841667104676,amazon/53004484,amazon/b01ahb9cn2...</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>2017-01-13T00:00:00.000Z</td>\n",
              "      <td>2017-07-03T23:33:15Z</td>\n",
              "      <td>2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z</td>\n",
              "      <td>...</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>http://reviews.bestbuy.com/3545/5620406/review...</td>\n",
              "      <td>great for beginner or experienced person. Boug...</td>\n",
              "      <td>very fast</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>truman</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>AVqkIhwDv8e3D1O-lebb</td>\n",
              "      <td>All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...</td>\n",
              "      <td>B01AHB9CN2</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>Electronics,iPad &amp; Tablets,All Tablets,Fire Ta...</td>\n",
              "      <td>841667104676,amazon/53004484,amazon/b01ahb9cn2...</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>2017-01-13T00:00:00.000Z</td>\n",
              "      <td>2017-07-03T23:33:15Z</td>\n",
              "      <td>2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z</td>\n",
              "      <td>...</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>http://reviews.bestbuy.com/3545/5620406/review...</td>\n",
              "      <td>Inexpensive tablet for him to use and learn on...</td>\n",
              "      <td>Beginner tablet for our 9 year old son.</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>DaveZ</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>AVqkIhwDv8e3D1O-lebb</td>\n",
              "      <td>All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...</td>\n",
              "      <td>B01AHB9CN2</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>Electronics,iPad &amp; Tablets,All Tablets,Fire Ta...</td>\n",
              "      <td>841667104676,amazon/53004484,amazon/b01ahb9cn2...</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>2017-01-13T00:00:00.000Z</td>\n",
              "      <td>2017-07-03T23:33:15Z</td>\n",
              "      <td>2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z</td>\n",
              "      <td>...</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>http://reviews.bestbuy.com/3545/5620406/review...</td>\n",
              "      <td>I've had my Fire HD 8 two weeks now and I love...</td>\n",
              "      <td>Good!!!</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Shacks</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>AVqkIhwDv8e3D1O-lebb</td>\n",
              "      <td>All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...</td>\n",
              "      <td>B01AHB9CN2</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>Electronics,iPad &amp; Tablets,All Tablets,Fire Ta...</td>\n",
              "      <td>841667104676,amazon/53004484,amazon/b01ahb9cn2...</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>2017-01-12T00:00:00.000Z</td>\n",
              "      <td>2017-07-03T23:33:15Z</td>\n",
              "      <td>2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z</td>\n",
              "      <td>...</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>http://reviews.bestbuy.com/3545/5620406/review...</td>\n",
              "      <td>I bought this for my grand daughter when she c...</td>\n",
              "      <td>Fantastic Tablet for kids</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>explore42</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 21 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e880e67a-01b9-4071-82ce-902aa23fa493')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e880e67a-01b9-4071-82ce-902aa23fa493 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e880e67a-01b9-4071-82ce-902aa23fa493');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 4
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "dt = full_data[[\"reviews.text\"]]\n",
        "dt[\"reviews.text\"] = dt[\"reviews.text\"].astype(str)\n",
        "from nltk.tokenize import word_tokenize\n",
        "\n",
        "### Tokenization\n",
        "\n",
        "nltk.download('punkt')\n",
        "tokens = dt['reviews.text'].apply(nltk.word_tokenize)\n",
        "# Print the tokens\n",
        "print(tokens)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "sIVzWzMJjXwA",
        "outputId": "57419b65-e566-4c68-f3c6-0381d6746675"
      },
      "execution_count": 5,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-5-97f82a842d2a>:2: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  dt[\"reviews.text\"] = dt[\"reviews.text\"].astype(str)\n",
            "[nltk_data] Downloading package punkt to /root/nltk_data...\n",
            "[nltk_data]   Package punkt is already up-to-date!\n"
          ]
        },
        {
          "output_type": "stream",
          "name": "stdout",
          "text": [
            "0       [This, product, so, far, has, not, disappointe...\n",
            "1       [great, for, beginner, or, experienced, person...\n",
            "2       [Inexpensive, tablet, for, him, to, use, and, ...\n",
            "3       [I, 've, had, my, Fire, HD, 8, two, weeks, now...\n",
            "4       [I, bought, this, for, my, grand, daughter, wh...\n",
            "                              ...                        \n",
            "4995    [Amazing, tablet, for, the, price, i, love, it...\n",
            "4996    [I, love, my, fire, ..., The, games, and, the,...\n",
            "4997    [Great, tablet, for, kids, but, the, screen, i...\n",
            "4998    [Great, product, easy, to, use, ., Having, dif...\n",
            "4999    [Love, this, tablet, ,, great, price, and, is,...\n",
            "Name: reviews.text, Length: 5000, dtype: object\n"
          ]
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "full_data.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 820
        },
        "id": "PKdpGwx9jpqc",
        "outputId": "fe9914f8-f91d-4a80-a396-9533c0fad058"
      },
      "execution_count": 6,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                     id                                               name  \\\n",
              "0  AVqkIhwDv8e3D1O-lebb  All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...   \n",
              "1  AVqkIhwDv8e3D1O-lebb  All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...   \n",
              "2  AVqkIhwDv8e3D1O-lebb  All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...   \n",
              "3  AVqkIhwDv8e3D1O-lebb  All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...   \n",
              "4  AVqkIhwDv8e3D1O-lebb  All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...   \n",
              "\n",
              "        asins   brand                                         categories  \\\n",
              "0  B01AHB9CN2  Amazon  Electronics,iPad & Tablets,All Tablets,Fire Ta...   \n",
              "1  B01AHB9CN2  Amazon  Electronics,iPad & Tablets,All Tablets,Fire Ta...   \n",
              "2  B01AHB9CN2  Amazon  Electronics,iPad & Tablets,All Tablets,Fire Ta...   \n",
              "3  B01AHB9CN2  Amazon  Electronics,iPad & Tablets,All Tablets,Fire Ta...   \n",
              "4  B01AHB9CN2  Amazon  Electronics,iPad & Tablets,All Tablets,Fire Ta...   \n",
              "\n",
              "                                                keys manufacturer  \\\n",
              "0  841667104676,amazon/53004484,amazon/b01ahb9cn2...       Amazon   \n",
              "1  841667104676,amazon/53004484,amazon/b01ahb9cn2...       Amazon   \n",
              "2  841667104676,amazon/53004484,amazon/b01ahb9cn2...       Amazon   \n",
              "3  841667104676,amazon/53004484,amazon/b01ahb9cn2...       Amazon   \n",
              "4  841667104676,amazon/53004484,amazon/b01ahb9cn2...       Amazon   \n",
              "\n",
              "               reviews.date     reviews.dateAdded  \\\n",
              "0  2017-01-13T00:00:00.000Z  2017-07-03T23:33:15Z   \n",
              "1  2017-01-13T00:00:00.000Z  2017-07-03T23:33:15Z   \n",
              "2  2017-01-13T00:00:00.000Z  2017-07-03T23:33:15Z   \n",
              "3  2017-01-13T00:00:00.000Z  2017-07-03T23:33:15Z   \n",
              "4  2017-01-12T00:00:00.000Z  2017-07-03T23:33:15Z   \n",
              "\n",
              "                                    reviews.dateSeen  ...  \\\n",
              "0  2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z  ...   \n",
              "1  2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z  ...   \n",
              "2  2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z  ...   \n",
              "3  2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z  ...   \n",
              "4  2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z  ...   \n",
              "\n",
              "   reviews.doRecommend reviews.id  reviews.numHelpful  reviews.rating  \\\n",
              "0                 True        NaN                 0.0             5.0   \n",
              "1                 True        NaN                 0.0             5.0   \n",
              "2                 True        NaN                 0.0             5.0   \n",
              "3                 True        NaN                 0.0             4.0   \n",
              "4                 True        NaN                 0.0             5.0   \n",
              "\n",
              "                                  reviews.sourceURLs  \\\n",
              "0  http://reviews.bestbuy.com/3545/5620406/review...   \n",
              "1  http://reviews.bestbuy.com/3545/5620406/review...   \n",
              "2  http://reviews.bestbuy.com/3545/5620406/review...   \n",
              "3  http://reviews.bestbuy.com/3545/5620406/review...   \n",
              "4  http://reviews.bestbuy.com/3545/5620406/review...   \n",
              "\n",
              "                                        reviews.text  \\\n",
              "0  This product so far has not disappointed. My c...   \n",
              "1  great for beginner or experienced person. Boug...   \n",
              "2  Inexpensive tablet for him to use and learn on...   \n",
              "3  I've had my Fire HD 8 two weeks now and I love...   \n",
              "4  I bought this for my grand daughter when she c...   \n",
              "\n",
              "                             reviews.title reviews.userCity  \\\n",
              "0                                   Kindle              NaN   \n",
              "1                                very fast              NaN   \n",
              "2  Beginner tablet for our 9 year old son.              NaN   \n",
              "3                                  Good!!!              NaN   \n",
              "4                Fantastic Tablet for kids              NaN   \n",
              "\n",
              "   reviews.userProvince  reviews.username  \n",
              "0                   NaN           Adapter  \n",
              "1                   NaN            truman  \n",
              "2                   NaN             DaveZ  \n",
              "3                   NaN            Shacks  \n",
              "4                   NaN         explore42  \n",
              "\n",
              "[5 rows x 21 columns]"
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e9d2bd45-3fe6-4445-883c-4a05cafa96d6\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>id</th>\n",
              "      <th>name</th>\n",
              "      <th>asins</th>\n",
              "      <th>brand</th>\n",
              "      <th>categories</th>\n",
              "      <th>keys</th>\n",
              "      <th>manufacturer</th>\n",
              "      <th>reviews.date</th>\n",
              "      <th>reviews.dateAdded</th>\n",
              "      <th>reviews.dateSeen</th>\n",
              "      <th>...</th>\n",
              "      <th>reviews.doRecommend</th>\n",
              "      <th>reviews.id</th>\n",
              "      <th>reviews.numHelpful</th>\n",
              "      <th>reviews.rating</th>\n",
              "      <th>reviews.sourceURLs</th>\n",
              "      <th>reviews.text</th>\n",
              "      <th>reviews.title</th>\n",
              "      <th>reviews.userCity</th>\n",
              "      <th>reviews.userProvince</th>\n",
              "      <th>reviews.username</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>AVqkIhwDv8e3D1O-lebb</td>\n",
              "      <td>All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...</td>\n",
              "      <td>B01AHB9CN2</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>Electronics,iPad &amp; Tablets,All Tablets,Fire Ta...</td>\n",
              "      <td>841667104676,amazon/53004484,amazon/b01ahb9cn2...</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>2017-01-13T00:00:00.000Z</td>\n",
              "      <td>2017-07-03T23:33:15Z</td>\n",
              "      <td>2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z</td>\n",
              "      <td>...</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>http://reviews.bestbuy.com/3545/5620406/review...</td>\n",
              "      <td>This product so far has not disappointed. My c...</td>\n",
              "      <td>Kindle</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Adapter</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>AVqkIhwDv8e3D1O-lebb</td>\n",
              "      <td>All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...</td>\n",
              "      <td>B01AHB9CN2</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>Electronics,iPad &amp; Tablets,All Tablets,Fire Ta...</td>\n",
              "      <td>841667104676,amazon/53004484,amazon/b01ahb9cn2...</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>2017-01-13T00:00:00.000Z</td>\n",
              "      <td>2017-07-03T23:33:15Z</td>\n",
              "      <td>2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z</td>\n",
              "      <td>...</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>http://reviews.bestbuy.com/3545/5620406/review...</td>\n",
              "      <td>great for beginner or experienced person. Boug...</td>\n",
              "      <td>very fast</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>truman</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>AVqkIhwDv8e3D1O-lebb</td>\n",
              "      <td>All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...</td>\n",
              "      <td>B01AHB9CN2</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>Electronics,iPad &amp; Tablets,All Tablets,Fire Ta...</td>\n",
              "      <td>841667104676,amazon/53004484,amazon/b01ahb9cn2...</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>2017-01-13T00:00:00.000Z</td>\n",
              "      <td>2017-07-03T23:33:15Z</td>\n",
              "      <td>2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z</td>\n",
              "      <td>...</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>http://reviews.bestbuy.com/3545/5620406/review...</td>\n",
              "      <td>Inexpensive tablet for him to use and learn on...</td>\n",
              "      <td>Beginner tablet for our 9 year old son.</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>DaveZ</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>AVqkIhwDv8e3D1O-lebb</td>\n",
              "      <td>All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...</td>\n",
              "      <td>B01AHB9CN2</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>Electronics,iPad &amp; Tablets,All Tablets,Fire Ta...</td>\n",
              "      <td>841667104676,amazon/53004484,amazon/b01ahb9cn2...</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>2017-01-13T00:00:00.000Z</td>\n",
              "      <td>2017-07-03T23:33:15Z</td>\n",
              "      <td>2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z</td>\n",
              "      <td>...</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.0</td>\n",
              "      <td>4.0</td>\n",
              "      <td>http://reviews.bestbuy.com/3545/5620406/review...</td>\n",
              "      <td>I've had my Fire HD 8 two weeks now and I love...</td>\n",
              "      <td>Good!!!</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>Shacks</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>AVqkIhwDv8e3D1O-lebb</td>\n",
              "      <td>All-New Fire HD 8 Tablet, 8 HD Display, Wi-Fi,...</td>\n",
              "      <td>B01AHB9CN2</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>Electronics,iPad &amp; Tablets,All Tablets,Fire Ta...</td>\n",
              "      <td>841667104676,amazon/53004484,amazon/b01ahb9cn2...</td>\n",
              "      <td>Amazon</td>\n",
              "      <td>2017-01-12T00:00:00.000Z</td>\n",
              "      <td>2017-07-03T23:33:15Z</td>\n",
              "      <td>2017-06-07T09:04:00.000Z,2017-04-30T00:45:00.000Z</td>\n",
              "      <td>...</td>\n",
              "      <td>True</td>\n",
              "      <td>NaN</td>\n",
              "      <td>0.0</td>\n",
              "      <td>5.0</td>\n",
              "      <td>http://reviews.bestbuy.com/3545/5620406/review...</td>\n",
              "      <td>I bought this for my grand daughter when she c...</td>\n",
              "      <td>Fantastic Tablet for kids</td>\n",
              "      <td>NaN</td>\n",
              "      <td>NaN</td>\n",
              "      <td>explore42</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "<p>5 rows × 21 columns</p>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e9d2bd45-3fe6-4445-883c-4a05cafa96d6')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e9d2bd45-3fe6-4445-883c-4a05cafa96d6 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e9d2bd45-3fe6-4445-883c-4a05cafa96d6');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 6
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "##Lower casing\n",
        "dt[\"text_lower\"] = dt[\"reviews.text\"].str.lower()\n",
        "dt.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 310
        },
        "id": "vRk4YE4Ijyo3",
        "outputId": "61bea001-6bd7-47e1-da20-482fd8bd892f"
      },
      "execution_count": 7,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-7-3d6ac7c61541>:2: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  dt[\"text_lower\"] = dt[\"reviews.text\"].str.lower()\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                        reviews.text  \\\n",
              "0  This product so far has not disappointed. My c...   \n",
              "1  great for beginner or experienced person. Boug...   \n",
              "2  Inexpensive tablet for him to use and learn on...   \n",
              "3  I've had my Fire HD 8 two weeks now and I love...   \n",
              "4  I bought this for my grand daughter when she c...   \n",
              "\n",
              "                                          text_lower  \n",
              "0  this product so far has not disappointed. my c...  \n",
              "1  great for beginner or experienced person. boug...  \n",
              "2  inexpensive tablet for him to use and learn on...  \n",
              "3  i've had my fire hd 8 two weeks now and i love...  \n",
              "4  i bought this for my grand daughter when she c...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6bff4e98-f5a6-4d39-9765-7cf24ac571bb\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>reviews.text</th>\n",
              "      <th>text_lower</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>This product so far has not disappointed. My c...</td>\n",
              "      <td>this product so far has not disappointed. my c...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>great for beginner or experienced person. Boug...</td>\n",
              "      <td>great for beginner or experienced person. boug...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Inexpensive tablet for him to use and learn on...</td>\n",
              "      <td>inexpensive tablet for him to use and learn on...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>I've had my Fire HD 8 two weeks now and I love...</td>\n",
              "      <td>i've had my fire hd 8 two weeks now and i love...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I bought this for my grand daughter when she c...</td>\n",
              "      <td>i bought this for my grand daughter when she c...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6bff4e98-f5a6-4d39-9765-7cf24ac571bb')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6bff4e98-f5a6-4d39-9765-7cf24ac571bb button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6bff4e98-f5a6-4d39-9765-7cf24ac571bb');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 7
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "##Removal of punctuation\n",
        "# drop the new column created in last cell\n",
        "dt.drop([\"text_lower\"], axis=1, inplace=True)\n",
        "\n",
        "PUNCT_TO_REMOVE = string.punctuation\n",
        "def remove_punctuation(reviews_text):\n",
        "    \"\"\"custom function to remove the punctuation\"\"\"\n",
        "    return reviews_text.translate(str.maketrans('', '', PUNCT_TO_REMOVE))\n",
        "\n",
        "dt[\"text_wo_punct\"] = dt[\"reviews.text\"].apply(lambda reviews_text: remove_punctuation(reviews_text))\n",
        "dt.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 397
        },
        "id": "8LeS69l_kDjw",
        "outputId": "10731875-5961-4063-f80d-6279728b9d96"
      },
      "execution_count": 8,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-8-adea8b5fe909>:3: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  dt.drop([\"text_lower\"], axis=1, inplace=True)\n",
            "<ipython-input-8-adea8b5fe909>:10: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  dt[\"text_wo_punct\"] = dt[\"reviews.text\"].apply(lambda reviews_text: remove_punctuation(reviews_text))\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                        reviews.text  \\\n",
              "0  This product so far has not disappointed. My c...   \n",
              "1  great for beginner or experienced person. Boug...   \n",
              "2  Inexpensive tablet for him to use and learn on...   \n",
              "3  I've had my Fire HD 8 two weeks now and I love...   \n",
              "4  I bought this for my grand daughter when she c...   \n",
              "\n",
              "                                       text_wo_punct  \n",
              "0  This product so far has not disappointed My ch...  \n",
              "1  great for beginner or experienced person Bough...  \n",
              "2  Inexpensive tablet for him to use and learn on...  \n",
              "3  Ive had my Fire HD 8 two weeks now and I love ...  \n",
              "4  I bought this for my grand daughter when she c...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-8b46e564-6907-4b09-b0ee-daf000b92fb9\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>reviews.text</th>\n",
              "      <th>text_wo_punct</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>This product so far has not disappointed. My c...</td>\n",
              "      <td>This product so far has not disappointed My ch...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>great for beginner or experienced person. Boug...</td>\n",
              "      <td>great for beginner or experienced person Bough...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Inexpensive tablet for him to use and learn on...</td>\n",
              "      <td>Inexpensive tablet for him to use and learn on...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>I've had my Fire HD 8 two weeks now and I love...</td>\n",
              "      <td>Ive had my Fire HD 8 two weeks now and I love ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I bought this for my grand daughter when she c...</td>\n",
              "      <td>I bought this for my grand daughter when she c...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-8b46e564-6907-4b09-b0ee-daf000b92fb9')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-8b46e564-6907-4b09-b0ee-daf000b92fb9 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-8b46e564-6907-4b09-b0ee-daf000b92fb9');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 8
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "##Removal of stopwords"
      ],
      "metadata": {
        "id": "O-ji74uLkx8R"
      },
      "execution_count": 9,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('stopwords')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "oDAGYh9olgSi",
        "outputId": "2f59da71-b60f-475a-ce77-67b041f52bfe"
      },
      "execution_count": 10,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package stopwords to /root/nltk_data...\n",
            "[nltk_data]   Package stopwords is already up-to-date!\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 10
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk.corpus import stopwords\n",
        "\", \".join(stopwords.words('english'))"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 122
        },
        "id": "sDVBZTTKlkl7",
        "outputId": "c32ce297-618c-480d-8f72-44539e3fcf52"
      },
      "execution_count": 11,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "\"i, me, my, myself, we, our, ours, ourselves, you, you're, you've, you'll, you'd, your, yours, yourself, yourselves, he, him, his, himself, she, she's, her, hers, herself, it, it's, its, itself, they, them, their, theirs, themselves, what, which, who, whom, this, that, that'll, these, those, am, is, are, was, were, be, been, being, have, has, had, having, do, does, did, doing, a, an, the, and, but, if, or, because, as, until, while, of, at, by, for, with, about, against, between, into, through, during, before, after, above, below, to, from, up, down, in, out, on, off, over, under, again, further, then, once, here, there, when, where, why, how, all, any, both, each, few, more, most, other, some, such, no, nor, not, only, own, same, so, than, too, very, s, t, can, will, just, don, don't, should, should've, now, d, ll, m, o, re, ve, y, ain, aren, aren't, couldn, couldn't, didn, didn't, doesn, doesn't, hadn, hadn't, hasn, hasn't, haven, haven't, isn, isn't, ma, mightn, mightn't, mustn, mustn't, needn, needn't, shan, shan't, shouldn, shouldn't, wasn, wasn't, weren, weren't, won, won't, wouldn, wouldn't\""
            ],
            "application/vnd.google.colaboratory.intrinsic+json": {
              "type": "string"
            }
          },
          "metadata": {},
          "execution_count": 11
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "STOPWORDS = set(stopwords.words('english'))\n",
        "def remove_stopwords(reviews_text):\n",
        "    \"\"\"custom function to remove the stopwords\"\"\"\n",
        "    return \" \".join([word for word in str(reviews_text).split() if word not in STOPWORDS])\n",
        "\n",
        "dt[\"text_wo_stop\"] = dt[\"text_wo_punct\"].apply(lambda reviews_text: remove_stopwords(reviews_text))\n",
        "dt.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 310
        },
        "id": "U4Bg_DTelrOg",
        "outputId": "9296a50b-ad9e-464d-99fa-aeec4540a15a"
      },
      "execution_count": 12,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-12-4ac466d30b8d>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  dt[\"text_wo_stop\"] = dt[\"text_wo_punct\"].apply(lambda reviews_text: remove_stopwords(reviews_text))\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                        reviews.text  \\\n",
              "0  This product so far has not disappointed. My c...   \n",
              "1  great for beginner or experienced person. Boug...   \n",
              "2  Inexpensive tablet for him to use and learn on...   \n",
              "3  I've had my Fire HD 8 two weeks now and I love...   \n",
              "4  I bought this for my grand daughter when she c...   \n",
              "\n",
              "                                       text_wo_punct  \\\n",
              "0  This product so far has not disappointed My ch...   \n",
              "1  great for beginner or experienced person Bough...   \n",
              "2  Inexpensive tablet for him to use and learn on...   \n",
              "3  Ive had my Fire HD 8 two weeks now and I love ...   \n",
              "4  I bought this for my grand daughter when she c...   \n",
              "\n",
              "                                        text_wo_stop  \n",
              "0  This product far disappointed My children love...  \n",
              "1  great beginner experienced person Bought gift ...  \n",
              "2  Inexpensive tablet use learn step NABI He thri...  \n",
              "3  Ive Fire HD 8 two weeks I love This tablet gre...  \n",
              "4  I bought grand daughter comes visit I set user...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-bf74ca30-353f-4c01-a53c-e3ec75218272\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>reviews.text</th>\n",
              "      <th>text_wo_punct</th>\n",
              "      <th>text_wo_stop</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>This product so far has not disappointed. My c...</td>\n",
              "      <td>This product so far has not disappointed My ch...</td>\n",
              "      <td>This product far disappointed My children love...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>great for beginner or experienced person. Boug...</td>\n",
              "      <td>great for beginner or experienced person Bough...</td>\n",
              "      <td>great beginner experienced person Bought gift ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Inexpensive tablet for him to use and learn on...</td>\n",
              "      <td>Inexpensive tablet for him to use and learn on...</td>\n",
              "      <td>Inexpensive tablet use learn step NABI He thri...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>I've had my Fire HD 8 two weeks now and I love...</td>\n",
              "      <td>Ive had my Fire HD 8 two weeks now and I love ...</td>\n",
              "      <td>Ive Fire HD 8 two weeks I love This tablet gre...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I bought this for my grand daughter when she c...</td>\n",
              "      <td>I bought this for my grand daughter when she c...</td>\n",
              "      <td>I bought grand daughter comes visit I set user...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-bf74ca30-353f-4c01-a53c-e3ec75218272')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-bf74ca30-353f-4c01-a53c-e3ec75218272 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-bf74ca30-353f-4c01-a53c-e3ec75218272');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 12
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## Removal of frequent word"
      ],
      "metadata": {
        "id": "FcVllT2rmCgL"
      },
      "execution_count": 13,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from collections import Counter\n",
        "cnt = Counter()\n",
        "for text in dt[\"text_wo_stop\"].values:\n",
        "    for word in text.split():\n",
        "        cnt[word] += 1\n",
        "\n",
        "cnt.most_common(10)"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "79MO6fsgmNmY",
        "outputId": "e9b5b8e0-fcac-41ff-9fa2-9f87663b8187"
      },
      "execution_count": 14,
      "outputs": [
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "[('I', 4381),\n",
              " ('tablet', 2015),\n",
              " ('great', 1248),\n",
              " ('use', 1113),\n",
              " ('price', 926),\n",
              " ('The', 887),\n",
              " ('good', 743),\n",
              " ('This', 741),\n",
              " ('Kindle', 708),\n",
              " ('one', 686)]"
            ]
          },
          "metadata": {},
          "execution_count": 14
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "FREQWORDS = set([w for (w, wc) in cnt.most_common(10)])\n",
        "def remove_freqwords(text):\n",
        "    \"\"\"custom function to remove the frequent words\"\"\"\n",
        "    return \" \".join([word for word in str(text).split() if word not in FREQWORDS])\n",
        "\n",
        "dt[\"text_wo_stopfreq\"] = dt[\"text_wo_stop\"].apply(lambda text: remove_freqwords(text))\n",
        "dt.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 310
        },
        "id": "XiuCus88m0Yv",
        "outputId": "7a14f107-82fd-48c4-8f8b-6f519169e02d"
      },
      "execution_count": 15,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-15-c981024f35eb>:6: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  dt[\"text_wo_stopfreq\"] = dt[\"text_wo_stop\"].apply(lambda text: remove_freqwords(text))\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                        reviews.text  \\\n",
              "0  This product so far has not disappointed. My c...   \n",
              "1  great for beginner or experienced person. Boug...   \n",
              "2  Inexpensive tablet for him to use and learn on...   \n",
              "3  I've had my Fire HD 8 two weeks now and I love...   \n",
              "4  I bought this for my grand daughter when she c...   \n",
              "\n",
              "                                       text_wo_punct  \\\n",
              "0  This product so far has not disappointed My ch...   \n",
              "1  great for beginner or experienced person Bough...   \n",
              "2  Inexpensive tablet for him to use and learn on...   \n",
              "3  Ive had my Fire HD 8 two weeks now and I love ...   \n",
              "4  I bought this for my grand daughter when she c...   \n",
              "\n",
              "                                        text_wo_stop  \\\n",
              "0  This product far disappointed My children love...   \n",
              "1  great beginner experienced person Bought gift ...   \n",
              "2  Inexpensive tablet use learn step NABI He thri...   \n",
              "3  Ive Fire HD 8 two weeks I love This tablet gre...   \n",
              "4  I bought grand daughter comes visit I set user...   \n",
              "\n",
              "                                    text_wo_stopfreq  \n",
              "0  product far disappointed My children love like...  \n",
              "1      beginner experienced person Bought gift loves  \n",
              "2  Inexpensive learn step NABI He thrilled learn ...  \n",
              "3  Ive Fire HD 8 two weeks love valueWe Prime Mem...  \n",
              "4  bought grand daughter comes visit set user ent...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-e8b02e60-ca2f-4740-ac21-b22dd5eca612\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>reviews.text</th>\n",
              "      <th>text_wo_punct</th>\n",
              "      <th>text_wo_stop</th>\n",
              "      <th>text_wo_stopfreq</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>This product so far has not disappointed. My c...</td>\n",
              "      <td>This product so far has not disappointed My ch...</td>\n",
              "      <td>This product far disappointed My children love...</td>\n",
              "      <td>product far disappointed My children love like...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>great for beginner or experienced person. Boug...</td>\n",
              "      <td>great for beginner or experienced person Bough...</td>\n",
              "      <td>great beginner experienced person Bought gift ...</td>\n",
              "      <td>beginner experienced person Bought gift loves</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Inexpensive tablet for him to use and learn on...</td>\n",
              "      <td>Inexpensive tablet for him to use and learn on...</td>\n",
              "      <td>Inexpensive tablet use learn step NABI He thri...</td>\n",
              "      <td>Inexpensive learn step NABI He thrilled learn ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>I've had my Fire HD 8 two weeks now and I love...</td>\n",
              "      <td>Ive had my Fire HD 8 two weeks now and I love ...</td>\n",
              "      <td>Ive Fire HD 8 two weeks I love This tablet gre...</td>\n",
              "      <td>Ive Fire HD 8 two weeks love valueWe Prime Mem...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I bought this for my grand daughter when she c...</td>\n",
              "      <td>I bought this for my grand daughter when she c...</td>\n",
              "      <td>I bought grand daughter comes visit I set user...</td>\n",
              "      <td>bought grand daughter comes visit set user ent...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-e8b02e60-ca2f-4740-ac21-b22dd5eca612')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-e8b02e60-ca2f-4740-ac21-b22dd5eca612 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-e8b02e60-ca2f-4740-ac21-b22dd5eca612');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 15
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## Removal of rare words"
      ],
      "metadata": {
        "id": "DyHzk0dhnWgF"
      },
      "execution_count": 16,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Drop the two columns which are no more needed\n",
        "dt.drop([\"text_wo_punct\", \"text_wo_stop\"], axis=1, inplace=True)\n",
        "\n",
        "n_rare_words = 10\n",
        "RAREWORDS = set([w for (w, wc) in cnt.most_common()[:-n_rare_words-1:-1]])\n",
        "def remove_rarewords(text):\n",
        "    \"\"\"custom function to remove the rare words\"\"\"\n",
        "    return \" \".join([word for word in str(text).split() if word not in RAREWORDS])\n",
        "\n",
        "dt[\"text_wo_stopfreqrare\"] = dt[\"text_wo_stopfreq\"].apply(lambda text: remove_rarewords(text))\n",
        "dt.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 397
        },
        "id": "AV1MoRu3rV6b",
        "outputId": "f468159e-f8cc-4725-de63-cf8aaaa6077e"
      },
      "execution_count": 17,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-17-a2cca37ebf83>:2: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  dt.drop([\"text_wo_punct\", \"text_wo_stop\"], axis=1, inplace=True)\n",
            "<ipython-input-17-a2cca37ebf83>:10: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  dt[\"text_wo_stopfreqrare\"] = dt[\"text_wo_stopfreq\"].apply(lambda text: remove_rarewords(text))\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                        reviews.text  \\\n",
              "0  This product so far has not disappointed. My c...   \n",
              "1  great for beginner or experienced person. Boug...   \n",
              "2  Inexpensive tablet for him to use and learn on...   \n",
              "3  I've had my Fire HD 8 two weeks now and I love...   \n",
              "4  I bought this for my grand daughter when she c...   \n",
              "\n",
              "                                    text_wo_stopfreq  \\\n",
              "0  product far disappointed My children love like...   \n",
              "1      beginner experienced person Bought gift loves   \n",
              "2  Inexpensive learn step NABI He thrilled learn ...   \n",
              "3  Ive Fire HD 8 two weeks love valueWe Prime Mem...   \n",
              "4  bought grand daughter comes visit set user ent...   \n",
              "\n",
              "                                text_wo_stopfreqrare  \n",
              "0  product far disappointed My children love like...  \n",
              "1      beginner experienced person Bought gift loves  \n",
              "2  Inexpensive learn step NABI He thrilled learn ...  \n",
              "3  Ive Fire HD 8 two weeks love valueWe Prime Mem...  \n",
              "4  bought grand daughter comes visit set user ent...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-6ba43936-478b-4618-a907-c4e14617494f\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>reviews.text</th>\n",
              "      <th>text_wo_stopfreq</th>\n",
              "      <th>text_wo_stopfreqrare</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>This product so far has not disappointed. My c...</td>\n",
              "      <td>product far disappointed My children love like...</td>\n",
              "      <td>product far disappointed My children love like...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>great for beginner or experienced person. Boug...</td>\n",
              "      <td>beginner experienced person Bought gift loves</td>\n",
              "      <td>beginner experienced person Bought gift loves</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Inexpensive tablet for him to use and learn on...</td>\n",
              "      <td>Inexpensive learn step NABI He thrilled learn ...</td>\n",
              "      <td>Inexpensive learn step NABI He thrilled learn ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>I've had my Fire HD 8 two weeks now and I love...</td>\n",
              "      <td>Ive Fire HD 8 two weeks love valueWe Prime Mem...</td>\n",
              "      <td>Ive Fire HD 8 two weeks love valueWe Prime Mem...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I bought this for my grand daughter when she c...</td>\n",
              "      <td>bought grand daughter comes visit set user ent...</td>\n",
              "      <td>bought grand daughter comes visit set user ent...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-6ba43936-478b-4618-a907-c4e14617494f')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-6ba43936-478b-4618-a907-c4e14617494f button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-6ba43936-478b-4618-a907-c4e14617494f');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 17
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "##Lemmatization"
      ],
      "metadata": {
        "id": "zZw8q1Odrmez"
      },
      "execution_count": 19,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "import nltk\n",
        "nltk.download('wordnet')"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/"
        },
        "id": "xhMqE0NBsN5d",
        "outputId": "d926af29-2e4b-4d33-c4e2-49daa605d6b5"
      },
      "execution_count": 20,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "[nltk_data] Downloading package wordnet to /root/nltk_data...\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "True"
            ]
          },
          "metadata": {},
          "execution_count": 20
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk.stem import WordNetLemmatizer\n",
        "\n",
        "lemmatizer = WordNetLemmatizer()\n",
        "def lemmatize_words(text):\n",
        "    return \" \".join([lemmatizer.lemmatize(word) for word in text.split()])\n",
        "\n",
        "dt[\"text_lemmatized\"] = dt[\"reviews.text\"].apply(lambda text: lemmatize_words(text))\n",
        "dt.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 310
        },
        "id": "2Y33y4rQsVdm",
        "outputId": "2e81d62d-ccd8-4f87-d249-637a0e5c9dc5"
      },
      "execution_count": 22,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-22-6c59e294ac6f>:7: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  dt[\"text_lemmatized\"] = dt[\"reviews.text\"].apply(lambda text: lemmatize_words(text))\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                        reviews.text  \\\n",
              "0  This product so far has not disappointed. My c...   \n",
              "1  great for beginner or experienced person. Boug...   \n",
              "2  Inexpensive tablet for him to use and learn on...   \n",
              "3  I've had my Fire HD 8 two weeks now and I love...   \n",
              "4  I bought this for my grand daughter when she c...   \n",
              "\n",
              "                                    text_wo_stopfreq  \\\n",
              "0  product far disappointed My children love like...   \n",
              "1      beginner experienced person Bought gift loves   \n",
              "2  Inexpensive learn step NABI He thrilled learn ...   \n",
              "3  Ive Fire HD 8 two weeks love valueWe Prime Mem...   \n",
              "4  bought grand daughter comes visit set user ent...   \n",
              "\n",
              "                                text_wo_stopfreqrare  \\\n",
              "0  product far disappointed My children love like...   \n",
              "1      beginner experienced person Bought gift loves   \n",
              "2  Inexpensive learn step NABI He thrilled learn ...   \n",
              "3  Ive Fire HD 8 two weeks love valueWe Prime Mem...   \n",
              "4  bought grand daughter comes visit set user ent...   \n",
              "\n",
              "                                     text_lemmatized  \n",
              "0  This product so far ha not disappointed. My ch...  \n",
              "1  great for beginner or experienced person. Boug...  \n",
              "2  Inexpensive tablet for him to use and learn on...  \n",
              "3  I've had my Fire HD 8 two week now and I love ...  \n",
              "4  I bought this for my grand daughter when she c...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-3d612395-e572-4f5e-911d-702f5f9e3820\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>reviews.text</th>\n",
              "      <th>text_wo_stopfreq</th>\n",
              "      <th>text_wo_stopfreqrare</th>\n",
              "      <th>text_lemmatized</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>This product so far has not disappointed. My c...</td>\n",
              "      <td>product far disappointed My children love like...</td>\n",
              "      <td>product far disappointed My children love like...</td>\n",
              "      <td>This product so far ha not disappointed. My ch...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>great for beginner or experienced person. Boug...</td>\n",
              "      <td>beginner experienced person Bought gift loves</td>\n",
              "      <td>beginner experienced person Bought gift loves</td>\n",
              "      <td>great for beginner or experienced person. Boug...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Inexpensive tablet for him to use and learn on...</td>\n",
              "      <td>Inexpensive learn step NABI He thrilled learn ...</td>\n",
              "      <td>Inexpensive learn step NABI He thrilled learn ...</td>\n",
              "      <td>Inexpensive tablet for him to use and learn on...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>I've had my Fire HD 8 two weeks now and I love...</td>\n",
              "      <td>Ive Fire HD 8 two weeks love valueWe Prime Mem...</td>\n",
              "      <td>Ive Fire HD 8 two weeks love valueWe Prime Mem...</td>\n",
              "      <td>I've had my Fire HD 8 two week now and I love ...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I bought this for my grand daughter when she c...</td>\n",
              "      <td>bought grand daughter comes visit set user ent...</td>\n",
              "      <td>bought grand daughter comes visit set user ent...</td>\n",
              "      <td>I bought this for my grand daughter when she c...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-3d612395-e572-4f5e-911d-702f5f9e3820')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-3d612395-e572-4f5e-911d-702f5f9e3820 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-3d612395-e572-4f5e-911d-702f5f9e3820');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 22
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "## Stemming"
      ],
      "metadata": {
        "id": "ZgSPuzLwsgEu"
      },
      "execution_count": 23,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from nltk.stem.porter import PorterStemmer\n",
        "\n",
        "# Drop the two columns\n",
        "dt.drop([\"text_wo_stopfreq\", \"text_wo_stopfreqrare\"], axis=1, inplace=True)\n",
        "\n",
        "stemmer = PorterStemmer()\n",
        "def stem_words(text):\n",
        "    return \" \".join([stemmer.stem(word) for word in text.split()])\n",
        "\n",
        "dt[\"text_stemmed\"] = dt[\"reviews.text\"].apply(lambda text: stem_words(text))\n",
        "dt.head()"
      ],
      "metadata": {
        "colab": {
          "base_uri": "https://localhost:8080/",
          "height": 397
        },
        "id": "WAz6lfjxs0lL",
        "outputId": "5fc41170-2f65-4a95-c940-d14d91be1b19"
      },
      "execution_count": 24,
      "outputs": [
        {
          "output_type": "stream",
          "name": "stderr",
          "text": [
            "<ipython-input-24-c96158ce09f6>:4: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  dt.drop([\"text_wo_stopfreq\", \"text_wo_stopfreqrare\"], axis=1, inplace=True)\n",
            "<ipython-input-24-c96158ce09f6>:10: SettingWithCopyWarning: \n",
            "A value is trying to be set on a copy of a slice from a DataFrame.\n",
            "Try using .loc[row_indexer,col_indexer] = value instead\n",
            "\n",
            "See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy\n",
            "  dt[\"text_stemmed\"] = dt[\"reviews.text\"].apply(lambda text: stem_words(text))\n"
          ]
        },
        {
          "output_type": "execute_result",
          "data": {
            "text/plain": [
              "                                        reviews.text  \\\n",
              "0  This product so far has not disappointed. My c...   \n",
              "1  great for beginner or experienced person. Boug...   \n",
              "2  Inexpensive tablet for him to use and learn on...   \n",
              "3  I've had my Fire HD 8 two weeks now and I love...   \n",
              "4  I bought this for my grand daughter when she c...   \n",
              "\n",
              "                                     text_lemmatized  \\\n",
              "0  This product so far ha not disappointed. My ch...   \n",
              "1  great for beginner or experienced person. Boug...   \n",
              "2  Inexpensive tablet for him to use and learn on...   \n",
              "3  I've had my Fire HD 8 two week now and I love ...   \n",
              "4  I bought this for my grand daughter when she c...   \n",
              "\n",
              "                                        text_stemmed  \n",
              "0  thi product so far ha not disappointed. my chi...  \n",
              "1  great for beginn or experienc person. bought a...  \n",
              "2  inexpens tablet for him to use and learn on, s...  \n",
              "3  i'v had my fire hd 8 two week now and i love i...  \n",
              "4  i bought thi for my grand daughter when she co...  "
            ],
            "text/html": [
              "\n",
              "  <div id=\"df-d0336518-c476-485f-b028-62c41a0291a4\">\n",
              "    <div class=\"colab-df-container\">\n",
              "      <div>\n",
              "<style scoped>\n",
              "    .dataframe tbody tr th:only-of-type {\n",
              "        vertical-align: middle;\n",
              "    }\n",
              "\n",
              "    .dataframe tbody tr th {\n",
              "        vertical-align: top;\n",
              "    }\n",
              "\n",
              "    .dataframe thead th {\n",
              "        text-align: right;\n",
              "    }\n",
              "</style>\n",
              "<table border=\"1\" class=\"dataframe\">\n",
              "  <thead>\n",
              "    <tr style=\"text-align: right;\">\n",
              "      <th></th>\n",
              "      <th>reviews.text</th>\n",
              "      <th>text_lemmatized</th>\n",
              "      <th>text_stemmed</th>\n",
              "    </tr>\n",
              "  </thead>\n",
              "  <tbody>\n",
              "    <tr>\n",
              "      <th>0</th>\n",
              "      <td>This product so far has not disappointed. My c...</td>\n",
              "      <td>This product so far ha not disappointed. My ch...</td>\n",
              "      <td>thi product so far ha not disappointed. my chi...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>1</th>\n",
              "      <td>great for beginner or experienced person. Boug...</td>\n",
              "      <td>great for beginner or experienced person. Boug...</td>\n",
              "      <td>great for beginn or experienc person. bought a...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>2</th>\n",
              "      <td>Inexpensive tablet for him to use and learn on...</td>\n",
              "      <td>Inexpensive tablet for him to use and learn on...</td>\n",
              "      <td>inexpens tablet for him to use and learn on, s...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>3</th>\n",
              "      <td>I've had my Fire HD 8 two weeks now and I love...</td>\n",
              "      <td>I've had my Fire HD 8 two week now and I love ...</td>\n",
              "      <td>i'v had my fire hd 8 two week now and i love i...</td>\n",
              "    </tr>\n",
              "    <tr>\n",
              "      <th>4</th>\n",
              "      <td>I bought this for my grand daughter when she c...</td>\n",
              "      <td>I bought this for my grand daughter when she c...</td>\n",
              "      <td>i bought thi for my grand daughter when she co...</td>\n",
              "    </tr>\n",
              "  </tbody>\n",
              "</table>\n",
              "</div>\n",
              "      <button class=\"colab-df-convert\" onclick=\"convertToInteractive('df-d0336518-c476-485f-b028-62c41a0291a4')\"\n",
              "              title=\"Convert this dataframe to an interactive table.\"\n",
              "              style=\"display:none;\">\n",
              "        \n",
              "  <svg xmlns=\"http://www.w3.org/2000/svg\" height=\"24px\"viewBox=\"0 0 24 24\"\n",
              "       width=\"24px\">\n",
              "    <path d=\"M0 0h24v24H0V0z\" fill=\"none\"/>\n",
              "    <path d=\"M18.56 5.44l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94zm-11 1L8.5 8.5l.94-2.06 2.06-.94-2.06-.94L8.5 2.5l-.94 2.06-2.06.94zm10 10l.94 2.06.94-2.06 2.06-.94-2.06-.94-.94-2.06-.94 2.06-2.06.94z\"/><path d=\"M17.41 7.96l-1.37-1.37c-.4-.4-.92-.59-1.43-.59-.52 0-1.04.2-1.43.59L10.3 9.45l-7.72 7.72c-.78.78-.78 2.05 0 2.83L4 21.41c.39.39.9.59 1.41.59.51 0 1.02-.2 1.41-.59l7.78-7.78 2.81-2.81c.8-.78.8-2.07 0-2.86zM5.41 20L4 18.59l7.72-7.72 1.47 1.35L5.41 20z\"/>\n",
              "  </svg>\n",
              "      </button>\n",
              "      \n",
              "  <style>\n",
              "    .colab-df-container {\n",
              "      display:flex;\n",
              "      flex-wrap:wrap;\n",
              "      gap: 12px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert {\n",
              "      background-color: #E8F0FE;\n",
              "      border: none;\n",
              "      border-radius: 50%;\n",
              "      cursor: pointer;\n",
              "      display: none;\n",
              "      fill: #1967D2;\n",
              "      height: 32px;\n",
              "      padding: 0 0 0 0;\n",
              "      width: 32px;\n",
              "    }\n",
              "\n",
              "    .colab-df-convert:hover {\n",
              "      background-color: #E2EBFA;\n",
              "      box-shadow: 0px 1px 2px rgba(60, 64, 67, 0.3), 0px 1px 3px 1px rgba(60, 64, 67, 0.15);\n",
              "      fill: #174EA6;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert {\n",
              "      background-color: #3B4455;\n",
              "      fill: #D2E3FC;\n",
              "    }\n",
              "\n",
              "    [theme=dark] .colab-df-convert:hover {\n",
              "      background-color: #434B5C;\n",
              "      box-shadow: 0px 1px 3px 1px rgba(0, 0, 0, 0.15);\n",
              "      filter: drop-shadow(0px 1px 2px rgba(0, 0, 0, 0.3));\n",
              "      fill: #FFFFFF;\n",
              "    }\n",
              "  </style>\n",
              "\n",
              "      <script>\n",
              "        const buttonEl =\n",
              "          document.querySelector('#df-d0336518-c476-485f-b028-62c41a0291a4 button.colab-df-convert');\n",
              "        buttonEl.style.display =\n",
              "          google.colab.kernel.accessAllowed ? 'block' : 'none';\n",
              "\n",
              "        async function convertToInteractive(key) {\n",
              "          const element = document.querySelector('#df-d0336518-c476-485f-b028-62c41a0291a4');\n",
              "          const dataTable =\n",
              "            await google.colab.kernel.invokeFunction('convertToInteractive',\n",
              "                                                     [key], {});\n",
              "          if (!dataTable) return;\n",
              "\n",
              "          const docLinkHtml = 'Like what you see? Visit the ' +\n",
              "            '<a target=\"_blank\" href=https://colab.research.google.com/notebooks/data_table.ipynb>data table notebook</a>'\n",
              "            + ' to learn more about interactive tables.';\n",
              "          element.innerHTML = '';\n",
              "          dataTable['output_type'] = 'display_data';\n",
              "          await google.colab.output.renderOutput(dataTable, element);\n",
              "          const docLink = document.createElement('div');\n",
              "          docLink.innerHTML = docLinkHtml;\n",
              "          element.appendChild(docLink);\n",
              "        }\n",
              "      </script>\n",
              "    </div>\n",
              "  </div>\n",
              "  "
            ]
          },
          "metadata": {},
          "execution_count": 24
        }
      ]
    },
    {
      "cell_type": "code",
      "source": [],
      "metadata": {
        "id": "S9iEopyus_Da"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}