task_TextCleaning.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "f042b2b3",
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd \n",
    "import matplotlib.pyplot as plt\n",
    "import nltk as nltk\n",
    "import string as str\n",
    "from nltk.tokenize import word_tokenize\n",
    "import nltk.data\n",
    "import csv\n",
    "import codecs\n",
    "from nltk.corpus import stopwords\n",
    "from string import punctuation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6994eaa0",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Language</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Nature, in the broadest sense, is the natural, physical, material world or universe.</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"Nature\" can refer to the phenomena of the physical world, and also to life in general.</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The study of nature is a large, if not the only, part of science.</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Although humans are part of nature, human activity is often understood as a separate category from other natural phenomena.</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[1] The word nature is borrowed from the Old French nature and is derived from the Latin word natura, or \"essential qualities, innate disposition\", and in ancient times, literally meant \"birth\".</td>\n",
       "      <td>English</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10332</th>\n",
       "      <td>ನಿಮ್ಮ ತಪ್ಪು ಏನು ಬಂದಿದೆಯೆಂದರೆ ಆ ದಿನದಿಂದ ನಿಮಗೆ ಒಳ್ಳೆಯದನ್ನು ನೀಡೋಣ.</td>\n",
       "      <td>Kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10333</th>\n",
       "      <td>ನಾರ್ಸಿಸಾ ತಾನು ಮೊದಲಿಗೆ ಹೆಣಗಾಡುತ್ತಿದ್ದ ಮಾರ್ಗಗಳನ್ನು ಬದಲಾಯಿಸಿದಳು ಆದರೆ ನಿಧಾನವಾಗಿ ಮತ್ತು ತಾಳ್ಮೆಯಿಂದ ಅವಳು ಒಂದು ದಿನ ಮೆಲ್ಲಿ ಆಗಲು ಪ್ರಾರಂಭಿಸಿದಳು ಮೆಲ್ಲಿ ಮತ್ತು ಟೆರ್ರಿ ಮತ್ತೆ ಮರಿಯನ್ ಅವರನ್ನು ಭೇಟಿಯಾಗಲು ಬಂದರು ಓಹ್ ಹಲ...</td>\n",
       "      <td>Kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10334</th>\n",
       "      <td>ಹೇಗೆ ' ನಾರ್ಸಿಸಿಸಮ್ ಈಗ ಮರಿಯನ್ ಅವರಿಗೆ ಸಂಭವಿಸಿದ ಎಲ್ಲವನ್ನೂ ಹೇಳಿದೆ ಮತ್ತು ಅವಳು ಆ ಸಮಯದಿಂದ ತುಂಬಾ ಬದಲಾಗಿದ್ದಾಳೆ.</td>\n",
       "      <td>Kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10335</th>\n",
       "      <td>ಅವಳು ಈಗ ಹೆಚ್ಚು ಚಿನ್ನದ ಬ್ರೆಡ್ ಬಯಸುವುದಿಲ್ಲ ಎಂದು ನಾನು ess ಹಿಸಿದ್ದೇನೆ.</td>\n",
       "      <td>Kannada</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>10336</th>\n",
       "      <td>ಟೆರ್ರಿ ನೀವು ನಿಜವಾಗಿಯೂ ಆ ದೇವದೂತನಂತೆ ಸ್ವಲ್ಪ ಕಾಣುತ್ತಿದ್ದೀರಿ ಆದರೆ ನಾನು ಏನು ನೋಡುತ್ತಿದ್ದೇನೆ ನೀವು ಹೇಗೆ ಅವನಾಗಬಹುದು ನೀವು ಇಬ್ಬರು ತುಂಬಾ ಒಳ್ಳೆಯವರು</td>\n",
       "      <td>Kannada</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>10337 rows × 2 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                          Text  \\\n",
       "0                                                                                                                         Nature, in the broadest sense, is the natural, physical, material world or universe.   \n",
       "1                                                                                                                      \"Nature\" can refer to the phenomena of the physical world, and also to life in general.   \n",
       "2                                                                                                                                            The study of nature is a large, if not the only, part of science.   \n",
       "3                                                                                  Although humans are part of nature, human activity is often understood as a separate category from other natural phenomena.   \n",
       "4           [1] The word nature is borrowed from the Old French nature and is derived from the Latin word natura, or \"essential qualities, innate disposition\", and in ancient times, literally meant \"birth\".   \n",
       "...                                                                                                                                                                                                        ...   \n",
       "10332                                                                                                                                          ನಿಮ್ಮ ತಪ್ಪು ಏನು ಬಂದಿದೆಯೆಂದರೆ ಆ ದಿನದಿಂದ ನಿಮಗೆ ಒಳ್ಳೆಯದನ್ನು ನೀಡೋಣ.   \n",
       "10333  ನಾರ್ಸಿಸಾ ತಾನು ಮೊದಲಿಗೆ ಹೆಣಗಾಡುತ್ತಿದ್ದ ಮಾರ್ಗಗಳನ್ನು ಬದಲಾಯಿಸಿದಳು ಆದರೆ ನಿಧಾನವಾಗಿ ಮತ್ತು ತಾಳ್ಮೆಯಿಂದ ಅವಳು ಒಂದು ದಿನ ಮೆಲ್ಲಿ ಆಗಲು ಪ್ರಾರಂಭಿಸಿದಳು ಮೆಲ್ಲಿ ಮತ್ತು ಟೆರ್ರಿ ಮತ್ತೆ ಮರಿಯನ್ ಅವರನ್ನು ಭೇಟಿಯಾಗಲು ಬಂದರು ಓಹ್ ಹಲ...   \n",
       "10334                                                                                                   ಹೇಗೆ ' ನಾರ್ಸಿಸಿಸಮ್ ಈಗ ಮರಿಯನ್ ಅವರಿಗೆ ಸಂಭವಿಸಿದ ಎಲ್ಲವನ್ನೂ ಹೇಳಿದೆ ಮತ್ತು ಅವಳು ಆ ಸಮಯದಿಂದ ತುಂಬಾ ಬದಲಾಗಿದ್ದಾಳೆ.   \n",
       "10335                                                                                                                                       ಅವಳು ಈಗ ಹೆಚ್ಚು ಚಿನ್ನದ ಬ್ರೆಡ್ ಬಯಸುವುದಿಲ್ಲ ಎಂದು ನಾನು ess ಹಿಸಿದ್ದೇನೆ.   \n",
       "10336                                                                   ಟೆರ್ರಿ ನೀವು ನಿಜವಾಗಿಯೂ ಆ ದೇವದೂತನಂತೆ ಸ್ವಲ್ಪ ಕಾಣುತ್ತಿದ್ದೀರಿ ಆದರೆ ನಾನು ಏನು ನೋಡುತ್ತಿದ್ದೇನೆ ನೀವು ಹೇಗೆ ಅವನಾಗಬಹುದು ನೀವು ಇಬ್ಬರು ತುಂಬಾ ಒಳ್ಳೆಯವರು   \n",
       "\n",
       "      Language  \n",
       "0      English  \n",
       "1      English  \n",
       "2      English  \n",
       "3      English  \n",
       "4      English  \n",
       "...        ...  \n",
       "10332  Kannada  \n",
       "10333  Kannada  \n",
       "10334  Kannada  \n",
       "10335  Kannada  \n",
       "10336  Kannada  \n",
       "\n",
       "[10337 rows x 2 columns]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "data=pd.read_csv(\"Language Detection.csv\")\n",
    "pd.set_option('display.max_colwidth',200)\n",
    "data.head(10337)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "a1aa873b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0                                                                                                                           Nature, in the broadest sense, is the natural, physical, material world or universe.\n",
      "1                                                                                                                        \"Nature\" can refer to the phenomena of the physical world, and also to life in general.\n",
      "2                                                                                                                                              The study of nature is a large, if not the only, part of science.\n",
      "3                                                                                    Although humans are part of nature, human activity is often understood as a separate category from other natural phenomena.\n",
      "4             [1] The word nature is borrowed from the Old French nature and is derived from the Latin word natura, or \"essential qualities, innate disposition\", and in ancient times, literally meant \"birth\".\n",
      "                                                                                                          ...                                                                                                   \n",
      "10332                                                                                                                                            ನಿಮ್ಮ ತಪ್ಪು ಏನು ಬಂದಿದೆಯೆಂದರೆ ಆ ದಿನದಿಂದ ನಿಮಗೆ ಒಳ್ಳೆಯದನ್ನು ನೀಡೋಣ.\n",
      "10333    ನಾರ್ಸಿಸಾ ತಾನು ಮೊದಲಿಗೆ ಹೆಣಗಾಡುತ್ತಿದ್ದ ಮಾರ್ಗಗಳನ್ನು ಬದಲಾಯಿಸಿದಳು ಆದರೆ ನಿಧಾನವಾಗಿ ಮತ್ತು ತಾಳ್ಮೆಯಿಂದ ಅವಳು ಒಂದು ದಿನ ಮೆಲ್ಲಿ ಆಗಲು ಪ್ರಾರಂಭಿಸಿದಳು ಮೆಲ್ಲಿ ಮತ್ತು ಟೆರ್ರಿ ಮತ್ತೆ ಮರಿಯನ್ ಅವರನ್ನು ಭೇಟಿಯಾಗಲು ಬಂದರು ಓಹ್ ಹಲ...\n",
      "10334                                                                                                     ಹೇಗೆ ' ನಾರ್ಸಿಸಿಸಮ್ ಈಗ ಮರಿಯನ್ ಅವರಿಗೆ ಸಂಭವಿಸಿದ ಎಲ್ಲವನ್ನೂ ಹೇಳಿದೆ ಮತ್ತು ಅವಳು ಆ ಸಮಯದಿಂದ ತುಂಬಾ ಬದಲಾಗಿದ್ದಾಳೆ.\n",
      "10335                                                                                                                                         ಅವಳು ಈಗ ಹೆಚ್ಚು ಚಿನ್ನದ ಬ್ರೆಡ್ ಬಯಸುವುದಿಲ್ಲ ಎಂದು ನಾನು ess ಹಿಸಿದ್ದೇನೆ.\n",
      "10336                                                                     ಟೆರ್ರಿ ನೀವು ನಿಜವಾಗಿಯೂ ಆ ದೇವದೂತನಂತೆ ಸ್ವಲ್ಪ ಕಾಣುತ್ತಿದ್ದೀರಿ ಆದರೆ ನಾನು ಏನು ನೋಡುತ್ತಿದ್ದೇನೆ ನೀವು ಹೇಗೆ ಅವನಾಗಬಹುದು ನೀವು ಇಬ್ಬರು ತುಂಬಾ ಒಳ್ಳೆಯವರು\n",
      "Name: Text, Length: 10337, dtype: object\n"
     ]
    }
   ],
   "source": [
    "column_a = data[\"Text\"]\n",
    "print(column_a)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2217e37a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df=data.copy()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "e4fb7a68",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "English       1385\n",
       "French        1014\n",
       "Spanish        819\n",
       "Portugeese     739\n",
       "Italian        698\n",
       "Russian        692\n",
       "Sweedish       676\n",
       "Malayalam      594\n",
       "Dutch          546\n",
       "Arabic         536\n",
       "Turkish        474\n",
       "German         470\n",
       "Tamil          469\n",
       "Danish         428\n",
       "Kannada        369\n",
       "Greek          365\n",
       "Hindi           63\n",
       "Name: Language, dtype: int64"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.Language.value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "6da16e2f",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~\n"
     ]
    }
   ],
   "source": [
    "punct=str.punctuation\n",
    "print(punct)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "id": "fdb123d3",
   "metadata": {},
   "outputs": [],
   "source": [
    "pd.set_option('display.max_colwidth',200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "id": "51c5e523",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Language</th>\n",
       "      <th>text_nopunc</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Nature, in the broadest sense, is the natural, physical, material world or universe.</td>\n",
       "      <td>English</td>\n",
       "      <td>nature in the broadest sense is the natural physical material world or universe</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"Nature\" can refer to the phenomena of the physical world, and also to life in general.</td>\n",
       "      <td>English</td>\n",
       "      <td>nature can refer to the phenomena of the physical world and also to life in general</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The study of nature is a large, if not the only, part of science.</td>\n",
       "      <td>English</td>\n",
       "      <td>the study of nature is a large if not the only part of science</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Although humans are part of nature, human activity is often understood as a separate category from other natural phenomena.</td>\n",
       "      <td>English</td>\n",
       "      <td>although humans are part of nature human activity is often understood as a separate category from other natural phenomena</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[1] The word nature is borrowed from the Old French nature and is derived from the Latin word natura, or \"essential qualities, innate disposition\", and in ancient times, literally meant \"birth\".</td>\n",
       "      <td>English</td>\n",
       "      <td>1 the word nature is borrowed from the old french nature and is derived from the latin word natura or essential qualities innate disposition and in ancient times literally meant birth</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>495</th>\n",
       "      <td>While generally praising the article on William Clarke Quantrill, he quoted its conclusion as an example of such \"waffling\", which then stated: \"Some historians [...] remember him as an opportunis...</td>\n",
       "      <td>English</td>\n",
       "      <td>while generally praising the article on william clarke quantrill he quoted its conclusion as an example of such waffling which then stated some historians  remember him as an opportunistic bloodth...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>496</th>\n",
       "      <td>\"[190] Other critics have made similar charges that, even if Wikipedia articles are factually accurate, they are often written in a poor, almost unreadable style.</td>\n",
       "      <td>English</td>\n",
       "      <td>190 other critics have made similar charges that even if wikipedia articles are factually accurate they are often written in a poor almost unreadable style</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>497</th>\n",
       "      <td>Frequent Wikipedia critic Andrew Orlowski commented, \"Even when a Wikipedia entry is 100 percent factually correct, and those facts have been carefully chosen, it all too often reads as if it has ...</td>\n",
       "      <td>English</td>\n",
       "      <td>frequent wikipedia critic andrew orlowski commented even when a wikipedia entry is 100 percent factually correct and those facts have been carefully chosen it all too often reads as if it has been...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>498</th>\n",
       "      <td>\"[191] A study of Wikipedia articles on cancer was conducted in 2010 by Yaacov Lawrence of the Kimmel Cancer Center at Thomas Jefferson University.</td>\n",
       "      <td>English</td>\n",
       "      <td>191 a study of wikipedia articles on cancer was conducted in 2010 by yaacov lawrence of the kimmel cancer center at thomas jefferson university</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>499</th>\n",
       "      <td>The study was limited to those articles that could be found in the Physician Data Query and excluded those written at the \"start\" class or \"stub\" class level.</td>\n",
       "      <td>English</td>\n",
       "      <td>the study was limited to those articles that could be found in the physician data query and excluded those written at the start class or stub class level</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>500 rows × 3 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                        Text  \\\n",
       "0                                                                                                                       Nature, in the broadest sense, is the natural, physical, material world or universe.   \n",
       "1                                                                                                                    \"Nature\" can refer to the phenomena of the physical world, and also to life in general.   \n",
       "2                                                                                                                                          The study of nature is a large, if not the only, part of science.   \n",
       "3                                                                                Although humans are part of nature, human activity is often understood as a separate category from other natural phenomena.   \n",
       "4         [1] The word nature is borrowed from the Old French nature and is derived from the Latin word natura, or \"essential qualities, innate disposition\", and in ancient times, literally meant \"birth\".   \n",
       "..                                                                                                                                                                                                       ...   \n",
       "495  While generally praising the article on William Clarke Quantrill, he quoted its conclusion as an example of such \"waffling\", which then stated: \"Some historians [...] remember him as an opportunis...   \n",
       "496                                       \"[190] Other critics have made similar charges that, even if Wikipedia articles are factually accurate, they are often written in a poor, almost unreadable style.   \n",
       "497  Frequent Wikipedia critic Andrew Orlowski commented, \"Even when a Wikipedia entry is 100 percent factually correct, and those facts have been carefully chosen, it all too often reads as if it has ...   \n",
       "498                                                      \"[191] A study of Wikipedia articles on cancer was conducted in 2010 by Yaacov Lawrence of the Kimmel Cancer Center at Thomas Jefferson University.   \n",
       "499                                           The study was limited to those articles that could be found in the Physician Data Query and excluded those written at the \"start\" class or \"stub\" class level.   \n",
       "\n",
       "    Language  \\\n",
       "0    English   \n",
       "1    English   \n",
       "2    English   \n",
       "3    English   \n",
       "4    English   \n",
       "..       ...   \n",
       "495  English   \n",
       "496  English   \n",
       "497  English   \n",
       "498  English   \n",
       "499  English   \n",
       "\n",
       "                                                                                                                                                                                                 text_nopunc  \n",
       "0                                                                                                                            nature in the broadest sense is the natural physical material world or universe  \n",
       "1                                                                                                                        nature can refer to the phenomena of the physical world and also to life in general  \n",
       "2                                                                                                                                             the study of nature is a large if not the only part of science  \n",
       "3                                                                                  although humans are part of nature human activity is often understood as a separate category from other natural phenomena  \n",
       "4                    1 the word nature is borrowed from the old french nature and is derived from the latin word natura or essential qualities innate disposition and in ancient times literally meant birth  \n",
       "..                                                                                                                                                                                                       ...  \n",
       "495  while generally praising the article on william clarke quantrill he quoted its conclusion as an example of such waffling which then stated some historians  remember him as an opportunistic bloodth...  \n",
       "496                                              190 other critics have made similar charges that even if wikipedia articles are factually accurate they are often written in a poor almost unreadable style  \n",
       "497  frequent wikipedia critic andrew orlowski commented even when a wikipedia entry is 100 percent factually correct and those facts have been carefully chosen it all too often reads as if it has been...  \n",
       "498                                                          191 a study of wikipedia articles on cancer was conducted in 2010 by yaacov lawrence of the kimmel cancer center at thomas jefferson university  \n",
       "499                                                the study was limited to those articles that could be found in the physician data query and excluded those written at the start class or stub class level  \n",
       "\n",
       "[500 rows x 3 columns]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    " #removing punctuation with tokenization\n",
    "def remove_punct(tokens):\n",
    "    filtered_sentence_stopwords =\"\"\n",
    "    PUNCT = punctuation\n",
    "    PUNCT+='\"\"'\n",
    "    PUNCT+='``'\n",
    "    PUNCT+=\"''\"+\"«\"+\"»\"\n",
    "    for w in tokens:\n",
    "        if w not in PUNCT:\n",
    "            filtered_sentence_stopwords+=w\n",
    "    return filtered_sentence_stopwords\n",
    "df['text_nopunc']=df['Text'].apply(lambda x:remove_punct(x.lower()))\n",
    "df.head(500)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "id": "75ea478e",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Language</th>\n",
       "      <th>text_nopunc</th>\n",
       "      <th>text_tokniz</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Nature, in the broadest sense, is the natural, physical, material world or universe.</td>\n",
       "      <td>English</td>\n",
       "      <td>nature in the broadest sense is the natural physical material world or universe</td>\n",
       "      <td>[nature, in, the, broadest, sense, is, the, natural, physical, material, world, or, universe]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"Nature\" can refer to the phenomena of the physical world, and also to life in general.</td>\n",
       "      <td>English</td>\n",
       "      <td>nature can refer to the phenomena of the physical world and also to life in general</td>\n",
       "      <td>[nature, can, refer, to, the, phenomena, of, the, physical, world, and, also, to, life, in, general]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The study of nature is a large, if not the only, part of science.</td>\n",
       "      <td>English</td>\n",
       "      <td>the study of nature is a large if not the only part of science</td>\n",
       "      <td>[the, study, of, nature, is, a, large, if, not, the, only, part, of, science]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Although humans are part of nature, human activity is often understood as a separate category from other natural phenomena.</td>\n",
       "      <td>English</td>\n",
       "      <td>although humans are part of nature human activity is often understood as a separate category from other natural phenomena</td>\n",
       "      <td>[although, humans, are, part, of, nature, human, activity, is, often, understood, as, a, separate, category, from, other, natural, phenomena]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[1] The word nature is borrowed from the Old French nature and is derived from the Latin word natura, or \"essential qualities, innate disposition\", and in ancient times, literally meant \"birth\".</td>\n",
       "      <td>English</td>\n",
       "      <td>1 the word nature is borrowed from the old french nature and is derived from the latin word natura or essential qualities innate disposition and in ancient times literally meant birth</td>\n",
       "      <td>[1, the, word, nature, is, borrowed, from, the, old, french, nature, and, is, derived, from, the, latin, word, natura, or, essential, qualities, innate, disposition, and, in, ancient, times, liter...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4995</th>\n",
       "      <td>Para administrar, por ejemplo, se realizan estadísticas, para organizar las visitas, artículos, entre otros; pudiendo ser poco usual para censar comunidades.</td>\n",
       "      <td>Spanish</td>\n",
       "      <td>para administrar por ejemplo se realizan estadísticas para organizar las visitas artículos entre otros pudiendo ser poco usual para censar comunidades</td>\n",
       "      <td>[para, administrar, por, ejemplo, se, realizan, estadísticas, para, organizar, las, visitas, artículos, entre, otros, pudiendo, ser, poco, usual, para, censar, comunidades]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4996</th>\n",
       "      <td>En cuanto a la atribución, Creative Commons, por su parte, no garantiza el contenido por los derechos de imagen que hayan cedido.</td>\n",
       "      <td>Spanish</td>\n",
       "      <td>en cuanto a la atribución creative commons por su parte no garantiza el contenido por los derechos de imagen que hayan cedido</td>\n",
       "      <td>[en, cuanto, a, la, atribución, creative, commons, por, su, parte, no, garantiza, el, contenido, por, los, derechos, de, imagen, que, hayan, cedido]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4997</th>\n",
       "      <td>[129]​ En la Wikipedia en alemán los artículos no son publicados hasta haber sido revisados por un editor experto.</td>\n",
       "      <td>Spanish</td>\n",
       "      <td>129​ en la wikipedia en alemán los artículos no son publicados hasta haber sido revisados por un editor experto</td>\n",
       "      <td>[129​, en, la, wikipedia, en, alemán, los, artículos, no, son, publicados, hasta, haber, sido, revisados, por, un, editor, experto]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4998</th>\n",
       "      <td>Se está barajando la posibilidad de aplicar la misma medida en la Wikipedia en inglés.</td>\n",
       "      <td>Spanish</td>\n",
       "      <td>se está barajando la posibilidad de aplicar la misma medida en la wikipedia en inglés</td>\n",
       "      <td>[se, está, barajando, la, posibilidad, de, aplicar, la, misma, medida, en, la, wikipedia, en, inglés]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4999</th>\n",
       "      <td>[130]​ Wikipedia está siendo editada por miles de personas en todo el mundo.</td>\n",
       "      <td>Spanish</td>\n",
       "      <td>130​ wikipedia está siendo editada por miles de personas en todo el mundo</td>\n",
       "      <td>[130​, wikipedia, está, siendo, editada, por, miles, de, personas, en, todo, el, mundo]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5000 rows × 4 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                    Text  \\\n",
       "0                                                                                                                   Nature, in the broadest sense, is the natural, physical, material world or universe.   \n",
       "1                                                                                                                \"Nature\" can refer to the phenomena of the physical world, and also to life in general.   \n",
       "2                                                                                                                                      The study of nature is a large, if not the only, part of science.   \n",
       "3                                                                            Although humans are part of nature, human activity is often understood as a separate category from other natural phenomena.   \n",
       "4     [1] The word nature is borrowed from the Old French nature and is derived from the Latin word natura, or \"essential qualities, innate disposition\", and in ancient times, literally meant \"birth\".   \n",
       "...                                                                                                                                                                                                  ...   \n",
       "4995                                       Para administrar, por ejemplo, se realizan estadísticas, para organizar las visitas, artículos, entre otros; pudiendo ser poco usual para censar comunidades.   \n",
       "4996                                                                   En cuanto a la atribución, Creative Commons, por su parte, no garantiza el contenido por los derechos de imagen que hayan cedido.   \n",
       "4997                                                                                  [129]​ En la Wikipedia en alemán los artículos no son publicados hasta haber sido revisados por un editor experto.   \n",
       "4998                                                                                                              Se está barajando la posibilidad de aplicar la misma medida en la Wikipedia en inglés.   \n",
       "4999                                                                                                                        [130]​ Wikipedia está siendo editada por miles de personas en todo el mundo.   \n",
       "\n",
       "     Language  \\\n",
       "0     English   \n",
       "1     English   \n",
       "2     English   \n",
       "3     English   \n",
       "4     English   \n",
       "...       ...   \n",
       "4995  Spanish   \n",
       "4996  Spanish   \n",
       "4997  Spanish   \n",
       "4998  Spanish   \n",
       "4999  Spanish   \n",
       "\n",
       "                                                                                                                                                                                  text_nopunc  \\\n",
       "0                                                                                                             nature in the broadest sense is the natural physical material world or universe   \n",
       "1                                                                                                         nature can refer to the phenomena of the physical world and also to life in general   \n",
       "2                                                                                                                              the study of nature is a large if not the only part of science   \n",
       "3                                                                   although humans are part of nature human activity is often understood as a separate category from other natural phenomena   \n",
       "4     1 the word nature is borrowed from the old french nature and is derived from the latin word natura or essential qualities innate disposition and in ancient times literally meant birth   \n",
       "...                                                                                                                                                                                       ...   \n",
       "4995                                   para administrar por ejemplo se realizan estadísticas para organizar las visitas artículos entre otros pudiendo ser poco usual para censar comunidades   \n",
       "4996                                                            en cuanto a la atribución creative commons por su parte no garantiza el contenido por los derechos de imagen que hayan cedido   \n",
       "4997                                                                          129​ en la wikipedia en alemán los artículos no son publicados hasta haber sido revisados por un editor experto   \n",
       "4998                                                                                                    se está barajando la posibilidad de aplicar la misma medida en la wikipedia en inglés   \n",
       "4999                                                                                                                130​ wikipedia está siendo editada por miles de personas en todo el mundo   \n",
       "\n",
       "                                                                                                                                                                                                  text_tokniz  \n",
       "0                                                                                                               [nature, in, the, broadest, sense, is, the, natural, physical, material, world, or, universe]  \n",
       "1                                                                                                        [nature, can, refer, to, the, phenomena, of, the, physical, world, and, also, to, life, in, general]  \n",
       "2                                                                                                                               [the, study, of, nature, is, a, large, if, not, the, only, part, of, science]  \n",
       "3                                                               [although, humans, are, part, of, nature, human, activity, is, often, understood, as, a, separate, category, from, other, natural, phenomena]  \n",
       "4     [1, the, word, nature, is, borrowed, from, the, old, french, nature, and, is, derived, from, the, latin, word, natura, or, essential, qualities, innate, disposition, and, in, ancient, times, liter...  \n",
       "...                                                                                                                                                                                                       ...  \n",
       "4995                             [para, administrar, por, ejemplo, se, realizan, estadísticas, para, organizar, las, visitas, artículos, entre, otros, pudiendo, ser, poco, usual, para, censar, comunidades]  \n",
       "4996                                                     [en, cuanto, a, la, atribución, creative, commons, por, su, parte, no, garantiza, el, contenido, por, los, derechos, de, imagen, que, hayan, cedido]  \n",
       "4997                                                                      [129​, en, la, wikipedia, en, alemán, los, artículos, no, son, publicados, hasta, haber, sido, revisados, por, un, editor, experto]  \n",
       "4998                                                                                                    [se, está, barajando, la, posibilidad, de, aplicar, la, misma, medida, en, la, wikipedia, en, inglés]  \n",
       "4999                                                                                                                  [130​, wikipedia, está, siendo, editada, por, miles, de, personas, en, todo, el, mundo]  \n",
       "\n",
       "[5000 rows x 4 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "def tokenize(text):\n",
    "    tokens=word_tokenize(text)\n",
    "    return tokens\n",
    "df['text_tokniz']=df['text_nopunc'].apply(lambda x : tokenize (x))\n",
    "df.head(5000)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "2e19d45d",
   "metadata": {},
   "outputs": [
    {
     "ename": "OSError",
     "evalue": "No such file or directory: 'C:\\\\Users\\\\Mona\\\\AppData\\\\Roaming\\\\nltk_data\\\\corpora\\\\stopwords\\\\tamil'",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mOSError\u001b[0m                                   Traceback (most recent call last)",
      "Input \u001b[1;32mIn [18]\u001b[0m, in \u001b[0;36m<cell line: 14>\u001b[1;34m()\u001b[0m\n\u001b[0;32m     12\u001b[0m stop_words_tu \u001b[38;5;241m=\u001b[39m stopwords\u001b[38;5;241m.\u001b[39mwords(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mTurkish\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[0;32m     13\u001b[0m stop_words_gr \u001b[38;5;241m=\u001b[39m stopwords\u001b[38;5;241m.\u001b[39mwords(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mGreek\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m---> 14\u001b[0m stop_words_gr \u001b[38;5;241m=\u001b[39m \u001b[43mstopwords\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mwords\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mtamil\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m\n\u001b[0;32m     15\u001b[0m stop_words_gr \u001b[38;5;241m=\u001b[39m stopwords\u001b[38;5;241m.\u001b[39mwords(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhindi\u001b[39m\u001b[38;5;124m'\u001b[39m)\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\nltk\\corpus\\reader\\wordlist.py:21\u001b[0m, in \u001b[0;36mWordListCorpusReader.words\u001b[1;34m(self, fileids, ignore_lines_startswith)\u001b[0m\n\u001b[0;32m     18\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mwords\u001b[39m(\u001b[38;5;28mself\u001b[39m, fileids\u001b[38;5;241m=\u001b[39m\u001b[38;5;28;01mNone\u001b[39;00m, ignore_lines_startswith\u001b[38;5;241m=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;130;01m\\n\u001b[39;00m\u001b[38;5;124m\"\u001b[39m):\n\u001b[0;32m     19\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m [\n\u001b[0;32m     20\u001b[0m         line\n\u001b[1;32m---> 21\u001b[0m         \u001b[38;5;28;01mfor\u001b[39;00m line \u001b[38;5;129;01min\u001b[39;00m line_tokenize(\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mraw\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfileids\u001b[49m\u001b[43m)\u001b[49m)\n\u001b[0;32m     22\u001b[0m         \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m line\u001b[38;5;241m.\u001b[39mstartswith(ignore_lines_startswith)\n\u001b[0;32m     23\u001b[0m     ]\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\nltk\\corpus\\reader\\api.py:218\u001b[0m, in \u001b[0;36mCorpusReader.raw\u001b[1;34m(self, fileids)\u001b[0m\n\u001b[0;32m    216\u001b[0m contents \u001b[38;5;241m=\u001b[39m []\n\u001b[0;32m    217\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m f \u001b[38;5;129;01min\u001b[39;00m fileids:\n\u001b[1;32m--> 218\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mopen\u001b[49m\u001b[43m(\u001b[49m\u001b[43mf\u001b[49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mas\u001b[39;00m fp:\n\u001b[0;32m    219\u001b[0m         contents\u001b[38;5;241m.\u001b[39mappend(fp\u001b[38;5;241m.\u001b[39mread())\n\u001b[0;32m    220\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m concat(contents)\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\nltk\\corpus\\reader\\api.py:231\u001b[0m, in \u001b[0;36mCorpusReader.open\u001b[1;34m(self, file)\u001b[0m\n\u001b[0;32m    223\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    224\u001b[0m \u001b[38;5;124;03mReturn an open stream that can be used to read the given file.\u001b[39;00m\n\u001b[0;32m    225\u001b[0m \u001b[38;5;124;03mIf the file's encoding is not None, then the stream will\u001b[39;00m\n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    228\u001b[0m \u001b[38;5;124;03m:param file: The file identifier of the file to read.\u001b[39;00m\n\u001b[0;32m    229\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[0;32m    230\u001b[0m encoding \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mencoding(file)\n\u001b[1;32m--> 231\u001b[0m stream \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_root\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mjoin\u001b[49m\u001b[43m(\u001b[49m\u001b[43mfile\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241m.\u001b[39mopen(encoding)\n\u001b[0;32m    232\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m stream\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\nltk\\data.py:334\u001b[0m, in \u001b[0;36mFileSystemPathPointer.join\u001b[1;34m(self, fileid)\u001b[0m\n\u001b[0;32m    332\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mjoin\u001b[39m(\u001b[38;5;28mself\u001b[39m, fileid):\n\u001b[0;32m    333\u001b[0m     _path \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mjoin(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_path, fileid)\n\u001b[1;32m--> 334\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mFileSystemPathPointer\u001b[49m\u001b[43m(\u001b[49m\u001b[43m_path\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\nltk\\compat.py:41\u001b[0m, in \u001b[0;36mpy3_data.<locals>._decorator\u001b[1;34m(*args, **kwargs)\u001b[0m\n\u001b[0;32m     39\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_decorator\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m     40\u001b[0m     args \u001b[38;5;241m=\u001b[39m (args[\u001b[38;5;241m0\u001b[39m], add_py3_data(args[\u001b[38;5;241m1\u001b[39m])) \u001b[38;5;241m+\u001b[39m args[\u001b[38;5;241m2\u001b[39m:]\n\u001b[1;32m---> 41\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m init_func(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
      "File \u001b[1;32m~\\anaconda3\\lib\\site-packages\\nltk\\data.py:312\u001b[0m, in \u001b[0;36mFileSystemPathPointer.__init__\u001b[1;34m(self, _path)\u001b[0m\n\u001b[0;32m    310\u001b[0m _path \u001b[38;5;241m=\u001b[39m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mabspath(_path)\n\u001b[0;32m    311\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m os\u001b[38;5;241m.\u001b[39mpath\u001b[38;5;241m.\u001b[39mexists(_path):\n\u001b[1;32m--> 312\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mOSError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNo such file or directory: \u001b[39m\u001b[38;5;132;01m%r\u001b[39;00m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m%\u001b[39m _path)\n\u001b[0;32m    313\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_path \u001b[38;5;241m=\u001b[39m _path\n",
      "\u001b[1;31mOSError\u001b[0m: No such file or directory: 'C:\\\\Users\\\\Mona\\\\AppData\\\\Roaming\\\\nltk_data\\\\corpora\\\\stopwords\\\\tamil'"
     ]
    }
   ],
   "source": [
    "stop_words_en = nltk.corpus.stopwords.words('english')\n",
    "stop_words_ge = stopwords.words('german')\n",
    "stop_words_fr = stopwords.words('french')\n",
    "stop_words_it = stopwords.words('italian')\n",
    "stop_words_po = stopwords.words('Portuguese')\n",
    "stop_words_ru = stopwords.words('Russian')\n",
    "stop_words_sp = stopwords.words('Spanish')\n",
    "stop_words_sw = stopwords.words('Swedish')\n",
    "stop_words_da = stopwords.words('Danish')\n",
    "stop_words_du = stopwords.words('Dutch')\n",
    "stop_words_ar = stopwords.words('Arabic')\n",
    "stop_words_tu = stopwords.words('Turkish')\n",
    "stop_words_gr = stopwords.words('Greek')\n",
    "stop_words_gr = stopwords.words('tamil')\n",
    "stop_words_gr = stopwords.words('hindi')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "32225a02",
   "metadata": {},
   "outputs": [],
   "source": [
    "def remove_stopwords(tok_l,stopwords):\n",
    "    filtered_sentence =[]\n",
    "    for w in tok_l:\n",
    "        if w not in stopwords:\n",
    "            filtered_sentence.append(w)\n",
    "    return filtered_sentence"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "26f3989c",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['filtered_sentence_stopwords']=df['text_tokniz'].apply(lambda x : remove_stopwords(x,stop_words_en))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "487f2186",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['filtered_sentence_stopwords']=df['filtered_sentence_stopwords'].apply(lambda x : remove_stopwords(x,stop_words_ge))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "id": "dd62176d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['filtered_sentence_stopwords']=df['filtered_sentence_stopwords'].apply(lambda x : remove_stopwords(x,stop_words_fr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "id": "b65d67bf",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['filtered_sentence_stopwords']=df['filtered_sentence_stopwords'].apply(lambda x : remove_stopwords(x,stop_words_ru))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "id": "500b99fb",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['filtered_sentence_stopwords']=df['filtered_sentence_stopwords'].apply(lambda x : remove_stopwords(x,stop_words_sp))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "id": "fbcaf9b7",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['filtered_sentence_stopwords']=df['filtered_sentence_stopwords'].apply(lambda x : remove_stopwords(x,stop_words_sw))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "id": "ac35f50d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['filtered_sentence_stopwords']=df['filtered_sentence_stopwords'].apply(lambda x : remove_stopwords(x,stop_words_da))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "id": "95405aec",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['filtered_sentence_stopwords']=df['filtered_sentence_stopwords'].apply(lambda x : remove_stopwords(x,stop_words_du))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "id": "71f0273b",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['filtered_sentence_stopwords']=df['filtered_sentence_stopwords'].apply(lambda x : remove_stopwords(x,stop_words_ar))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "5896028a",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['filtered_sentence_stopwords']=df['filtered_sentence_stopwords'].apply(lambda x : remove_stopwords(x,stop_words_tu))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "id": "94f83cfe",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['filtered_sentence_stopwords']=df['filtered_sentence_stopwords'].apply(lambda x : remove_stopwords(x,stop_words_gr))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "id": "77d885fd",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['filtered_sentence_stopwords']=df['filtered_sentence_stopwords'].apply(lambda x : remove_stopwords(x,stop_words_it))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "id": "cfa6fcd1",
   "metadata": {},
   "outputs": [],
   "source": [
    "df['filtered_sentence_stopwords']=df['filtered_sentence_stopwords'].apply(lambda x : remove_stopwords(x,stop_words_po))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "id": "a30f7e52",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>Text</th>\n",
       "      <th>Language</th>\n",
       "      <th>text_nopunc</th>\n",
       "      <th>text_tokniz</th>\n",
       "      <th>filtered_sentence_stopwords</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>Nature, in the broadest sense, is the natural, physical, material world or universe.</td>\n",
       "      <td>English</td>\n",
       "      <td>nature in the broadest sense is the natural physical material world or universe</td>\n",
       "      <td>[nature, in, the, broadest, sense, is, the, natural, physical, material, world, or, universe]</td>\n",
       "      <td>[nature, broadest, sense, natural, physical, material, world, universe]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>\"Nature\" can refer to the phenomena of the physical world, and also to life in general.</td>\n",
       "      <td>English</td>\n",
       "      <td>nature can refer to the phenomena of the physical world and also to life in general</td>\n",
       "      <td>[nature, can, refer, to, the, phenomena, of, the, physical, world, and, also, to, life, in, general]</td>\n",
       "      <td>[nature, refer, phenomena, physical, world, life, general]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>The study of nature is a large, if not the only, part of science.</td>\n",
       "      <td>English</td>\n",
       "      <td>the study of nature is a large if not the only part of science</td>\n",
       "      <td>[the, study, of, nature, is, a, large, if, not, the, only, part, of, science]</td>\n",
       "      <td>[study, nature, large, part, science]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>Although humans are part of nature, human activity is often understood as a separate category from other natural phenomena.</td>\n",
       "      <td>English</td>\n",
       "      <td>although humans are part of nature human activity is often understood as a separate category from other natural phenomena</td>\n",
       "      <td>[although, humans, are, part, of, nature, human, activity, is, often, understood, as, a, separate, category, from, other, natural, phenomena]</td>\n",
       "      <td>[although, humans, part, nature, human, activity, often, understood, separate, category, natural, phenomena]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>[1] The word nature is borrowed from the Old French nature and is derived from the Latin word natura, or \"essential qualities, innate disposition\", and in ancient times, literally meant \"birth\".</td>\n",
       "      <td>English</td>\n",
       "      <td>1 the word nature is borrowed from the old french nature and is derived from the latin word natura or essential qualities innate disposition and in ancient times literally meant birth</td>\n",
       "      <td>[1, the, word, nature, is, borrowed, from, the, old, french, nature, and, is, derived, from, the, latin, word, natura, or, essential, qualities, innate, disposition, and, in, ancient, times, liter...</td>\n",
       "      <td>[1, word, nature, borrowed, old, french, nature, derived, latin, word, natura, essential, qualities, innate, disposition, ancient, times, literally, meant, birth]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>...</th>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "      <td>...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3275</th>\n",
       "      <td>La première grande philosophie de la nature se trouve dans l'œuvre d'Aristote.</td>\n",
       "      <td>French</td>\n",
       "      <td>la première grande philosophie de la nature se trouve dans lœuvre daristote</td>\n",
       "      <td>[la, première, grande, philosophie, de, la, nature, se, trouve, dans, lœuvre, daristote]</td>\n",
       "      <td>[première, grande, philosophie, nature, trouve, lœuvre, daristote]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3276</th>\n",
       "      <td>Selon lui, « la nature ne fait rien en  vain », ce qui signifie que la nature est une puissance orientée vers certaines  fins.</td>\n",
       "      <td>French</td>\n",
       "      <td>selon lui  la nature ne fait rien en  vain  ce qui signifie que la nature est une puissance orientée vers certaines  fins</td>\n",
       "      <td>[selon, lui, la, nature, ne, fait, rien, en, vain, ce, qui, signifie, que, la, nature, est, une, puissance, orientée, vers, certaines, fins]</td>\n",
       "      <td>[selon, nature, fait, rien, vain, signifie, nature, puissance, orientée, vers, certaines, fins]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3277</th>\n",
       "      <td>L'explication  scientifique consistera donc à découvrir cette finalité.</td>\n",
       "      <td>French</td>\n",
       "      <td>lexplication  scientifique consistera donc à découvrir cette finalité</td>\n",
       "      <td>[lexplication, scientifique, consistera, donc, à, découvrir, cette, finalité]</td>\n",
       "      <td>[lexplication, scientifique, consistera, donc, découvrir, cette, finalité]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3278</th>\n",
       "      <td>Par exemple, si l'eau monte dans un tuyau dont on aspire l'air, c'est parce que \"la nature a horreur du vide\".</td>\n",
       "      <td>French</td>\n",
       "      <td>par exemple si leau monte dans un tuyau dont on aspire lair cest parce que la nature a horreur du vide</td>\n",
       "      <td>[par, exemple, si, leau, monte, dans, un, tuyau, dont, on, aspire, lair, cest, parce, que, la, nature, a, horreur, du, vide]</td>\n",
       "      <td>[exemple, leau, monte, tuyau, dont, aspire, lair, cest, parce, nature, horreur, vide]</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3279</th>\n",
       "      <td>Selon Jan Patocka, « On peut demander si cette philosophie ne nous offre pas un cas typique d'anthropomorphisme »[8].</td>\n",
       "      <td>French</td>\n",
       "      <td>selon jan patocka  on peut demander si cette philosophie ne nous offre pas un cas typique danthropomorphisme 8</td>\n",
       "      <td>[selon, jan, patocka, on, peut, demander, si, cette, philosophie, ne, nous, offre, pas, un, cas, typique, danthropomorphisme, 8]</td>\n",
       "      <td>[selon, jan, patocka, peut, demander, cette, philosophie, offre, cas, typique, danthropomorphisme, 8]</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>3280 rows × 5 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                                                                                                                                                                    Text  \\\n",
       "0                                                                                                                   Nature, in the broadest sense, is the natural, physical, material world or universe.   \n",
       "1                                                                                                                \"Nature\" can refer to the phenomena of the physical world, and also to life in general.   \n",
       "2                                                                                                                                      The study of nature is a large, if not the only, part of science.   \n",
       "3                                                                            Although humans are part of nature, human activity is often understood as a separate category from other natural phenomena.   \n",
       "4     [1] The word nature is borrowed from the Old French nature and is derived from the Latin word natura, or \"essential qualities, innate disposition\", and in ancient times, literally meant \"birth\".   \n",
       "...                                                                                                                                                                                                  ...   \n",
       "3275                                                                                                                      La première grande philosophie de la nature se trouve dans l'œuvre d'Aristote.   \n",
       "3276                                                                      Selon lui, « la nature ne fait rien en  vain », ce qui signifie que la nature est une puissance orientée vers certaines  fins.   \n",
       "3277                                                                                                                             L'explication  scientifique consistera donc à découvrir cette finalité.   \n",
       "3278                                                                                      Par exemple, si l'eau monte dans un tuyau dont on aspire l'air, c'est parce que \"la nature a horreur du vide\".   \n",
       "3279                                                                               Selon Jan Patocka, « On peut demander si cette philosophie ne nous offre pas un cas typique d'anthropomorphisme »[8].   \n",
       "\n",
       "     Language  \\\n",
       "0     English   \n",
       "1     English   \n",
       "2     English   \n",
       "3     English   \n",
       "4     English   \n",
       "...       ...   \n",
       "3275   French   \n",
       "3276   French   \n",
       "3277   French   \n",
       "3278   French   \n",
       "3279   French   \n",
       "\n",
       "                                                                                                                                                                                  text_nopunc  \\\n",
       "0                                                                                                             nature in the broadest sense is the natural physical material world or universe   \n",
       "1                                                                                                         nature can refer to the phenomena of the physical world and also to life in general   \n",
       "2                                                                                                                              the study of nature is a large if not the only part of science   \n",
       "3                                                                   although humans are part of nature human activity is often understood as a separate category from other natural phenomena   \n",
       "4     1 the word nature is borrowed from the old french nature and is derived from the latin word natura or essential qualities innate disposition and in ancient times literally meant birth   \n",
       "...                                                                                                                                                                                       ...   \n",
       "3275                                                                                                              la première grande philosophie de la nature se trouve dans lœuvre daristote   \n",
       "3276                                                                selon lui  la nature ne fait rien en  vain  ce qui signifie que la nature est une puissance orientée vers certaines  fins   \n",
       "3277                                                                                                                    lexplication  scientifique consistera donc à découvrir cette finalité   \n",
       "3278                                                                                   par exemple si leau monte dans un tuyau dont on aspire lair cest parce que la nature a horreur du vide   \n",
       "3279                                                                           selon jan patocka  on peut demander si cette philosophie ne nous offre pas un cas typique danthropomorphisme 8   \n",
       "\n",
       "                                                                                                                                                                                                  text_tokniz  \\\n",
       "0                                                                                                               [nature, in, the, broadest, sense, is, the, natural, physical, material, world, or, universe]   \n",
       "1                                                                                                        [nature, can, refer, to, the, phenomena, of, the, physical, world, and, also, to, life, in, general]   \n",
       "2                                                                                                                               [the, study, of, nature, is, a, large, if, not, the, only, part, of, science]   \n",
       "3                                                               [although, humans, are, part, of, nature, human, activity, is, often, understood, as, a, separate, category, from, other, natural, phenomena]   \n",
       "4     [1, the, word, nature, is, borrowed, from, the, old, french, nature, and, is, derived, from, the, latin, word, natura, or, essential, qualities, innate, disposition, and, in, ancient, times, liter...   \n",
       "...                                                                                                                                                                                                       ...   \n",
       "3275                                                                                                                 [la, première, grande, philosophie, de, la, nature, se, trouve, dans, lœuvre, daristote]   \n",
       "3276                                                             [selon, lui, la, nature, ne, fait, rien, en, vain, ce, qui, signifie, que, la, nature, est, une, puissance, orientée, vers, certaines, fins]   \n",
       "3277                                                                                                                            [lexplication, scientifique, consistera, donc, à, découvrir, cette, finalité]   \n",
       "3278                                                                             [par, exemple, si, leau, monte, dans, un, tuyau, dont, on, aspire, lair, cest, parce, que, la, nature, a, horreur, du, vide]   \n",
       "3279                                                                         [selon, jan, patocka, on, peut, demander, si, cette, philosophie, ne, nous, offre, pas, un, cas, typique, danthropomorphisme, 8]   \n",
       "\n",
       "                                                                                                                                             filtered_sentence_stopwords  \n",
       "0                                                                                                [nature, broadest, sense, natural, physical, material, world, universe]  \n",
       "1                                                                                                             [nature, refer, phenomena, physical, world, life, general]  \n",
       "2                                                                                                                                  [study, nature, large, part, science]  \n",
       "3                                                           [although, humans, part, nature, human, activity, often, understood, separate, category, natural, phenomena]  \n",
       "4     [1, word, nature, borrowed, old, french, nature, derived, latin, word, natura, essential, qualities, innate, disposition, ancient, times, literally, meant, birth]  \n",
       "...                                                                                                                                                                  ...  \n",
       "3275                                                                                                  [première, grande, philosophie, nature, trouve, lœuvre, daristote]  \n",
       "3276                                                                     [selon, nature, fait, rien, vain, signifie, nature, puissance, orientée, vers, certaines, fins]  \n",
       "3277                                                                                          [lexplication, scientifique, consistera, donc, découvrir, cette, finalité]  \n",
       "3278                                                                               [exemple, leau, monte, tuyau, dont, aspire, lair, cest, parce, nature, horreur, vide]  \n",
       "3279                                                               [selon, jan, patocka, peut, demander, cette, philosophie, offre, cas, typique, danthropomorphisme, 8]  \n",
       "\n",
       "[3280 rows x 5 columns]"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.head(3280)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ea4b3a91",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}