get_tweets.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import tweepy\n",
    "import json\n",
    "import pandas as pd\n",
    "\n",
    "from IPython.core.interactiveshell import InteractiveShell\n",
    "InteractiveShell.ast_node_interactivity = \"all\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "def login():\n",
    "    # TODO: Insert your keys and tokens here\n",
    "    consumer_key  = ''\n",
    "    consumer_secret  = ''\n",
    "    access_token  = ''\n",
    "    access_token_secret  = ''\n",
    "    \n",
    "    auth = tweepy.OAuthHandler(consumer_key, consumer_secret)\n",
    "    auth.set_access_token(access_token, access_token_secret)\n",
    "    api = tweepy.API(auth, wait_on_rate_limit=True)\n",
    "    \n",
    "    return api\n",
    "\n",
    "def twitter_search(api, text_query, count=100, **kw):\n",
    "    try:\n",
    "        tweets = api.search(q=text_query, count=count, **kw)\n",
    "    except BaseException as e:\n",
    "        print('failed on_status,',str(e))\n",
    "        time.sleep(3)\n",
    "        \n",
    "    tweets_list = []\n",
    "    for tweet in tweets:\n",
    "        tweets_list.append((tweet.id, tweet.created_at,tweet.user.screen_name,tweet.text))\n",
    "        \n",
    "    df = pd.DataFrame(list(tweets_list), columns=['ID', 'Time', \"Screen Name\", \"Text\"])\n",
    "    return df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "api = login()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Search Twitter Once"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Time</th>\n",
       "      <th>Screen Name</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1260191732223946752</td>\n",
       "      <td>2020-05-12 12:54:56</td>\n",
       "      <td>PrincesDihya216</td>\n",
       "      <td>RT @ElonMus46662263: X Æ A-12’s first day of s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1260191716172316672</td>\n",
       "      <td>2020-05-12 12:54:53</td>\n",
       "      <td>mterezac</td>\n",
       "      <td>@raissa_mnss o mundo ta preparado pra: X Æ A-12</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1260191694097612801</td>\n",
       "      <td>2020-05-12 12:54:47</td>\n",
       "      <td>kaylaleanneee</td>\n",
       "      <td>RT @TokyoSage: X AE A-12 after watching one ep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1260191676531912704</td>\n",
       "      <td>2020-05-12 12:54:43</td>\n",
       "      <td>PrincesDihya216</td>\n",
       "      <td>RT @ElonMus46662263: X Æ A-12 when he’s late f...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1260191655753404416</td>\n",
       "      <td>2020-05-12 12:54:38</td>\n",
       "      <td>lewownardoRECAP</td>\n",
       "      <td>RT @Trundle: 📍Nem o X Æ A-12 Localiza</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    ID                Time      Screen Name  \\\n",
       "0  1260191732223946752 2020-05-12 12:54:56  PrincesDihya216   \n",
       "1  1260191716172316672 2020-05-12 12:54:53         mterezac   \n",
       "2  1260191694097612801 2020-05-12 12:54:47    kaylaleanneee   \n",
       "3  1260191676531912704 2020-05-12 12:54:43  PrincesDihya216   \n",
       "4  1260191655753404416 2020-05-12 12:54:38  lewownardoRECAP   \n",
       "\n",
       "                                                Text  \n",
       "0  RT @ElonMus46662263: X Æ A-12’s first day of s...  \n",
       "1    @raissa_mnss o mundo ta preparado pra: X Æ A-12  \n",
       "2  RT @TokyoSage: X AE A-12 after watching one ep...  \n",
       "3  RT @ElonMus46662263: X Æ A-12 when he’s late f...  \n",
       "4              RT @Trundle: 📍Nem o X Æ A-12 Localiza  "
      ]
     },
     "execution_count": 20,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "# Put whatever you want as text query\n",
    "tweets_df = twitter_search(api, text_query=\"X Æ A-12\")\n",
    "tweets_df.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Search on Loop\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "100\n",
      "1\n",
      "2\n"
     ]
    }
   ],
   "source": [
    "import math\n",
    "import time\n",
    "\n",
    "all_df = pd.DataFrame(columns=['ID', 'Time', \"Screen Name\", \"Text\"])\n",
    "\n",
    "# Put whatever you want here\n",
    "text_query = 'X Æ A-12'\n",
    "\n",
    "\n",
    "for _ in range(10): # How many times do you want to do this?\n",
    "    new_tweets = twitter_search(api, text_query, since_id=all_df.ID.max())\n",
    "        \n",
    "    print(len(new_tweets))\n",
    "    if len(new_tweets) > 0:\n",
    "        all_df = all_df.append(new_tweets, ignore_index=True)\n",
    "\n",
    "    time.sleep(900) #Sleep for 15 minutes"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<class 'pandas.core.frame.DataFrame'>\n",
      "RangeIndex: 103 entries, 0 to 102\n",
      "Data columns (total 4 columns):\n",
      " #   Column       Non-Null Count  Dtype         \n",
      "---  ------       --------------  -----         \n",
      " 0   ID           103 non-null    object        \n",
      " 1   Time         103 non-null    datetime64[ns]\n",
      " 2   Screen Name  103 non-null    object        \n",
      " 3   Text         103 non-null    object        \n",
      "dtypes: datetime64[ns](1), object(3)\n",
      "memory usage: 3.3+ KB\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Time</th>\n",
       "      <th>Screen Name</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>1260196265037492225</td>\n",
       "      <td>2020-05-12 13:12:57</td>\n",
       "      <td>hezriqsyaffrin</td>\n",
       "      <td>RT @TokyoSage: X AE A-12 after watching one ep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1260196201556848642</td>\n",
       "      <td>2020-05-12 13:12:42</td>\n",
       "      <td>Astyycow</td>\n",
       "      <td>RT @TokyoSage: X AE A-12 after watching one ep...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1260196130715062274</td>\n",
       "      <td>2020-05-12 13:12:25</td>\n",
       "      <td>lindatshields</td>\n",
       "      <td>RT @colladoivan21: Elon musk and his son X Æ A...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1260196119528697856</td>\n",
       "      <td>2020-05-12 13:12:22</td>\n",
       "      <td>jihahzkria</td>\n",
       "      <td>RT @Grimezsz: •X, the unknown variable ⚔️\\n•Æ,...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1260196109097500675</td>\n",
       "      <td>2020-05-12 13:12:20</td>\n",
       "      <td>chlorowodorowy</td>\n",
       "      <td>RT @miliondollameat: X Æ A-12’s first pet http...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                    ID                Time     Screen Name  \\\n",
       "0  1260196265037492225 2020-05-12 13:12:57  hezriqsyaffrin   \n",
       "1  1260196201556848642 2020-05-12 13:12:42        Astyycow   \n",
       "2  1260196130715062274 2020-05-12 13:12:25   lindatshields   \n",
       "3  1260196119528697856 2020-05-12 13:12:22      jihahzkria   \n",
       "4  1260196109097500675 2020-05-12 13:12:20  chlorowodorowy   \n",
       "\n",
       "                                                Text  \n",
       "0  RT @TokyoSage: X AE A-12 after watching one ep...  \n",
       "1  RT @TokyoSage: X AE A-12 after watching one ep...  \n",
       "2  RT @colladoivan21: Elon musk and his son X Æ A...  \n",
       "3  RT @Grimezsz: •X, the unknown variable ⚔️\\n•Æ,...  \n",
       "4  RT @miliondollameat: X Æ A-12’s first pet http...  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Time</th>\n",
       "      <th>Screen Name</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>1260192514440491011</td>\n",
       "      <td>2020-05-12 12:58:03</td>\n",
       "      <td>iamdisasterxx</td>\n",
       "      <td>RT @tang_guin: white people gonna learn how to...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>1260192503707447301</td>\n",
       "      <td>2020-05-12 12:58:00</td>\n",
       "      <td>Why_PinkDiamond</td>\n",
       "      <td>RT @angiesey02: X Æ A-12 first day of at pre s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>1260196321098706944</td>\n",
       "      <td>2020-05-12 13:13:10</td>\n",
       "      <td>liondro</td>\n",
       "      <td>RT @TheWeirdWorld: We live in a generation whe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>1260196350433648640</td>\n",
       "      <td>2020-05-12 13:13:17</td>\n",
       "      <td>EPIC_ONE_</td>\n",
       "      <td>I was shocked when I hear..\\nElon Musk shared ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>1260196344322568194</td>\n",
       "      <td>2020-05-12 13:13:16</td>\n",
       "      <td>merymonsul</td>\n",
       "      <td>RT @angiesey02: X Æ A-12 first day of at pre s...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      ID                Time      Screen Name  \\\n",
       "98   1260192514440491011 2020-05-12 12:58:03    iamdisasterxx   \n",
       "99   1260192503707447301 2020-05-12 12:58:00  Why_PinkDiamond   \n",
       "100  1260196321098706944 2020-05-12 13:13:10          liondro   \n",
       "101  1260196350433648640 2020-05-12 13:13:17        EPIC_ONE_   \n",
       "102  1260196344322568194 2020-05-12 13:13:16       merymonsul   \n",
       "\n",
       "                                                  Text  \n",
       "98   RT @tang_guin: white people gonna learn how to...  \n",
       "99   RT @angiesey02: X Æ A-12 first day of at pre s...  \n",
       "100  RT @TheWeirdWorld: We live in a generation whe...  \n",
       "101  I was shocked when I hear..\\nElon Musk shared ...  \n",
       "102  RT @angiesey02: X Æ A-12 first day of at pre s...  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/plain": [
       "103"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>ID</th>\n",
       "      <th>Time</th>\n",
       "      <th>Screen Name</th>\n",
       "      <th>Text</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>98</th>\n",
       "      <td>1260192514440491011</td>\n",
       "      <td>2020-05-12 12:58:03</td>\n",
       "      <td>iamdisasterxx</td>\n",
       "      <td>RT @tang_guin: white people gonna learn how to...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>99</th>\n",
       "      <td>1260192503707447301</td>\n",
       "      <td>2020-05-12 12:58:00</td>\n",
       "      <td>Why_PinkDiamond</td>\n",
       "      <td>RT @angiesey02: X Æ A-12 first day of at pre s...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>100</th>\n",
       "      <td>1260196321098706944</td>\n",
       "      <td>2020-05-12 13:13:10</td>\n",
       "      <td>liondro</td>\n",
       "      <td>RT @TheWeirdWorld: We live in a generation whe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>101</th>\n",
       "      <td>1260196350433648640</td>\n",
       "      <td>2020-05-12 13:13:17</td>\n",
       "      <td>EPIC_ONE_</td>\n",
       "      <td>I was shocked when I hear..\\nElon Musk shared ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>102</th>\n",
       "      <td>1260196344322568194</td>\n",
       "      <td>2020-05-12 13:13:16</td>\n",
       "      <td>merymonsul</td>\n",
       "      <td>RT @angiesey02: X Æ A-12 first day of at pre s...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                      ID                Time      Screen Name  \\\n",
       "98   1260192514440491011 2020-05-12 12:58:03    iamdisasterxx   \n",
       "99   1260192503707447301 2020-05-12 12:58:00  Why_PinkDiamond   \n",
       "100  1260196321098706944 2020-05-12 13:13:10          liondro   \n",
       "101  1260196350433648640 2020-05-12 13:13:17        EPIC_ONE_   \n",
       "102  1260196344322568194 2020-05-12 13:13:16       merymonsul   \n",
       "\n",
       "                                                  Text  \n",
       "98   RT @tang_guin: white people gonna learn how to...  \n",
       "99   RT @angiesey02: X Æ A-12 first day of at pre s...  \n",
       "100  RT @TheWeirdWorld: We live in a generation whe...  \n",
       "101  I was shocked when I hear..\\nElon Musk shared ...  \n",
       "102  RT @angiesey02: X Æ A-12 first day of at pre s...  "
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "all_df.info()\n",
    "all_df.head()\n",
    "all_df.ID.nunique()\n",
    "all_df.tail()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "small_sklearn_kernel",
   "language": "python",
   "name": "small_sklearn_kernel"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}