scrap_pixel_22.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import requests\n",
    "import pandas as pd\n",
    "from collections import OrderedDict\n",
    "from bs4 import BeautifulSoup, NavigableString, Tag"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [],
   "source": [
    "baseurl = \"https://pixel.usm.my\"\n",
    "\n",
    "headers = {\n",
    "    'User-Agent':'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.5223.82 Safari/537.36 Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/15.4 Safari/605.1.15'\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "r = requests.get('https://pixel.usm.my/explore?cat=1')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "soup = BeautifulSoup(r.content, 'lxml')\n",
    "#student_url_class_name = \n",
    "#students_list = soup.find_all('div', class_ = \"max-w-sm\")\n",
    "#body = soup.find_all('body')\n",
    "students_list = soup.find_all('div', class_ = \"max-w-sm\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "ename": "RuntimeError",
     "evalue": "Cannot use HTMLSession within an existing event loop. Use AsyncHTMLSession instead.",
     "output_type": "error",
     "traceback": [
      "\u001b[1;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[1;31mRuntimeError\u001b[0m                              Traceback (most recent call last)",
      "\u001b[1;32md:\\web_scraper\\scrap_pixel_22.ipynb Cell 5\u001b[0m in \u001b[0;36m<cell line: 9>\u001b[1;34m()\u001b[0m\n\u001b[0;32m      <a href='vscode-notebook-cell:/d%3A/web_scraper/scrap_pixel_22.ipynb#X16sZmlsZQ%3D%3D?line=5'>6</a>\u001b[0m url \u001b[39m=\u001b[39m \u001b[39m'\u001b[39m\u001b[39mhttps://quizlet.com/class/8536895/\u001b[39m\u001b[39m'\u001b[39m \n\u001b[0;32m      <a href='vscode-notebook-cell:/d%3A/web_scraper/scrap_pixel_22.ipynb#X16sZmlsZQ%3D%3D?line=7'>8</a>\u001b[0m response \u001b[39m=\u001b[39m session\u001b[39m.\u001b[39mget(url)\n\u001b[1;32m----> <a href='vscode-notebook-cell:/d%3A/web_scraper/scrap_pixel_22.ipynb#X16sZmlsZQ%3D%3D?line=8'>9</a>\u001b[0m response\u001b[39m.\u001b[39;49mhtml\u001b[39m.\u001b[39;49mrender()  \u001b[39m# render the webpage\u001b[39;00m\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/web_scraper/scrap_pixel_22.ipynb#X16sZmlsZQ%3D%3D?line=10'>11</a>\u001b[0m \u001b[39m# access html page source with html.html\u001b[39;00m\n\u001b[0;32m     <a href='vscode-notebook-cell:/d%3A/web_scraper/scrap_pixel_22.ipynb#X16sZmlsZQ%3D%3D?line=11'>12</a>\u001b[0m soup \u001b[39m=\u001b[39m BeautifulSoup(response\u001b[39m.\u001b[39mhtml\u001b[39m.\u001b[39mhtml, \u001b[39m'\u001b[39m\u001b[39mhtml.parser\u001b[39m\u001b[39m'\u001b[39m) \n",
      "File \u001b[1;32mc:\\Users\\sin-yee.teoh\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\requests_html.py:586\u001b[0m, in \u001b[0;36mHTML.render\u001b[1;34m(self, retries, script, wait, scrolldown, sleep, reload, timeout, keep_page)\u001b[0m\n\u001b[0;32m    541\u001b[0m \u001b[39mdef\u001b[39;00m \u001b[39mrender\u001b[39m(\u001b[39mself\u001b[39m, retries: \u001b[39mint\u001b[39m \u001b[39m=\u001b[39m \u001b[39m8\u001b[39m, script: \u001b[39mstr\u001b[39m \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m, wait: \u001b[39mfloat\u001b[39m \u001b[39m=\u001b[39m \u001b[39m0.2\u001b[39m, scrolldown\u001b[39m=\u001b[39m\u001b[39mFalse\u001b[39;00m, sleep: \u001b[39mint\u001b[39m \u001b[39m=\u001b[39m \u001b[39m0\u001b[39m, reload: \u001b[39mbool\u001b[39m \u001b[39m=\u001b[39m \u001b[39mTrue\u001b[39;00m, timeout: Union[\u001b[39mfloat\u001b[39m, \u001b[39mint\u001b[39m] \u001b[39m=\u001b[39m \u001b[39m8.0\u001b[39m, keep_page: \u001b[39mbool\u001b[39m \u001b[39m=\u001b[39m \u001b[39mFalse\u001b[39;00m):\n\u001b[0;32m    542\u001b[0m     \u001b[39m\"\"\"Reloads the response in Chromium, and replaces HTML content\u001b[39;00m\n\u001b[0;32m    543\u001b[0m \u001b[39m    with an updated version, with JavaScript executed.\u001b[39;00m\n\u001b[0;32m    544\u001b[0m \n\u001b[1;32m   (...)\u001b[0m\n\u001b[0;32m    583\u001b[0m \u001b[39m    Chromium into your home directory (``~/.pyppeteer``).\u001b[39;00m\n\u001b[0;32m    584\u001b[0m \u001b[39m    \"\"\"\u001b[39;00m\n\u001b[1;32m--> 586\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mbrowser \u001b[39m=\u001b[39m \u001b[39mself\u001b[39;49m\u001b[39m.\u001b[39;49msession\u001b[39m.\u001b[39;49mbrowser  \u001b[39m# Automatically create a event loop and browser\u001b[39;00m\n\u001b[0;32m    587\u001b[0m     content \u001b[39m=\u001b[39m \u001b[39mNone\u001b[39;00m\n\u001b[0;32m    589\u001b[0m     \u001b[39m# Automatically set Reload to False, if example URL is being used.\u001b[39;00m\n",
      "File \u001b[1;32mc:\\Users\\sin-yee.teoh\\AppData\\Local\\Programs\\Python\\Python310\\lib\\site-packages\\requests_html.py:729\u001b[0m, in \u001b[0;36mHTMLSession.browser\u001b[1;34m(self)\u001b[0m\n\u001b[0;32m    727\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mloop \u001b[39m=\u001b[39m asyncio\u001b[39m.\u001b[39mget_event_loop()\n\u001b[0;32m    728\u001b[0m     \u001b[39mif\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mloop\u001b[39m.\u001b[39mis_running():\n\u001b[1;32m--> 729\u001b[0m         \u001b[39mraise\u001b[39;00m \u001b[39mRuntimeError\u001b[39;00m(\u001b[39m\"\u001b[39m\u001b[39mCannot use HTMLSession within an existing event loop. Use AsyncHTMLSession instead.\u001b[39m\u001b[39m\"\u001b[39m)\n\u001b[0;32m    730\u001b[0m     \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_browser \u001b[39m=\u001b[39m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39mloop\u001b[39m.\u001b[39mrun_until_complete(\u001b[39msuper\u001b[39m()\u001b[39m.\u001b[39mbrowser)\n\u001b[0;32m    731\u001b[0m \u001b[39mreturn\u001b[39;00m \u001b[39mself\u001b[39m\u001b[39m.\u001b[39m_browser\n",
      "\u001b[1;31mRuntimeError\u001b[0m: Cannot use HTMLSession within an existing event loop. Use AsyncHTMLSession instead."
     ]
    }
   ],
   "source": [
    "from requests_html import HTMLSession\n",
    "from bs4 import BeautifulSoup \n",
    "\n",
    "session = HTMLSession()\n",
    "\n",
    "url = 'https://quizlet.com/class/8536895/' \n",
    "\n",
    "response = session.get(url)\n",
    "response.html.render()  # render the webpage\n",
    "\n",
    "# access html page source with html.html\n",
    "soup = BeautifulSoup(response.html.html, 'html.parser') \n",
    "\n",
    "b = soup.find_all(\"a\", class_=\"UILink\")\n",
    "\n",
    "print(len(b))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3.10.3 64-bit",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.3"
  },
  "orig_nbformat": 4,
  "vscode": {
   "interpreter": {
    "hash": "6df761b4066193265fabac3b3f06fc34705548320002f02836cd9baf3fb5a8e0"
   }
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}