web_scraper.ipynb

{
  "nbformat": 4,
  "nbformat_minor": 0,
  "metadata": {
    "colab": {
      "provenance": [],
      "toc_visible": true,
      "include_colab_link": true
    },
    "kernelspec": {
      "name": "python3",
      "display_name": "Python 3"
    },
    "language_info": {
      "name": "python"
    }
  },
  "cells": [
    {
      "cell_type": "markdown",
      "metadata": {
        "id": "view-in-github",
        "colab_type": "text"
      },
      "source": [
        "<a href=\"https://colab.research.google.com/github/rohitkadu/web-scraper-python/blob/main/web_scraper.ipynb\" target=\"_parent\"><img src=\"https://colab.research.google.com/assets/colab-badge.svg\" alt=\"Open In Colab\"/></a>"
      ]
    },
    {
      "cell_type": "code",
      "execution_count": null,
      "metadata": {
        "id": "5kxI_gbR0qMs"
      },
      "outputs": [],
      "source": [
        "import requests\n",
        "from bs4 import BeautifulSoup"
      ]
    },
    {
      "cell_type": "code",
      "source": [
        "url = input(\"Enter url : \")\n",
        "# url = \"https://www.moneycontrol.com/stocksmarketsindia/\""
      ],
      "metadata": {
        "id": "H2B5FTWG04NH"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "r = requests.get(url)\n",
        "htmlContent = r.content"
      ],
      "metadata": {
        "id": "R4zHJR9Q1XIZ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "soup = BeautifulSoup(htmlContent, 'html.parser')"
      ],
      "metadata": {
        "id": "rVNPsssS2Avd"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get the title\n",
        "title = soup.title\n",
        "print(title.get_text())"
      ],
      "metadata": {
        "id": "qfbtZdTu2HQT"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get all meta tags\n",
        "metaTags = soup.find_all('meta')\n",
        "for mTag in metaTags:\n",
        "  print(mTag)"
      ],
      "metadata": {
        "id": "3WhC0Yj_268C"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get all p tags \n",
        "pTags = soup.find_all('p')\n",
        "for p in pTags:\n",
        "  print(p)"
      ],
      "metadata": {
        "id": "OYC4TaJw3oDw"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get all a tags\n",
        "aTags = soup.find_all('a')\n",
        "for a in aTags:\n",
        "  print(a)"
      ],
      "metadata": {
        "id": "uPFD2G0Q4RIE"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get clickable anchor links\n",
        "# anchors = soup.find_all('a')\n",
        "# for hrefs in anchors:\n",
        "#   print(hrefs.get('href'))\n",
        "\n",
        "from urllib.parse import urljoin\n",
        "\n",
        "anchors = soup.find_all('a')\n",
        "\n",
        "for hrefs in anchors:\n",
        "    href = hrefs.get('href')\n",
        "    if href and href != '#' and not href.startswith(('javascript')):\n",
        "        if not href.startswith('http'):  # Check if the link is relative\n",
        "            href = urljoin(url, href)  # Convert relative link to absolute URL using the base URL\n",
        "        print(href)\n",
        "\n"
      ],
      "metadata": {
        "id": "KQjgBZo87Q20"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "#Get all images using img src\n",
        "from tabulate import tabulate\n",
        "\n",
        "imgTags = soup.find_all('img')\n",
        "\n",
        "if imgTags:\n",
        "    rows = []\n",
        "    for img in imgTags:\n",
        "        src = img.get('src', 'No source URL')\n",
        "        alt = img.get('alt', 'No alt text')\n",
        "        rows.append([src, alt])\n",
        "    \n",
        "    print(tabulate(rows, headers=['Alt Text', 'Source URL'], tablefmt='fancy_grid'))\n",
        "else:\n",
        "    print(\"No <img> tags found\")\n"
      ],
      "metadata": {
        "id": "Cha-Cpze5xug"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get all tables\n",
        "tableTags = soup.find_all('table')\n",
        "print(tableTags)"
      ],
      "metadata": {
        "id": "zmh2YWn14bCY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get all tables in HTML page\n",
        "tables = soup.find_all(\"table\")\n",
        "for table in tables:\n",
        "    # Extract the table title\n",
        "    title = table.find_previous(\"h2\").text if table.find_previous(\"h2\") else \"\"\n",
        "\n",
        "    rows = []\n",
        "    headers = []\n",
        "    for row in table.find_all(\"tr\"):\n",
        "        cells = [cell.get_text(strip=True) for cell in row.find_all([\"td\", \"th\"])]\n",
        "        if not headers:\n",
        "            headers = cells\n",
        "        else:\n",
        "            rows.append(cells)\n",
        "\n",
        "    print(f\"Table Title: {title}\")\n",
        "    if headers:\n",
        "        print(tabulate(rows, headers=headers, tablefmt=\"fancy_grid\"))\n",
        "    else:\n",
        "        print(tabulate(rows, tablefmt=\"fancy_grid\"))\n",
        "    print()  # Print a new line between tables"
      ],
      "metadata": {
        "id": "iVA6BqbJ9dka"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.display import Image, SVG\n",
        "\n",
        "# Find all image tags\n",
        "image_tags = soup.find_all(\"img\")\n",
        "\n",
        "# Extract and display images\n",
        "for image_tag in image_tags:\n",
        "    image_url = image_tag[\"src\"]\n",
        "    if image_url.startswith(\"http\"):\n",
        "        display(Image(url=image_url))\n",
        "    else:\n",
        "        display(Image(url=url + image_url))\n"
      ],
      "metadata": {
        "id": "95-BeeB-_qjQ"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.display import Image, SVG\n",
        "\n",
        "# Find all SVG tags\n",
        "svg_tags = soup.find_all(\"svg\")\n",
        "\n",
        "# Extract and display SVGs\n",
        "for svg_tag in svg_tags:\n",
        "    svg_code = svg_tag.prettify()\n",
        "    display(SVG(svg_code))"
      ],
      "metadata": {
        "id": "AHzForJxBgoq"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.display import Audio, display\n",
        "\n",
        "audio_tags = soup.find_all(\"audio\")\n",
        "\n",
        "# Extract and display audio files\n",
        "for audio_tag in audio_tags:\n",
        "    audio_url = audio_tag.get(\"src\")\n",
        "    if audio_url:\n",
        "        audio_link = f'<a href=\"{audio_url}\">Link</a>'\n",
        "        display(audio_link)\n",
        "        display(Audio(url=audio_url, autoplay=False))"
      ],
      "metadata": {
        "id": "2MqmH-v8Hluy"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "from IPython.display import HTML\n",
        "\n",
        "video_tags = soup.find_all(\"video\")\n",
        "\n",
        "# Extract and display videos\n",
        "for video_tag in video_tags:\n",
        "    video_url = video_tag.get(\"src\")\n",
        "    if video_url:\n",
        "        video_html = f'<video src=\"{video_url}\" controls width=\"400\"></video>'\n",
        "        display(HTML(video_html))"
      ],
      "metadata": {
        "id": "KpGAx_yoF7GY"
      },
      "execution_count": null,
      "outputs": []
    },
    {
      "cell_type": "code",
      "source": [
        "# Get entire source code HTML \n",
        "print(soup.prettify)"
      ],
      "metadata": {
        "id": "hVEHXisVIEiW"
      },
      "execution_count": null,
      "outputs": []
    }
  ]
}