export_movies.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "from neo4j import GraphDatabase\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import string\n",
    "\n",
    "import requests\n",
    "from requests.auth import HTTPBasicAuth\n",
    "import time\n",
    "from selenium import webdriver\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "from selenium.webdriver.support import expected_conditions as EC\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.support.ui import WebDriverWait\n",
    "from selenium.webdriver.firefox.options import Options\n",
    "from selenium.common.exceptions import *"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "\n",
    "\n",
    "https://cloud.collectorz.com/wisundstrom/movies/export/all --this gets you to the export page\n",
    "\n",
    "need to write selenium script to hit the button, set firefox default dl location\n",
    "also I probably need the selenium script to make sure the export columns are set up how I like em.\n",
    "https://selenium-python.readthedocs.io/navigating.html\n",
    "\n",
    "https://cloud.collectorz.com/wisundstrom/movies/export/download -- once you click the generate button, this link becomes valid\n",
    "https://sqa.stackexchange.com/questions/2197/how-to-download-a-file-using-seleniums-webdriver --use something like this to do headless firefox downloads\n",
    "\n",
    "NICE! I just realized that clz lets you generate a special link that lets you get at the movies without logging in\n",
    "username: wisundstrom\n",
    "pw: clzpw.text\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "# r = requests.get('https://cloud.collectorz.com/wisundstrom/movies?t=8bccef186b8a05138308f5de07f9879c09f5617ecc163866ba5b8ccb8509de7e')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "# r.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "# from neo4j import GraphDatabase\n",
    "\n",
    "# uri = \"bolt://localhost:7687\"\n",
    "\n",
    "# driver = GraphDatabase.driver(uri, auth=('jupyter', 'jup'))\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {},
   "outputs": [],
   "source": [
    "def click_waiter(xpath):\n",
    "\n",
    "    def clickable(driver):\n",
    "        element = driver.find_element_by_xpath(xpath)\n",
    "        try:\n",
    "            element.click()\n",
    "        except ElementClickInterceptedException:\n",
    "            return False\n",
    "        return element\n",
    "    \n",
    "    return clickable"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 69,
   "metadata": {},
   "outputs": [],
   "source": [
    "profile = webdriver.FirefoxProfile()\n",
    "profile.set_preference(\"browser.download.folderList\", 2)\n",
    "profile.set_preference(\"browser.download.manager.showWhenStarting\", False)\n",
    "profile.set_preference(\"browser.download.dir\", '/home/will/wmdb/wmdb-repo')\n",
    "profile.set_preference(\"browser.download.useDownloadDir\", True)\n",
    "profile.set_preference(\"browser.helperApps.neverAsk.saveToDisk\", \"text/csv\")\n",
    "from selenium.webdriver.firefox.options import Options\n",
    "\n",
    "options = Options()\n",
    "options.headless = True\n",
    "driver = webdriver.Firefox(options = options, firefox_profile=profile)\n",
    "driver.implicitly_wait(10)\n",
    "\n",
    "# browser.download.useDownloadDir \n",
    "driver.get()\n",
    "assert(\"Wisundstrom's movies\") in driver.title\n",
    "\n",
    "driver.find_element_by_id(\"cookiescript_accept\").click()\n",
    "\n",
    "element = WebDriverWait(driver, 15, .5).until(click_waiter(\"/html/body/div[2]/div[1]/div[1]/div[1]/a[1]\"))\n",
    "driver.find_element_by_xpath(\"/html/body/div[2]/div[1]/div[2]/div[2]/div[2]/a[1]\").click()\n",
    "time.sleep(.5)\n",
    "set_cols = driver.find_element_by_xpath(\"/html/body/div[2]/div[3]/div/div/form/div/div[2]/div[1]/h2/button\")\n",
    "# element2 = WebDriverWait(driver, 15, .5).until(click_waiter(\"/html/body/div[2]/div[3]/div/div/form/div/div[2]/div[1]/h2/button\"))\n",
    "set_cols.click()\n",
    "cols_preset = driver.find_element_by_xpath(\"/html/body/div[4]/div/div/div/div[2]/div/div/div[1]/div[2]/div[4]/div[1]\")\n",
    "cols_preset.click()\n",
    "driver.find_element_by_xpath(\"/html/body/div[4]/div/div/div/div[3]/button[1]\").click()\n",
    "driver.find_element_by_xpath(\"/html/body/div[2]/div[3]/div/div/form/div/div[5]/div/a[1]\").click()\n",
    "\n",
    "download = WebDriverWait(driver, 10).until(EC.visibility_of_element_located((By.XPATH, \"/html/body/div[2]/div[3]/div/div/form/div/div[5]/div/a[3]\")))\n",
    "download.click()\n",
    "time.sleep(3)\n",
    "driver.close()\n",
    "\n",
    "\n",
    "# driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "ename": "ElementClickInterceptedException",
     "evalue": "Message: Element <a class=\"btn-interface btn-toggle-menu \" href=\"#\"> is not clickable at point (25,27) because another element <div id=\"cookiescript_injected\"> obscures it\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mElementClickInterceptedException\u001b[0m          Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-7-778aff7756cc>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m\u001b[0m\n\u001b[1;32m      8\u001b[0m \u001b[0melement\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mwait\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0muntil\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mEC\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0melement_to_be_clickable\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mBy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mXPATH\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0;34m\"/html/body/div[2]/div[1]/div[1]/div[1]/a[1]\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      9\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 10\u001b[0;31m \u001b[0melement\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mclick\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     12\u001b[0m \u001b[0;31m# export = driver.find_element_by_xpath(\"/html/body/div[2]/div[1]/div[2]/div[2]/div[2]/a[1]\")\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/selenium/webdriver/remote/webelement.py\u001b[0m in \u001b[0;36mclick\u001b[0;34m(self)\u001b[0m\n\u001b[1;32m     78\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mclick\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     79\u001b[0m         \u001b[0;34m\"\"\"Clicks the element.\"\"\"\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 80\u001b[0;31m         \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_execute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mCommand\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mCLICK_ELEMENT\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     81\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     82\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0msubmit\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/selenium/webdriver/remote/webelement.py\u001b[0m in \u001b[0;36m_execute\u001b[0;34m(self, command, params)\u001b[0m\n\u001b[1;32m    631\u001b[0m             \u001b[0mparams\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;34m{\u001b[0m\u001b[0;34m}\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    632\u001b[0m         \u001b[0mparams\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'id'\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_id\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 633\u001b[0;31m         \u001b[0;32mreturn\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0m_parent\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mcommand\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    634\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    635\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0mfind_element\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mby\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0mBy\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mID\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;32mNone\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/selenium/webdriver/remote/webdriver.py\u001b[0m in \u001b[0;36mexecute\u001b[0;34m(self, driver_command, params)\u001b[0m\n\u001b[1;32m    319\u001b[0m         \u001b[0mresponse\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcommand_executor\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mexecute\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdriver_command\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mparams\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    320\u001b[0m         \u001b[0;32mif\u001b[0m \u001b[0mresponse\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 321\u001b[0;31m             \u001b[0mself\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merror_handler\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mcheck_response\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mresponse\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    322\u001b[0m             response['value'] = self._unwrap_value(\n\u001b[1;32m    323\u001b[0m                 response.get('value', None))\n",
      "\u001b[0;32m~/anaconda3/lib/python3.7/site-packages/selenium/webdriver/remote/errorhandler.py\u001b[0m in \u001b[0;36mcheck_response\u001b[0;34m(self, response)\u001b[0m\n\u001b[1;32m    240\u001b[0m                 \u001b[0malert_text\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mvalue\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;34m'alert'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mget\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m'text'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    241\u001b[0m             \u001b[0;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0malert_text\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 242\u001b[0;31m         \u001b[0;32mraise\u001b[0m \u001b[0mexception_class\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mmessage\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mscreen\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mstacktrace\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    243\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    244\u001b[0m     \u001b[0;32mdef\u001b[0m \u001b[0m_value_or_default\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mself\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mobj\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mkey\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mdefault\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mElementClickInterceptedException\u001b[0m: Message: Element <a class=\"btn-interface btn-toggle-menu \" href=\"#\"> is not clickable at point (25,27) because another element <div id=\"cookiescript_injected\"> obscures it\n"
     ]
    }
   ],
   "source": [
    "wait = WebDriverWait(driver, 60)\n",
    "\n",
    "element = wait.until(EC.visibility_of_element_located((By.XPATH, \"/html/body/div[2]/div[3]/div/div/form/div/div[5]/div/a[3]\")))\n",
    "\n",
    "element.click()\n",
    "\n",
    "# export = driver.find_element_by_xpath(\"/html/body/div[2]/div[1]/div[2]/div[2]/div[2]/a[1]\")\n",
    "/html/body/div[2]/div[1]/div[2]/div[2]/div[2]/a[1]\n",
    "                                      \n",
    "                                      \n",
    "# Export to CSV / TXT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "('xpath', '/html/body/div[2]/div[1]/div[1]/div[1]/a[1]')"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "ecclick."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "<span class=\"fa fa-fw fa-file-export\"></span> Export to CSV / TXT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "\"//cloud.collectorz.com/wisundstrom/movies/export/all\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "\n",
    "elem.clear()\n",
    "elem.send_keys(\"pycon\")\n",
    "elem.send_keys(Keys.RETURN)\n",
    "assert \"No results found.\" not in driver.page_source\n",
    "driver.close()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}