EmailParser-Images.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import mailbox\n",
    "import email.utils\n",
    "import pandas as pd\n",
    "import exifread\n",
    "from datetime import datetime, timezone\n",
    "import pytz\n",
    "from ipywidgets import IntProgress, IntText"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "filename = \"tgthorley.mbox\"\n",
    "filepath = './data/test/'\n",
    "\n",
    "mbox = mailbox.mbox(filepath + filename)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_email_summary(message):\n",
    "    emaildate = email.utils.parsedate_to_datetime(message['Date']).astimezone(pytz.utc)\n",
    "    email_sum = {\n",
    "            \"email_date\": emaildate,\n",
    "            \"_id\": message['Message-ID'],\n",
    "            \"subject\": message['Subject']\n",
    "        }\n",
    "    return email_sum"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "def create_file(part):\n",
    "    try:\n",
    "        filename = part.get_filename()\n",
    "        filepath = \"./attachements/\"+filename\n",
    "        fb = open(filepath,'wb')\n",
    "        fb.write(part.get_payload(decode=True))\n",
    "        fb.close()\n",
    "        return filepath\n",
    "    except:\n",
    "        return \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "#takes an image path and retruns exif data\n",
    "def parse_img(imgpath):\n",
    "    try:\n",
    "        f = open(imgpath, 'rb')\n",
    "        tags = exifread.process_file(f)\n",
    "        return tags\n",
    "    except:\n",
    "        print(\"Failed to extract exif %s\" % imgpath)\n",
    "        return []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "#takes a message with an attached immage  and returns the exif data\n",
    "def getexif_from_attached_image(image):\n",
    "    try:\n",
    "        filepath = create_file(image)\n",
    "        tags = parse_img(filepath)\n",
    "        return tags\n",
    "    except:\n",
    "        print (\"File not found\", \"./attachements/\"+image.get_filename())\n",
    "        return {}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [],
   "source": [
    "def compare_sent_time_and_img_creation(tags, message):\n",
    "    try:\n",
    "        emaildate = email.utils.parsedate_to_datetime(message['Date']).astimezone(pytz.utc)\n",
    "        email_sum = create_email_summary(message)\n",
    "        if \"EXIF DateTimeOriginal\" in tags:\n",
    "            \n",
    "            imgdate = datetime.strptime(tags[\"EXIF DateTimeOriginal\"].values, '%Y:%m:%d %H:%M:%S').astimezone(pytz.utc)\n",
    "            timediff = emaildate - imgdate\n",
    "            if timediff.total_seconds() > 0 and timediff.total_seconds() < 86400:\n",
    "                print ('WARNING: Image taken less than a day after sending\\n', message['Message-ID'], message['Subject'] )\n",
    "            email_sum[\"img_date\"] = imgdate\n",
    "            email_sum[\"timediff\"] = timediff\n",
    "            \n",
    "        return email_sum\n",
    "    except: \n",
    "        print(\"Error\", message['Subject'])\n",
    "        return {}\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 84,
   "metadata": {},
   "outputs": [],
   "source": [
    "def _convert_to_degress(value, ref):\n",
    "    if ref.values == \"N\" or ref.values == \"E\":\n",
    "        d = float(value.values[0].num) / float(value.values[0].den)\n",
    "        m = float(value.values[1].num) / float(value.values[1].den)\n",
    "        s = float(value.values[2].num) / float(value.values[2].den)\n",
    "    elif ref.values == \"S\" or ref.values == \"W\":\n",
    "        d = -(float(value.values[0].num) / float(value.values[0].den))\n",
    "        m = -(float(value.values[1].num) / float(value.values[1].den))\n",
    "        s = -(float(value.values[2].num) / float(value.values[2].den))  \n",
    "    else: print(ref.values)\n",
    "\n",
    "    return d + (m / 60.0) + (s / 3600.0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 91,
   "metadata": {},
   "outputs": [],
   "source": [
    "def extract_geo(tags, email_sum):\n",
    "    if 'GPS GPSLatitude' in tags:\n",
    "        lat = tags['GPS GPSLatitude']\n",
    "        latRef = tags['GPS GPSLatitudeRef']\n",
    "        lat_decimal = _convert_to_degress(lat, latRef)\n",
    "        long = tags['GPS GPSLongitude']\n",
    "        longRef = tags['GPS GPSLongitudeRef']\n",
    "        long_decimal = _convert_to_degress(long, longRef)\n",
    "        email_sum['GPS GPSLatitude']= lat_decimal\n",
    "        email_sum['GPS GPSLongitude']= long_decimal\n",
    "        #print (\"https://www.geoplaner.com/?z=10;p=\"+str(lat_decimal)+\",\"+str(long_decimal)+\";\")\n",
    "    return email_sum\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e95fad1334024dd1854f27323e6bd82a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "IntProgress(value=0, description='Processing emails 2364', max=2364, style=ProgressStyle(description_width='in…"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "91db89cd38a4407c9dd41302f019fc15",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "IntText(value=0, description='Processed so far: ', style=DescriptionStyle(description_width='initial'))"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "https://www.geoplaner.com/?z=10;p=32.65623055555555,-16.91846388888889;\n",
      "https://www.geoplaner.com/?z=10;p=51.867000555555556,-2.2303864722222224;\n",
      "\n",
      "\n",
      "https://www.geoplaner.com/?z=10;p=51.71088027777778,-1.9603111666666666;\n",
      "https://www.geoplaner.com/?z=10;p=51.71088027777778,-1.9603111666666666;\n",
      "\n",
      "\n",
      "https://www.geoplaner.com/?z=10;p=51.867000555555556,-2.230389111111111;\n",
      "\n",
      "\n",
      "https://www.geoplaner.com/?z=10;p=51.710880555555555,-1.9601722222222222;\n",
      "https://www.geoplaner.com/?z=10;p=51.710880555555555,-1.9601722222222222;\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "https://www.geoplaner.com/?z=10;p=51.71088788888889,-1.9602635;\n",
      "https://www.geoplaner.com/?z=10;p=51.71090316666667,-1.9602563055555555;\n",
      "https://www.geoplaner.com/?z=10;p=51.71090316666667,-1.9602800277777777;\n",
      "https://www.geoplaner.com/?z=10;p=51.710906972222226,-1.9602930555555556;\n",
      "https://www.geoplaner.com/?z=10;p=51.710914611111114,-1.9602980277777777;\n",
      "https://www.geoplaner.com/?z=10;p=51.710906972222226,-1.9602985277777778;\n",
      "https://www.geoplaner.com/?z=10;p=51.71090316666667,-1.9603143055555554;\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "https://www.geoplaner.com/?z=10;p=51.7109375,-1.960258611111111;\n",
      "https://www.geoplaner.com/?z=10;p=51.7109375,-1.960258611111111;\n",
      "https://www.geoplaner.com/?z=10;p=51.710914611111114,-1.9602569444444444;\n",
      "https://www.geoplaner.com/?z=10;p=51.71091841666667,-1.9602390277777777;\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Failed to extract exif ./attachements/fullsizeoutput_ed3.jpeg\n",
      "\n",
      "\n",
      "https://www.geoplaner.com/?z=10;p=51.71092986111111,-1.96027525;\n",
      "https://www.geoplaner.com/?z=10;p=50.334605555555555,-4.635688888888889;\n",
      "https://www.geoplaner.com/?z=10;p=51.86709594444444,-2.2308375833333334;\n",
      "https://www.geoplaner.com/?z=10;p=51.86709594444444,-2.2308375833333334;\n",
      "https://www.geoplaner.com/?z=10;p=51.86709594444444,-2.2308375833333334;\n",
      "https://www.geoplaner.com/?z=10;p=51.86709594444444,-2.2308375833333334;\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "https://www.geoplaner.com/?z=10;p=51.86709594444444,-2.2308285000000003;\n",
      "https://www.geoplaner.com/?z=10;p=51.86709594444444,-2.2308375833333334;\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "emails = []\n",
    "i=0\n",
    "progressBar = IntProgress(description='Processing emails '+str(len(mbox)), min=0, max=len(mbox), style = {'description_width': 'initial'})\n",
    "progressText = IntText(value=0, description='Processed so far: ', style = {'description_width': 'initial'})\n",
    "display(progressBar, progressText)\n",
    "\n",
    "for message in mbox:\n",
    "    email_sum = {}\n",
    "    if message.get_content_maintype() == 'multipart':\n",
    "        for part in message.walk():\n",
    "            if 'image' in part.get_content_type():\n",
    "                tags = getexif_from_attached_image(part)\n",
    "                email_sum.update(compare_sent_time_and_img_creation(tags, message))\n",
    "                email_sum.update(extract_geo(tags, email_sum))\n",
    "    emails.append(email_sum)\n",
    "    i= i+1\n",
    "    progressBar.value = i\n",
    "    progressText.value = i\n",
    "            "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df = pd.DataFrame(emails)\n",
    "df.sort_values(\"timediff\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}