DataRetrieval.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import scipy\n",
    "from config import api_key, app_id\n",
    "import requests\n",
    "from pprint import pprint"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [],
   "source": [
    "base_organization_url = \"https://api.data.charitynavigator.org/v2/Organizations\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 46,
   "metadata": {},
   "outputs": [],
   "source": [
    "size_range_options = {\n",
    "    \"UP_TO_3.5MM\": 0,\n",
    "    \"3.5MM_TO_13.5MM\": 1,\n",
    "    \"13.5MM_AND_UP\": 2\n",
    "}\n",
    "\n",
    "default_search_params = {\n",
    "    \"app_id\": app_id,\n",
    "    \"app_key\": api_key,\n",
    "    \"rated\": \"true\",\n",
    "    \"pageSize\": 1000,\n",
    "    \"pageNum\": 1\n",
    "}"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 47,
   "metadata": {},
   "outputs": [],
   "source": [
    "def createOrganizationSearchURL(size_search_params=default_search_params):\n",
    "    query_params = [f\"{param}={value}\" for (param, value) in size_search_params.items()]\n",
    "    query_string = \"&\".join(query_params)\n",
    "    url = f\"{base_organization_url}?{query_string}\"\n",
    "    return url\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# size_search = default_search_params\n",
    "url = createOrganizationSearchURL()\n",
    "print(url)\n",
    "response = requests.get(url)\n",
    "print(response.status_code)\n",
    "data = response.json()\n",
    "pprint(data)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 49,
   "metadata": {},
   "outputs": [],
   "source": [
    "results = []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "search_params = default_search_params\n",
    "\n",
    "for page_num in range(1, 20):\n",
    "    print(page_num)\n",
    "    search_params[\"pageNum\"] = page_num\n",
    "    print(search_params)\n",
    "    organizations_url = createOrganizationSearchURL(search_params)\n",
    "    response = requests.get(organizations_url)\n",
    "    if (response.status_code != 200):\n",
    "        break;\n",
    "    else:\n",
    "        result_df = pd.DataFrame(response.json())\n",
    "        results.append(result_df)\n",
    "\n",
    "total_results = pd.concat(results)\n",
    "total_results.to_csv(\"All_Organizations.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 63,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>advisories</th>\n",
       "      <th>category</th>\n",
       "      <th>cause</th>\n",
       "      <th>charityName</th>\n",
       "      <th>charityNavigatorURL</th>\n",
       "      <th>currentRating</th>\n",
       "      <th>donationAddress</th>\n",
       "      <th>ein</th>\n",
       "      <th>irsClassification</th>\n",
       "      <th>mailingAddress</th>\n",
       "      <th>mission</th>\n",
       "      <th>orgID</th>\n",
       "      <th>organization</th>\n",
       "      <th>tagLine</th>\n",
       "      <th>websiteURL</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>{'severity': None, 'active': {'_rapid_links': ...</td>\n",
       "      <td>{'categoryName': 'Research and Public Policy',...</td>\n",
       "      <td>{'causeID': 35, 'causeName': 'Non-Medical Scie...</td>\n",
       "      <td>Mount Desert Island Biological Laboratory</td>\n",
       "      <td>https://www.charitynavigator.org/?bay=search.s...</td>\n",
       "      <td>{'score': 92.61, 'ratingID': 131800, 'publicat...</td>\n",
       "      <td>{'country': None, 'stateOrProvince': 'ME', 'ci...</td>\n",
       "      <td>010202467</td>\n",
       "      <td>{'deductibility': 'Contributions are deductibl...</td>\n",
       "      <td>{'country': None, 'stateOrProvince': 'ME', 'ci...</td>\n",
       "      <td>The MDI Biological Laboratory is a rapidly gro...</td>\n",
       "      <td>5954</td>\n",
       "      <td>{'charityName': 'Mount Desert Island Biologica...</td>\n",
       "      <td>Connecting Science, Environment, and Health</td>\n",
       "      <td>http://www.mdibl.org/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>{'severity': None, 'active': {'_rapid_links': ...</td>\n",
       "      <td>{'categoryName': 'Community Development', 'cat...</td>\n",
       "      <td>{'causeID': 42, 'causeName': 'United Ways', 'c...</td>\n",
       "      <td>United Way of Eastern Maine</td>\n",
       "      <td>https://www.charitynavigator.org/?bay=search.s...</td>\n",
       "      <td>{'score': 81.84, 'ratingID': 138738, 'publicat...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>010211478</td>\n",
       "      <td>{'deductibility': 'Contributions are deductibl...</td>\n",
       "      <td>{'country': None, 'stateOrProvince': 'ME', 'ci...</td>\n",
       "      <td>Working with the communities we serve, the Uni...</td>\n",
       "      <td>12517</td>\n",
       "      <td>{'charityName': 'United Way of Eastern Maine',...</td>\n",
       "      <td>Live united</td>\n",
       "      <td>https://www.unitedwayem.org/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>{'severity': None, 'active': {'_rapid_links': ...</td>\n",
       "      <td>{'categoryName': 'Health', 'categoryID': 5, 'c...</td>\n",
       "      <td>{'causeID': 14, 'causeName': 'Medical Research...</td>\n",
       "      <td>The Jackson Laboratory</td>\n",
       "      <td>https://www.charitynavigator.org/?bay=search.s...</td>\n",
       "      <td>{'score': 93.16, 'ratingID': 131608, 'publicat...</td>\n",
       "      <td>{'country': None, 'stateOrProvince': 'ME', 'ci...</td>\n",
       "      <td>010211513</td>\n",
       "      <td>{'deductibility': 'Contributions are deductibl...</td>\n",
       "      <td>{'country': None, 'stateOrProvince': 'ME', 'ci...</td>\n",
       "      <td>The Jackson Laboratory, founded in 1929, is an...</td>\n",
       "      <td>3916</td>\n",
       "      <td>{'charityName': 'The Jackson Laboratory', 'ein...</td>\n",
       "      <td>Leading the search for tomorrow's cures</td>\n",
       "      <td>http://www.jax.org</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>{'severity': None, 'active': {'_rapid_links': ...</td>\n",
       "      <td>{'categoryName': 'Arts, Culture, Humanities', ...</td>\n",
       "      <td>{'causeID': 6, 'causeName': 'Libraries, Histor...</td>\n",
       "      <td>Maine Historical Society</td>\n",
       "      <td>https://www.charitynavigator.org/?bay=search.s...</td>\n",
       "      <td>{'score': 80.31, 'ratingID': 124121, 'publicat...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>010211530</td>\n",
       "      <td>{'deductibility': 'Contributions are deductibl...</td>\n",
       "      <td>{'country': None, 'stateOrProvince': 'ME', 'ci...</td>\n",
       "      <td>Founded in 1822, the Maine Historical Society ...</td>\n",
       "      <td>15533</td>\n",
       "      <td>{'charityName': 'Maine Historical Society', 'e...</td>\n",
       "      <td>Preserving history, engaging minds, connecting...</td>\n",
       "      <td>http://www.mainehistory.org/</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>{'severity': None, 'active': {'_rapid_links': ...</td>\n",
       "      <td>{'categoryName': 'Human Services', 'categoryID...</td>\n",
       "      <td>{'causeID': 16, 'causeName': 'Youth Developmen...</td>\n",
       "      <td>Boys &amp; Girls Clubs of Southern Maine</td>\n",
       "      <td>https://www.charitynavigator.org/?bay=search.s...</td>\n",
       "      <td>{'score': 91.54, 'ratingID': 135033, 'publicat...</td>\n",
       "      <td>NaN</td>\n",
       "      <td>010211543</td>\n",
       "      <td>{'deductibility': 'Contributions are deductibl...</td>\n",
       "      <td>{'country': None, 'stateOrProvince': 'ME', 'ci...</td>\n",
       "      <td>For more than 100 years, Boys &amp; Girls Clubs of...</td>\n",
       "      <td>15222</td>\n",
       "      <td>{'charityName': 'Boys &amp; Girls Clubs of Souther...</td>\n",
       "      <td>Great Futures Start Here</td>\n",
       "      <td>http://www.bgcmaine.org/home</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                          advisories  \\\n",
       "0  {'severity': None, 'active': {'_rapid_links': ...   \n",
       "1  {'severity': None, 'active': {'_rapid_links': ...   \n",
       "2  {'severity': None, 'active': {'_rapid_links': ...   \n",
       "3  {'severity': None, 'active': {'_rapid_links': ...   \n",
       "4  {'severity': None, 'active': {'_rapid_links': ...   \n",
       "\n",
       "                                            category  \\\n",
       "0  {'categoryName': 'Research and Public Policy',...   \n",
       "1  {'categoryName': 'Community Development', 'cat...   \n",
       "2  {'categoryName': 'Health', 'categoryID': 5, 'c...   \n",
       "3  {'categoryName': 'Arts, Culture, Humanities', ...   \n",
       "4  {'categoryName': 'Human Services', 'categoryID...   \n",
       "\n",
       "                                               cause  \\\n",
       "0  {'causeID': 35, 'causeName': 'Non-Medical Scie...   \n",
       "1  {'causeID': 42, 'causeName': 'United Ways', 'c...   \n",
       "2  {'causeID': 14, 'causeName': 'Medical Research...   \n",
       "3  {'causeID': 6, 'causeName': 'Libraries, Histor...   \n",
       "4  {'causeID': 16, 'causeName': 'Youth Developmen...   \n",
       "\n",
       "                                 charityName  \\\n",
       "0  Mount Desert Island Biological Laboratory   \n",
       "1                United Way of Eastern Maine   \n",
       "2                     The Jackson Laboratory   \n",
       "3                   Maine Historical Society   \n",
       "4       Boys & Girls Clubs of Southern Maine   \n",
       "\n",
       "                                 charityNavigatorURL  \\\n",
       "0  https://www.charitynavigator.org/?bay=search.s...   \n",
       "1  https://www.charitynavigator.org/?bay=search.s...   \n",
       "2  https://www.charitynavigator.org/?bay=search.s...   \n",
       "3  https://www.charitynavigator.org/?bay=search.s...   \n",
       "4  https://www.charitynavigator.org/?bay=search.s...   \n",
       "\n",
       "                                       currentRating  \\\n",
       "0  {'score': 92.61, 'ratingID': 131800, 'publicat...   \n",
       "1  {'score': 81.84, 'ratingID': 138738, 'publicat...   \n",
       "2  {'score': 93.16, 'ratingID': 131608, 'publicat...   \n",
       "3  {'score': 80.31, 'ratingID': 124121, 'publicat...   \n",
       "4  {'score': 91.54, 'ratingID': 135033, 'publicat...   \n",
       "\n",
       "                                     donationAddress        ein  \\\n",
       "0  {'country': None, 'stateOrProvince': 'ME', 'ci...  010202467   \n",
       "1                                                NaN  010211478   \n",
       "2  {'country': None, 'stateOrProvince': 'ME', 'ci...  010211513   \n",
       "3                                                NaN  010211530   \n",
       "4                                                NaN  010211543   \n",
       "\n",
       "                                   irsClassification  \\\n",
       "0  {'deductibility': 'Contributions are deductibl...   \n",
       "1  {'deductibility': 'Contributions are deductibl...   \n",
       "2  {'deductibility': 'Contributions are deductibl...   \n",
       "3  {'deductibility': 'Contributions are deductibl...   \n",
       "4  {'deductibility': 'Contributions are deductibl...   \n",
       "\n",
       "                                      mailingAddress  \\\n",
       "0  {'country': None, 'stateOrProvince': 'ME', 'ci...   \n",
       "1  {'country': None, 'stateOrProvince': 'ME', 'ci...   \n",
       "2  {'country': None, 'stateOrProvince': 'ME', 'ci...   \n",
       "3  {'country': None, 'stateOrProvince': 'ME', 'ci...   \n",
       "4  {'country': None, 'stateOrProvince': 'ME', 'ci...   \n",
       "\n",
       "                                             mission  orgID  \\\n",
       "0  The MDI Biological Laboratory is a rapidly gro...   5954   \n",
       "1  Working with the communities we serve, the Uni...  12517   \n",
       "2  The Jackson Laboratory, founded in 1929, is an...   3916   \n",
       "3  Founded in 1822, the Maine Historical Society ...  15533   \n",
       "4  For more than 100 years, Boys & Girls Clubs of...  15222   \n",
       "\n",
       "                                        organization  \\\n",
       "0  {'charityName': 'Mount Desert Island Biologica...   \n",
       "1  {'charityName': 'United Way of Eastern Maine',...   \n",
       "2  {'charityName': 'The Jackson Laboratory', 'ein...   \n",
       "3  {'charityName': 'Maine Historical Society', 'e...   \n",
       "4  {'charityName': 'Boys & Girls Clubs of Souther...   \n",
       "\n",
       "                                             tagLine  \\\n",
       "0        Connecting Science, Environment, and Health   \n",
       "1                                        Live united   \n",
       "2            Leading the search for tomorrow's cures   \n",
       "3  Preserving history, engaging minds, connecting...   \n",
       "4                           Great Futures Start Here   \n",
       "\n",
       "                     websiteURL  \n",
       "0         http://www.mdibl.org/  \n",
       "1  https://www.unitedwayem.org/  \n",
       "2            http://www.jax.org  \n",
       "3  http://www.mainehistory.org/  \n",
       "4  http://www.bgcmaine.org/home  "
      ]
     },
     "execution_count": 63,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "total_results.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.6"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}