data-cleaning-new-data-set.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import matplotlib\n",
    "%matplotlib inline\n",
    "import math\n",
    "import os"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make the column name representing the states to be unique across all files as 'State/UT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['new-facilities/mid-day-meal.csv',\n",
       " 'new-facilities/drinking-water.csv',\n",
       " 'new-facilities/boys-toilet.csv',\n",
       " 'new-facilities/hand-wash-facility.csv',\n",
       " 'new-facilities/electricitiy-connection.csv',\n",
       " 'new-facilities/girls-toilet.csv',\n",
       " 'new-facilities/play-ground.csv',\n",
       " 'new-facilities/text-books-received.csv',\n",
       " 'new-facilities/computer-aided-learning.csv',\n",
       " 'new-facilities/kitchen-shed.csv',\n",
       " 'new-facilities/boundary-wall.csv',\n",
       " 'new-facilities/computer-availability.csv',\n",
       " 'new-facilities/ramp-facility.csv']"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#get all the path of files which we are interested in\n",
    "import glob\n",
    "#paths = glob.glob('new-data-set/**/*.csv',recursive=True)\n",
    "paths = glob.glob('new-facilities/**/*.csv',recursive=True)\n",
    "paths"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "{'States/UT', 'All Schools 2012-13 (Functional Computers)', 'States/UTs', 'State/UTs'}\n",
      "\n",
      "{'new-facilities/mid-day-meal.csv', 'new-facilities/computer-aided-learning.csv', 'new-facilities/computer-availability.csv', 'new-facilities/boys-toilet.csv', 'new-facilities/ramp-facility.csv', 'new-facilities/text-books-received.csv', 'new-facilities/electricitiy-connection.csv'}\n"
     ]
    }
   ],
   "source": [
    "#check whether there are any other column which contains the name india, state or UT or union or territories\n",
    "\n",
    "columnlist = []\n",
    "similiar_columns = []\n",
    "similiar_file_paths = []\n",
    "for path in paths:\n",
    "    if(os.path.isfile(path)) :\n",
    "        data = pd.read_csv(path)\n",
    "        columnlist = list(data.columns)\n",
    "        allIsWell = False\n",
    "        #print(path,end=None)\n",
    "        for column in columnlist:\n",
    "            if (column != 'State/UT'):\n",
    "                col_lower = column.lower()\n",
    "                if(col_lower.count('india') > 0 or \\\n",
    "                   col_lower.count('state') > 0 or \n",
    "                   col_lower.count('union') > 0 or \n",
    "                   col_lower.count('territor') > 0 or\n",
    "                   col_lower.count('ut') > 0\n",
    "                  ):\n",
    "                    #print(column,end=None)\n",
    "                    similiar_columns.append(column)\n",
    "                    similiar_file_paths.append(path)\n",
    "        #print('')\n",
    "print(set(similiar_columns))\n",
    "print()\n",
    "print(set(similiar_file_paths))\n",
    "# the items listed should be changed to state/ut because they are representing the states information.\n",
    "# they are manually checked and removed\n",
    "#some items are listed - manually checked that they are not representing states (so cool :-)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    " In the above case, I consider the lists 'India/ State/ UTs', 'States \\\\UT', 'STATE/U.T.', 'States', 'States\\\\UT', 'India/ State/ UTs', 'States \\\\UT', 'States\\\\UT', 'States/Union Territories', 'States/UTs', 'State/UT (1)', 'STATE/U.T.', 'States', 'States\\\\UTs', 'State/U.T.', 'All India/State/Union Territory' represent the same information 'State/UT'. Let me replace these names with 'State/UT'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "new-facilities/mid-day-meal.csv\n",
      "new-facilities/computer-aided-learning.csv\n",
      "new-facilities/computer-availability.csv\n",
      "new-facilities/boys-toilet.csv\n",
      "new-facilities/ramp-facility.csv\n",
      "new-facilities/text-books-received.csv\n",
      "new-facilities/electricitiy-connection.csv\n"
     ]
    }
   ],
   "source": [
    "replace_list = ['India/ State/ UTs', 'States \\\\UT', 'STATE/U.T.', 'States', 'States\\\\UT',\n",
    "                'India/ State/ UTs', 'States \\\\UT', 'States\\\\UT', 'States/Union Territories', \n",
    "                'States/UT', 'States/UTs', 'State/UTs',\n",
    "                'State/UT (1)', 'STATE/U.T.', 'States', 'States\\\\UTs', 'State/U.T.', \n",
    "                'India/ State/ UTs', 'States \\\\UT', 'STATE/U.T.', 'States', 'States\\\\UT', \n",
    "                'States/Union Territories',\n",
    "                'All India/State/Union Territory']\n",
    "replace_value = 'State/UT'\n",
    "unique_file_paths = set(similiar_file_paths)\n",
    "\n",
    "for file in unique_file_paths:\n",
    "    data = pd.read_csv(file)\n",
    "    columnlist = list(data.columns)\n",
    "    changed = False\n",
    "    for column in columnlist:\n",
    "        if(column in replace_list):\n",
    "            data.rename(columns={column:replace_value},index=str,inplace=True)\n",
    "            changed = True\n",
    "    if (changed):\n",
    "        data.to_csv(file,index=False)\n",
    "        print(file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# to list down files which does not have column 'State/UT'\n",
    "columnlist = []\n",
    "state_column_not_present_files = []\n",
    "for path in paths:\n",
    "    if(os.path.isfile(path)) :\n",
    "        data = pd.read_csv(path)\n",
    "        columnlist = list(data.columns)\n",
    "        allIsWell = False\n",
    "        for column in columnlist:\n",
    "            if (column == 'State/UT'):\n",
    "                allIsWell = True\n",
    "        if(not allIsWell):\n",
    "            print(path)\n",
    "            state_column_not_present_files.append(path)\n",
    "            \n",
    "# The changes are listed and it should be minimum\n",
    "# they can be changed manually. Now all the columns are same which represents the states\n",
    "\n",
    "# when no file path is displayed then hooray . Now every file contains a column 'State/UT' column"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "set()\n",
      "\n",
      "set()\n"
     ]
    }
   ],
   "source": [
    "columnlist = []\n",
    "similiar_columns = []\n",
    "similiar_file_paths = []\n",
    "for path in state_column_not_present_files:\n",
    "    if(os.path.isfile(path)) :\n",
    "        data = pd.read_csv(path)\n",
    "        columnlist = list(data.columns)\n",
    "        allIsWell = False\n",
    "        #print(path,end=None)\n",
    "        for column in columnlist:\n",
    "            if (column != 'State/UT'):\n",
    "                col_lower = column.lower()\n",
    "                if(col_lower.count('india') > 0 or \\\n",
    "                   col_lower.count('state') > 0 or \n",
    "                   col_lower.count('union') > 0 or \n",
    "                   col_lower.count('territor') > 0 or\n",
    "                   col_lower.count('ut') > 0\n",
    "                  ):\n",
    "                    #print(column,end=None)\n",
    "                    similiar_columns.append(column)\n",
    "                    similiar_file_paths.append(path)\n",
    "        #print('')\n",
    "print(set(similiar_columns))\n",
    "print()\n",
    "print(set(similiar_file_paths))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Some of the values such as \n",
    "\n",
    "'India/ State/ UTs', 'States \\\\UT', 'STATE/U.T.', 'States', 'States\\\\UT', 'States/Union Territories'\n",
    "\n",
    "where already replaced, but yet it still shows. I don't know why. Let us check the code above. If we could not find problem then just change it manually\n",
    "\n",
    "The last three code shells are running back and forth and now we made sure that\n",
    "every file contains a column 'State/UT'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "'hihh'"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#looking for a method \n",
    "\n",
    "#contains\n",
    "temp = 'hihH'\n",
    "print(temp.count('h') )\n",
    "\n",
    "# to lower case\n",
    "temp.lower()\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Make the state names unique across all files"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['Andaman and Nicobar Islands',\n",
       " 'Andhra Pradesh',\n",
       " 'Arunachal Pradesh',\n",
       " 'Assam',\n",
       " 'Bihar',\n",
       " 'Chandigarh',\n",
       " 'Chhattisgarh',\n",
       " 'Dadra and Nagar Haveli',\n",
       " 'Daman and Diu',\n",
       " 'Delhi',\n",
       " 'Goa',\n",
       " 'Gujarat',\n",
       " 'Haryana',\n",
       " 'Himachal Pradesh',\n",
       " 'Jammu and Kashmir',\n",
       " 'Jharkhand',\n",
       " 'Karnataka',\n",
       " 'Kerala',\n",
       " 'Lakshadweep',\n",
       " 'Madhya Pradesh',\n",
       " 'Maharashtra',\n",
       " 'Manipur',\n",
       " 'Meghalaya',\n",
       " 'Mizoram',\n",
       " 'Nagaland',\n",
       " 'Orissa',\n",
       " 'Puducherry',\n",
       " 'Punjab',\n",
       " 'Rajasthan',\n",
       " 'Sikkim',\n",
       " 'Tamil Nadu',\n",
       " 'Tripura',\n",
       " 'Uttar Pradesh',\n",
       " 'Uttarakhand',\n",
       " 'West Bengal']"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#some NaN values are also present as part of state. which needs to be avoided\n",
    "\n",
    "state_names = []\n",
    "for path in paths:\n",
    "    if(os.path.isfile(path)) :\n",
    "        data = pd.read_csv(path)\n",
    "        temp = data['State/UT']\n",
    "        tempList = []\n",
    "        for x in temp:\n",
    "            if(isinstance(x,str)):\n",
    "                tempList.append(x)\n",
    "            else:\n",
    "                print(path)\n",
    "                \n",
    "        state_names = state_names + tempList\n",
    "        \n",
    "state_names_unique = set(state_names)\n",
    "sorted(state_names_unique)\n",
    "\n",
    "# following are the list of names. figure out the duplications and change the dataframe to make it the same\n",
    "# If you see something absurd then it means that we have wrongly changed the column names. So go back and start\n",
    "# with fresh dataset and see what can be done"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "\n",
    "for path in paths:\n",
    "    if(os.path.isfile(path)) :\n",
    "        data = pd.read_csv(path)\n",
    "        #print(data['State/UT'][0:10])\n",
    "        data.replace(['A&N Islands','A & N Islands', 'A. and N. Islands', 'Andaman  Nicobar Islands', 'Andaman & Nicobar Islands'], \n",
    "                     ['Andaman and Nicobar Islands', 'Andaman and Nicobar Islands', 'Andaman and Nicobar Islands', 'Andaman and Nicobar Islands', 'Andaman and Nicobar Islands'], \n",
    "                     inplace=True)\n",
    "        \n",
    "        data.replace(['Bihar^'],\n",
    "                     ['Bihar'], \n",
    "                     inplace=True)\n",
    "                     \n",
    "        data.replace(['Chhatisgarh','Chattisgarh#'], \n",
    "                     ['Chhattisgarh','Chhattisgarh'], \n",
    "                     inplace=True)\n",
    "\n",
    "        data.replace(['D. and N. Haveli', 'Dadra  Nagar Haveli', 'Dadra & Nagar Haveli', 'D&N Haveli'], \n",
    "                     ['Dadra and Nagar Haveli', 'Dadra and Nagar Haveli', 'Dadra and Nagar Haveli','Dadra and Nagar Haveli'], \n",
    "                     inplace=True)\n",
    "        \n",
    "        data.replace(['Daman  Diu', 'Daman & Diu', 'Daman Diu'], \n",
    "                     ['Daman and Diu','Daman and Diu','Daman and Diu'], \n",
    "                     inplace=True)\n",
    "        \n",
    "        data.replace(['INDIA', 'All India', 'All States', 'ALL INDIA','All States\\\\UT' ], \n",
    "                     ['India','India','India','India','India'], \n",
    "                     inplace=True)\n",
    "        \n",
    "        data.replace(['Jharkhand#'], \n",
    "                     ['Jharkhand'], \n",
    "                     inplace=True)\n",
    "        \n",
    "        data.replace(['Jammu  Kashmir', 'Jammu & Kashmir', 'Jammu & Kashmir'], \n",
    "                     ['Jammu and Kashmir', 'Jammu and Kashmir', 'Jammu and Kashmir'], \n",
    "                     inplace=True)\n",
    "        \n",
    "        data.replace(['Madhya Pradesh^^'], \n",
    "                     ['Madhya Pradesh'], \n",
    "                     inplace=True)\n",
    "        \n",
    "        data.replace(['Odisha'], \n",
    "                     ['Orissa'], \n",
    "                     inplace=True)\n",
    "        \n",
    "        data.replace(['Pondicherry'], \n",
    "                     ['Puducherry'], \n",
    "                     inplace=True)\n",
    "        \n",
    "        data.replace(['Sikkim@'], \n",
    "                     ['Sikkim'], \n",
    "                     inplace=True)\n",
    "        \n",
    "        data.replace(['Tamilnadu'], \n",
    "                     ['Tamil Nadu'], \n",
    "                     inplace=True)\n",
    "     \n",
    "        data.replace(['Uttar Pradesh^^^'], \n",
    "                     ['Uttar Pradesh'], \n",
    "                     inplace=True)\n",
    "        \n",
    "        data.replace(['Uttrakhand', 'Uttaranchal','Uttarakhand#$'], \n",
    "                     ['Uttarakhand', 'Uttarakhand', 'Uttarakhand'], \n",
    "                     inplace=True)\n",
    "\n",
    "        data['State/UT'] = data['State/UT'].str.strip()\n",
    "        \n",
    "        #print(path)\n",
    "        # index=False makes sure that the CSV will not have row indexes stored\n",
    "        data.to_csv(path, index=False) \n",
    "        \n",
    "        #break"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "No. of states and union territories -  35\n"
     ]
    }
   ],
   "source": [
    "# print the states to check whether anything is missed out\n",
    "# if found then go back and fix it\n",
    "state_names = []\n",
    "for path in paths:\n",
    "    if(os.path.isfile(path)) :\n",
    "        data = pd.read_csv(path)\n",
    "        state_names = state_names + list(data['State/UT'])\n",
    "sorted(set(state_names))\n",
    "print('No. of states and union territories - ', len(sorted(set(state_names))))\n",
    "\n",
    "# it is listed as 38 (but it should be 35)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "data": {
      "text/plain": [
       "35"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#'Sample Average',  'Total', 'India' need to be removed since they don't represent the states or union territories\n",
    "all_states = []\n",
    "for path in paths:\n",
    "    if(os.path.isfile(path)) :\n",
    "        data = pd.read_csv(path)\n",
    "        data = data.loc[(data['State/UT']!='Sample Average') & (data['State/UT']!='Total') & (data['State/UT']!='India')]\n",
    "        all_states = all_states + list(data['State/UT'])\n",
    "        data.to_csv(path,index=False)\n",
    "        #data = data[data['State/UT']!='Sample Average']\n",
    "all_states = set(all_states)        \n",
    "len(sorted(all_states))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### Check whether any dataset is missing any state information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "#all_states - contains all the state names \n",
    "# there are 35 states\n",
    "state_names = []\n",
    "for path in paths:\n",
    "    if(os.path.isfile(path)) :\n",
    "        data = pd.read_csv(path)\n",
    "        state_names = data['State/UT']\n",
    "        no_of_states = len(state_names)\n",
    "        if(no_of_states != len(all_states)):\n",
    "            print(path)\n",
    "            \n",
    "# oh no. almost everything has some state information missing"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [],
   "source": [
    "# let us look at files and see how much of state info is missing in each file\n",
    "state_names = []\n",
    "for path in paths:\n",
    "    if(os.path.isfile(path)) :\n",
    "        data = pd.read_csv(path)\n",
    "        state_names = data['State/UT']\n",
    "        no_of_states = len(state_names)\n",
    "        if(no_of_states != len(all_states)):\n",
    "            print(path+' - ',no_of_states)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# there are some files which have more than 35 rows.\n",
    "# Let us see what are they\n",
    "# let us look at files and see how much of state info is present in each\n",
    "\n",
    "state_names = []\n",
    "for path in paths:\n",
    "    if(os.path.isfile(path)) :\n",
    "        data = pd.read_csv(path)\n",
    "        state_names = data['State/UT']\n",
    "        #state_names = set(state_names)\n",
    "        no_of_states = len(state_names)\n",
    "        if(no_of_states > 35):\n",
    "            print(path+' - ',no_of_states)\n",
    "            \n",
    "# Let us examine the files by hand\n",
    "# What we found was there were sub group information such as GENDER and School Type\n",
    "# For each of this, 35 state information is present and hence the increase in rows\n",
    "\n",
    "# Unique state names will be 35 in number"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "temp = 'new-data-set/faculty-strength-quality-measure/School_Drop_Out_Rate-6-11_6-14_and_6-16_Years_Old_1.csv'\n",
    "data = pd.read_csv(temp)\n",
    "file_state_names = list(data['State/UT'])\n",
    "diff_list = list(set(all_states) - set(file_state_names))\n",
    "if(len(diff_list) > 0) :\n",
    "    print(path+' - ',diff_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "temp = 'new-data-set/school-type-comparison/primary-school-gross-enrollment-ratio.csv'\n",
    "data = pd.read_csv(temp)\n",
    "file_state_names = list(data['State/UT'])\n",
    "diff_list = list(set(all_states) - set(file_state_names))\n",
    "if(len(diff_list) > 0) :\n",
    "    print(path+' - ',diff_list)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "file_state_names = []\n",
    "for path in paths:\n",
    "    if(os.path.isfile(path)) :\n",
    "        data = pd.read_csv(path)\n",
    "        file_state_names = list(data['State/UT'])\n",
    "        diff_list = list(set(all_states) - set(file_state_names))\n",
    "        if(len(diff_list) > 0) :\n",
    "            print(path+' - ',diff_list)\n",
    "            \n",
    "# some files are listed \n",
    "# we added missing rows with NaN value \n",
    "# Hoooray - now all files have all state info (with NaN for missed ones)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# go through each file and see what is missing\n",
    "# add a row and update NaN values\n",
    "\n",
    "# Initally for a single file it is added and manually checked and so repeated for the rest\n",
    "file_state_names = []\n",
    "for path in paths:\n",
    "    if(os.path.isfile(path)) :\n",
    "        data = pd.read_csv(path)\n",
    "        file_state_names = list(data['State/UT'])\n",
    "        diff_list = list(set(all_states) - set(file_state_names))\n",
    "        if(len(diff_list) > 0) :\n",
    "            for state in diff_list:\n",
    "                idx = data.shape[0]\n",
    "                data.loc[idx] = np.NaN\n",
    "                data.loc[idx,'State/UT'] = state\n",
    "                #print(data.loc[idx])\n",
    "            data.to_csv(path,index=False)\n",
    "            #print(path)\n",
    "            #break\n",
    "#data['State/UT'][file_state_names]\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 127,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/kandy/anaconda3/lib/python3.5/site-packages/pandas/core/ops.py:792: FutureWarning: elementwise comparison failed; returning scalar instead, but in the future will perform elementwise comparison\n",
      "  result = getattr(x, name)(y)\n"
     ]
    },
    {
     "ename": "TypeError",
     "evalue": "invalid type comparison",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mTypeError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-127-a78f81187251>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      6\u001b[0m     \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mos\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0misfile\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m      7\u001b[0m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mpd\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mread_csv\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mpath\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 8\u001b[0;31m         \u001b[0mdata\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mdata\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0many\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;34m==\u001b[0m \u001b[0;34m'NA'\u001b[0m\u001b[0;34m]\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      9\u001b[0m         \u001b[0;32mif\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mshape\u001b[0m\u001b[0;34m[\u001b[0m\u001b[0;36m0\u001b[0m\u001b[0;34m]\u001b[0m \u001b[0;34m>\u001b[0m \u001b[0;36m0\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     10\u001b[0m             \u001b[0mprint\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mdata\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;32m/home/kandy/anaconda3/lib/python3.5/site-packages/pandas/core/ops.py\u001b[0m in \u001b[0;36mwrapper\u001b[0;34m(self, other, axis)\u001b[0m\n\u001b[1;32m    853\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    854\u001b[0m             \u001b[0;32mwith\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0merrstate\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mall\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;34m'ignore'\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 855\u001b[0;31m                 \u001b[0mres\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mna_op\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mvalues\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mother\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    856\u001b[0m             \u001b[0;32mif\u001b[0m \u001b[0misscalar\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mres\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    857\u001b[0m                 raise TypeError('Could not compare %s type with Series' %\n",
      "\u001b[0;32m/home/kandy/anaconda3/lib/python3.5/site-packages/pandas/core/ops.py\u001b[0m in \u001b[0;36mna_op\u001b[0;34m(x, y)\u001b[0m\n\u001b[1;32m    792\u001b[0m                     \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mgetattr\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mname\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    793\u001b[0m                 \u001b[0;32mif\u001b[0m \u001b[0mresult\u001b[0m \u001b[0;32mis\u001b[0m \u001b[0mNotImplemented\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m--> 794\u001b[0;31m                     \u001b[0;32mraise\u001b[0m \u001b[0mTypeError\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m\"invalid type comparison\"\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m    795\u001b[0m             \u001b[0;32mexcept\u001b[0m \u001b[0mAttributeError\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m    796\u001b[0m                 \u001b[0mresult\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0mop\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mx\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0my\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mTypeError\u001b[0m: invalid type comparison"
     ]
    }
   ],
   "source": [
    "'''\n",
    "# In some cases I saw 'NA' representing Null information\n",
    "# May be it is good to replace it with NaN\n",
    "# Let us do it\n",
    "\n",
    "for path in paths:\n",
    "    if(os.path.isfile(path)) :\n",
    "        data = pd.read_csv(path)\n",
    "        data = data[data.any() == 'NA']\n",
    "        if(data.shape[0] > 0):\n",
    "            print(data)\n",
    "            break\n",
    "        \n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "anaconda-cloud": {},
  "kernelspec": {
   "display_name": "Python [default]",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}