fixing scrape failures

Author: talos

notice-of-property-value-pdf/scrape.js

@@ -4,7 +4,6 @@ var textract = require('textract');
 var _ = require('underscore');
 var walk = require('walk');
 var path = require('path');
-//var fs = require('fs');
 var incomeTest1 =
   /(\s*)Gross Income:(\s*)We estimated gross income at \$(.*)\./;
 var incomeTest2 = /Estimated Gross Income:(\s*)\$(.*)/;
@@ -62,19 +61,11 @@ walkOptions = {
       });
     },
     directories: function (root, dirStatsArray, next) {
-      // dirStatsArray is an array of `stat` objects with the additional
-      // attributes
-      // * type
-      // * error
-      // * name
-
       next();
     },
     file: function (root, fileStats, next) {
-      //fs.readFile(fileStats.name, function () {
-      //});
       if (fileStats.type === 'file' &&
-          fileStats.name.match(/Notice of Property Value\.pdf/)) {
+          fileStats.name.match(/Notice of Property Value\.pdf/i)) {
         var fullPath = path.join(root, fileStats.name);
         textract('application/pdf', fullPath, {
           preserveLineBreaks:true
@@ -93,37 +84,10 @@ walkOptions = {
       }
     },
     errors: function (root, nodeStatsArray, next) {
+      console.error('walk error');
       next();
     }
   }
 };
 
-//_.each(process.argv, function (path, i) {
-//  if (i < 2) {
-//    return;
-//  }
-//  var absPath = __dirname + "/" + path;
-//
-//  var synchronize = setInterval(function () {
-//    if (lock === true) {
-//      return;
-//    } else {
-//      lock = true;
-//    }
-//    textract('application/pdf', absPath, {
-//      preserveLineBreaks:true
-//    }, function(error, text) {
-//      if (error) {
-//        console.error("Could not read '" + absPath + "': " + error);
-//      } else {
-//        console.log(
-//          _.values(_.pick(parse(path, text.split('\n')), headers)).join(','));
-//      }
-//      lock = false;
-//    });
-//    clearInterval(synchronize);
-//  }, 10);
-//});
-
 walk.walk(process.argv[2], walkOptions);
-//walk.walkSync(process.argv[2], walkOptions);

quarterly-property-tax-bills-pdf/run.js

@@ -22,51 +22,34 @@ var headers = [
 
 function makeCallback(next) {
   return function(taxDoc) {
-    if (!taxDoc) {
-      next();
-      return;
-    }
-    taxDoc.activityThrough = new Date(
-      taxDoc.activityThrough).toISOString().split('T')[0];
-    taxDoc.annualPropertyTax = taxDoc.annualPropertyTax ? Number(
-      taxDoc.annualPropertyTax.replace(/[$,]/g, '')) : taxDoc.annualPropertyTax;
-    taxDoc.billableAssessedValue = taxDoc.billableAssessedValue ? Number(
-      taxDoc.billableAssessedValue.replace(/[$,]/g, '')) :
-      taxDoc.billableAssessedValue;
-    taxDoc.taxRate = taxDoc.taxRate ?
-      Number(taxDoc.taxRate.replace(/[%]/g, '')) :
-      taxDoc.taxRate;
-    _.each(taxDoc, function (v, k) {
-      if (typeof v === 'string') {
-        if (v.search(',') !== -1) {
-          v = v.replace(/'"'/g, '\\"');
-          taxDoc[k] = '"' + v + '"';
+    if (taxDoc) {
+      taxDoc.activityThrough = new Date(
+        taxDoc.activityThrough).toISOString().split('T')[0];
+      taxDoc.annualPropertyTax = taxDoc.annualPropertyTax ? Number(
+        taxDoc.annualPropertyTax.replace(/[$,]/g, '')) :
+        taxDoc.annualPropertyTax;
+      taxDoc.billableAssessedValue = taxDoc.billableAssessedValue ? Number(
+        taxDoc.billableAssessedValue.replace(/[$,]/g, '')) :
+        taxDoc.billableAssessedValue;
+      taxDoc.taxRate = taxDoc.taxRate ?
+        Number(taxDoc.taxRate.replace(/[%]/g, '')) :
+        taxDoc.taxRate;
+      _.each(taxDoc, function (v, k) {
+        if (typeof v === 'string') {
+          if (v.search(',') !== -1) {
+            v = v.replace(/'"'/g, '\\"');
+            taxDoc[k] = '"' + v + '"';
+          }
         }
-      }
-    });
-    console.log(_.values(_.pick(taxDoc, headers)).join(','));
+      });
+      console.log(_.values(_.pick(taxDoc, headers)).join(','));
+    }
     next();
   };
 }
 
 console.log(headers.join(','));
 
-//_.each(process.argv, function (path, i) {
-//  if (i < 2) {
-//    return;
-//  }
-//  var synchronize = setInterval(function () {
-//    if (lock === true) {
-//      return;
-//    } else {
-//      lock = true;
-//    }
-//
-//    scraper(path, callback);
-//    clearInterval(synchronize);
-//  }, 10);
-//});
-
 walkOptions = {
   listeners: {
     names: function (root, nodeNamesArray) {

quarterly-property-tax-bills-pdf/scrape.js

@@ -9,17 +9,17 @@ var scrape = function (filepath, callback) {
     if (err) {
       console.error(err);
       console.error('error with: ' + filepath);
+      callback();
     } else {
       try {
         var taxDoc = parse_pdf(text.split(" "));
         callback(taxDoc);
-        return;
       } catch (err) {
         console.error(err);
         console.error('error with: ' + filepath);
+        callback();
       }
     }
-    callback();
   })
 }