Update instructions

README.md

@@ -66,7 +66,7 @@ Data scraped for each recipe includes:
 
 
 ## Run
-
+First run `pip install scrapy`
 ### Run the spider:
 
 - Set `DEBUG = False` in `recipescrape/spiders/recipes.py`
@@ -78,10 +78,14 @@ Data scraped for each recipe includes:
 - Run `scrapy crawl recipes -L WARN --logfile=scrapelog.txt -s JOBDIR=recipes/spider-1`
 
 ## Extras
-Spider to extract all the nutrient names:
+
+### Spider to extract all the nutrient names:
+
+- Run `pip install pandas`
 - Make sure `DEBUG = True` in `nutrient_list.py`
 - Run `scrapy crawl nutrients -o nutrients.csv -L WARN -s JOBDIR=nutrients/spider-1`
 - Outputs 'items_count', 'nutri_count', 'nutri_list', 'url' in `nutrients.csv` in which the last row contains the unified list of nutritients found so far.
 
+### Combine CSV files generated
 
-<!-- https://www.health.harvard.edu/staying-healthy/listing_of_vitamins -->
+- There are many ways to combine CSV files, a sample python file `/extras/combine_csv.py` is included for quick reference

extras/combine_csv.py

@@ -1,5 +1,9 @@
 import pandas as pd
 
+# just included for reference
+# this can be improved by using loops, reading the dir for CSV files.
+
+# read the created CSV files 
 df1 = pd.read_csv('1.csv')
 
 print(df1.head())
@@ -17,11 +21,13 @@
 
 print(f'total: {len(df1)+len(df2)+len(df3)}')
 
+# generate a single combined CSV
 df4 = pd.concat([df1, df2, df3])
 print(df4.head())
 combined_len = len(df4)
 print(combined_len)
 
+# drop duplicated entries if any
 df4 = df4.drop_duplicates()
 print(df4.head())
 unique_len = len(df4)
@@ -30,6 +36,7 @@
 
 df4.to_csv('combined.csv', index=False)
 
+# view the combined CSV with unique entries
 df5 = pd.read_csv('combined.csv')
 print(df5.head())
 print(len(df5))

recipescrape/spiders/recipes.py

@@ -4,7 +4,7 @@
 import re
 import logging
 
-DEBUG = True
+DEBUG = False
 FIELDS = [
     # Recipe and related information
     'name', 'url', 'category', 'author', 'summary', 'rating', 'rating_count', 'review_count', 'ingredients', 'directions', 'prep', 'cook', 'total', 'servings', 'yield',