Comments and more comments.

Author: Apaezmx

.gitignore

@@ -37,7 +37,3 @@
 ehthumbs.db
 Thumbs.db
 *.*~
-
-# Data folders #
-################
-air/models

air/api.py

@@ -8,21 +8,33 @@
 LOSS_LENGTH = 0
 
 def make_register():
+  """ Creates a dict registry object to hold all annotated definitions.
+  Returns: A dictionary with all definitions in this file.
+  """
   registry = {}
   def registrar(func):
       registry[func.__name__] = func
       return func
   registrar.all = registry
   return registrar
+  
+# Dictionary holding all definitions in this file.
 endpoint = make_register()
 
 def handleNaN(val):
+  """ Turns all NaN values to 0.
+  Returns: 0 if val == NaN, otherwise the value.
+  """
   if math.isnan(val):
     return 0
   return val
 
 @endpoint
 def infer_types(args, files):
+  """ Given a model handle, returns a dictionary of column-name to type JSON.
+  Args: Model handle
+  Returns: a JSON holding the column-name to type map.
+  """
   try:
     model = get_model(args['handle'])
   except Exception as e:
@@ -31,6 +43,10 @@ def infer_types(args, files):
 
 @endpoint
 def infer(args, files):
+  """ Given a model handle and input values, this def runs the model inference graph and returns the predictions.
+  Args: Model handle, input values.
+  Returns: A JSON containing all the model predictions.
+  """
   #clear_session()  # Clears TF graphs.
   try:
     model = get_model(args['handle'])
@@ -45,6 +61,10 @@ def infer(args, files):
 
 @endpoint
 def train(args, files):
+  """ Runs the training for the given model.
+  Args: Model handle.
+  Returns: A JSON confirming that training has been kicked-off.
+  """
   clear_session()  # Clears TF graphs.
   clear_thread_cache()  # We need to clear keras models since graph is deleted.
   try:
@@ -56,6 +76,10 @@ def train(args, files):
 
 @endpoint
 def train_status(args, files):
+  """ Grabs the metrics from disk and returns them for the given model handle.
+  Args: Model handle.
+  Returns: A JSON with a dictionary of keras_model_name -> metric_name -> list(metric values)
+  """
   try:
     model = get_model(args['handle'])
   except:
@@ -73,6 +97,14 @@ def train_status(args, files):
 
 @endpoint
 def upload_csv(args, files):
+  """ Takes in a csv and creates a Model around it.
+  CSV needs to have a feature per column. Also needs to have at least one column marked as output by prepending 
+  'output_' to the column name (first row in file). Types will be conservatively infered from the input (ie type will be
+  string as long as one cell contains a non-numeric character).
+  
+  Files: Path to tmp CSV file on server (handled by the framework).
+  Returns: A JSON with the model handle just created, and the infered feature types.
+  """
   if 'upload' not in files:
       print 'Files not specified in upload: ' + files
       return 'No file specified'
@@ -94,8 +126,13 @@ def upload_csv(args, files):
 
   return json.dumps({'status': 'OK', 'handle': model.get_handle(), 'types': model.types})
 
-# Calls endpoint with a map of arguments.
 def resolve_endpoint(endpoint_str, args, files):
+  """ Reroutes the request to the matching endpoint definition. 
+  See make_registrar for more information.
+  Params: arguments and files needed to run the endpoint (every endpoint receives both dictionaries). Also receives the
+  name of the endpoint.
+  Returns: The output of the endpoint.
+  """
   if endpoint_str in endpoint.all:
       return endpoint.all[endpoint_str](args, files)
   else:

air/config.py

@@ -1,9 +1,11 @@
 import os
 
 class Config():
-  mc = {}
-  ROOT_PATH = os.path.dirname(os.path.realpath(__file__))
-  STATIC_PATH = os.path.dirname(os.path.realpath(__file__)) + '/static'
+  """ Static global class holding important parameters.
+  """
+  mc = {}  # Memchached object shared across threads.
+  ROOT_PATH = os.path.dirname(os.path.realpath(__file__))  # Path to the project's directory.
+  STATIC_PATH = os.path.dirname(os.path.realpath(__file__)) + '/static'  # Path to static resources.
 
   def set_mc(self, mc):
     self.mc = mc
@@ -12,4 +14,4 @@ def get_mc(self):
     return self.mc
 
 global config
-config = Config()
+config = Config()  # Singleton class object to be used across the project.

air/db.py

@@ -12,18 +12,23 @@
 
 MODEL_PATH = '/models'
 HANDLE_LENGTH = 10
-keras_cache = {}
+keras_cache = {}  # Local thread memory cache.
 
 def clear_thread_cache():
   keras_cache = {}
 
 def handle2path(handle):
+  """ Translates a handle to a full path on the FS. """
   return config.ROOT_PATH + MODEL_PATH + "/" + handle
 
 def path2handle(path):
+  """ Translates a full path to a handle. """
   return path.split('/')[-1]
 
 def new_model():
+  """ Construcs an empty Model assiging it a new random hex ID and persiting it to disk. 
+  Returns: The Model instance.
+  """
   filename = random_hex()
   while os.path.isfile(filename):
     filename = random_hex()
@@ -38,24 +43,34 @@ def new_model():
   return model
 
 def save_model(model):
+  """ Saves the given model to disk. """
   with open(model.model_path, 'w+') as f:
     f.write(model.to_json())
     config.get_mc().set(path2handle(model.model_path), model.to_json())
 
 def get_model(handle):
+  """ Fetches the model from memory or disk with a matching handle.
+  Returns: The Model instance if the model is found, None otherwise.
+  """
   mem_try = config.get_mc().get(handle)
   if mem_try:
     m = Model()
     m.from_json(mem_try)
     return m
   model_path = config.ROOT_PATH + MODEL_PATH + "/" + handle
-  with open(model_path, "r") as f:
-    model = Model()
-    model.from_json(f.read())
-  config.get_mc().set(handle, model.to_json())
-  return model
+  try:
+    with open(model_path, "r") as f:
+      model = Model()
+      model.from_json(f.read())
+      config.get_mc().set(handle, model.to_json())
+    return model
+  except:
+    return None
 
 def parse_val(value):
+  """ Infers the type of the value by trying to parse it to different formats.
+  Returns: The parse value and the type.
+  """
   if not value:
     return value, None
   tests = [
@@ -83,6 +98,8 @@ def parse_val(value):
   return value.decode('utf-8', 'ignore'), 'str'
 
 def persist_keras_model(handle, model):
+  """ Persists a keras model to disk.
+  """
   model_dir = config.ROOT_PATH + MODEL_PATH
 
   # Clear first all previously persisted models.
@@ -94,6 +111,9 @@ def persist_keras_model(handle, model):
   model.save(os.path.join(model_dir, name))
 
 def _load_keras_model(handle):
+  """ Loads a keras model from disk.
+  Returns: The keras model instance if found, None otherwise.
+  """
   name = handle + '_keras'
   print 'load ' + name
   model_dir = config.ROOT_PATH + MODEL_PATH
@@ -106,6 +126,9 @@ def _load_keras_model(handle):
       return model
 
 def load_keras_model(handle):
+  """ Loads a keras model from cache or disk.
+  Returns: The keras model instance.
+  """
   if handle in keras_cache:
     print 'From thread cache'
     return keras_cache[handle]
@@ -114,12 +137,20 @@ def load_keras_model(handle):
   return model
 
 def delete_model(handle):
+  """ Deletes all models with the given handle if found. """
   model_dir = config.ROOT_PATH + MODEL_PATH
   for f in os.listdir(model_dir):
     if re.search(handle + ".*", f):
         os.remove(os.path.join(model_dir, f))
+        config.get_mc().delete(handle)
+        
 
 def load_csvs(file_list):  
+  """ Loads csv from files and returns the parsed value dictionary.
+  Params: The list of files.
+  Returns: Three dictionaries. The first is feature-name -> value_list, the second one feature_name -> type and the
+  third one feature_name -> [min_value, max_value] if applies.
+  """
   print 'File of csvs to load ' + unicode(file_list)
   data = {}
   types = {}
@@ -129,7 +160,7 @@ def load_csvs(file_list):
         reader = csv.reader(read_f)
         headers = []
         for row in reader:
-          if not headers:
+          if not headers:  # If first row, load the headers assuming they are contained in the first row.
             headers = row
             output_headers = 0
             for h in headers:
@@ -138,7 +169,7 @@ def load_csvs(file_list):
               output_headers += 1 if h.startswith('output_') else 0
             if not output_headers:
               return 'No outputs defined in CSV. Please define columns as outputs by preppending \'output_\'.', ''
-          else:
+          else:  # If not first row, parse values assuming the headers dictionary has been already filled.
             for idx, value in enumerate(row):
               val, typ = parse_val(value)
               data[headers[idx]].append(val)
@@ -147,22 +178,25 @@ def load_csvs(file_list):
                 types[headers[idx]] = typ
     else:
       print 'WARN: CSV %s not found' % f
+    
     # Fix '' values, and standardize formats.
     for header, column in data.iteritems():
       for idx, value in enumerate(column):
         if not value:
           data[header][idx] = 0 if types[header] != 'str' else ''
         else:
           data[header][idx] = unicode(data[header][idx]) if types[header] == 'str' else data[header][idx]
+    
     # Normalize numeric inputs to -1 to 1.
     norms = {}
-    
     for header, column in data.iteritems():
       if types[header] != 'str':
         floor = float(min(column))
         ceil = float(max(column))
         norms[header] = (floor, ceil)
         data[header] = [(x-floor)/(ceil - floor) for x in column]
+    
+    # Run some last verifications so that all features have the same amount of rows.
     length = 0
     for header, column in data.iteritems():
       if not length:
@@ -173,6 +207,7 @@ def load_csvs(file_list):
     return data, types, norms
 
 def random_hex():
+  """ Creates a random hex string ID """
   ran = random.randrange(16**HANDLE_LENGTH)
   return "%010x" % ran
 

air/keras_utils.py

@@ -4,6 +4,7 @@
 import tensorflow as tf
 
 def activate(activation, tensor):
+  """ Maps a string activation to the keras backend function """
   if activation == 'tanh':
     return K.tanh(tensor)
   elif activation == 'sigmoid':