Fixes o19s#4 - Enhancement/explicitize (o19s#37)

* Remove TMDB from top level rebuild

* add repr for Judgment

* Which tmdb json for movies

* make downloads explicit per directory

* Forgot download fixes

* Take file obj, not filename

* setup to explicit reset, create feature set

* Fix notebook judgment opening

* Missed a setup -> explicit call

* Fix netfix notebooks judg. access

* Stupid paren

* Fix feature creation on need a bigger bot

* change feature creation in netfix ES

* wrong featureset used

* filename to file obj

* correct feature store name

* Correct feature config name

* Download ranky before training

* Remove breakpoint

* Remove special download functions

Author: softwaredoug

ltr/__init__.py

@@ -3,5 +3,4 @@
 from .download import download
 from .evaluate import evaluate, rre_table
 from .train import train
-from .setup import setup
 from .search import search

ltr/client/base_client.py

@@ -35,7 +35,7 @@ def reset_ltr(self, index):
         pass
 
     @abstractmethod
-    def create_featureset(self, index, name, config):
+    def create_featureset(self, index, name, ftr_config):
         pass
 
     @abstractmethod

ltr/client/elastic_client.py

@@ -93,8 +93,8 @@ def reset_ltr(self, index):
         resp = requests.put(self.elastic_ep)
         resp_msg(msg="Initialize Default LTR feature store".format(), resp=resp)
 
-    def create_featureset(self, index, name, config):
-        resp = requests.post('{}/_featureset/{}'.format(self.elastic_ep, name), json=config)
+    def create_featureset(self, index, name, ftr_config):
+        resp = requests.post('{}/_featureset/{}'.format(self.elastic_ep, name), json=ftr_config)
         resp_msg(msg="Create {} feature set".format(name), resp=resp)
 
     def log_query(self, index, featureset, ids, params={}):

ltr/client/solr_client.py

@@ -90,10 +90,10 @@ def validate_featureset(self, name, config):
             if 'store' not in feature or feature['store'] != name:
                 raise ValueError("Feature {} needs to be created with \"store\": \"{}\" ".format(feature['name'], name))
 
-    def create_featureset(self, index, name, config):
-        self.validate_featureset(name, config)
+    def create_featureset(self, index, name, ftr_config):
+        self.validate_featureset(name, ftr_config)
         resp = requests.put('{}/{}/schema/feature-store'.format(
-            self.solr_base_ep, index, name), json=config)
+            self.solr_base_ep, index, name), json=ftr_config)
         resp_msg(msg='Created {} feature store under {}:'.format(name, index), resp=resp)
 
 

ltr/date_genre_judgments.py

@@ -98,8 +98,8 @@ def synthesize(client, judgmentsOutFile='genre_by_date_judgments.txt', autoNegat
                                     docId=movie['id'],
                                     keywords=negGenre)
                 judgments.append(judgment)
-
-    judgments_to_file(judgmentsOutFile, judgmentsList=judgments)
+    with open(judgmentsOutFile, 'w') as f:
+        judgments_to_file(f, judgmentsList=judgments)
 
     print('Done')
     return judgments

ltr/download.py

@@ -1,17 +1,22 @@
 import requests
 from os import path
 
-def download_uri(uri):
+def download_one(uri, dest='data/', force=False):
     import os
 
-    if not os.path.exists('data'):
-        os.makedirs('data')
+    if not os.path.exists(dest):
+        os.makedirs(dest)
+
+    if not os.path.isdir(dest):
+        raise ValueError("dest {} is not a directory".format(dest))
 
     filename = uri[uri.rfind('/') + 1:]
-    filepath = 'data/{}'.format(filename)
+    filepath = os.path.join(dest, filename)
     if path.exists(filepath):
-        print(filepath + ' already exists')
-        return
+        if not force:
+            print(filepath + ' already exists')
+            return
+        print("exists but force=True, Downloading anyway")
 
     with open(filepath, 'wb') as out:
         print('GET {}'.format(uri))
@@ -20,32 +25,6 @@ def download_uri(uri):
             if chunk:
                 out.write(chunk)
 
-
-def download():
-    resources = [
-        'http://es-learn-to-rank.labs.o19s.com/tmdb.json',
-        'http://es-learn-to-rank.labs.o19s.com/blog.jsonl',
-        'http://es-learn-to-rank.labs.o19s.com/osc_judgments.txt',
-        'http://es-learn-to-rank.labs.o19s.com/RankyMcRankFace.jar',
-        'http://es-learn-to-rank.labs.o19s.com/title_judgments.txt',
-        'http://es-learn-to-rank.labs.o19s.com/title_judgments_binary.txt',
-        'http://es-learn-to-rank.labs.o19s.com/genome_judgments.txt',
-        'http://es-learn-to-rank.labs.o19s.com/sample_judgments_train.txt'
-    ]
-
-    for uri in resources:
-        download_uri(uri)
-
-    print('Done.')
-
-def download_msmarco():
-    resources = [
-        'https://msmarco.blob.core.windows.net/msmarcoranking/msmarco-docs.tsv.gz',
-        'https://msmarco.blob.core.windows.net/msmarcoranking/msmarco-docs-lookup.tsv.gz',
-        'https://msmarco.blob.core.windows.net/msmarcoranking/msmarco-doctrain-qrels.tsv.gz',
-        'https://msmarco.blob.core.windows.net/msmarcoranking/msmarco-doctrain-queries.tsv.gz']
-    for uri in resources:
-        download_uri(uri)
-
-    print('Done.')
-
+def download(uris, dest='data/', force=False):
+    for uri in uris:
+        download_one(uri=uri, dest=dest, force=force)

ltr/helpers/movies.py

@@ -13,23 +13,22 @@ def __call__(self, *args):
         return self.memo[args]
 
 @Memoize
-def load_movies():
-    return json.load(open('data/tmdb.json'))
+def load_movies(json_path):
+    return json.load(open(json_path))
 
-def get_movie(tmdb_id):
-    movies = load_movies()
+def get_movie(tmdb_id, movies='data/tmdb.json'):
+    movies = load_movies(movies)
     tmdb_id=str(tmdb_id)
     return movies[tmdb_id]
 
-
 def noop(src_movie, base_doc):
     return base_doc
 
 
-def indexable_movies(enrich=noop):
+def indexable_movies(enrich=noop, movies='data/tmdb.json'):
     """ Generates TMDB movies, similar to how ES Bulk indexing
-        uses a generator to generate bulk index/update actions """
-    movies = load_movies()
+    uses a generator to generate bulk index/update actions"""
+    movies = load_movies(movies)
     idx = 0
     for movieId, tmdbMovie in movies.items():
         try:
@@ -61,5 +60,3 @@ def indexable_movies(enrich=noop):
             idx += 1
         except KeyError as k: # Ignore any movies missing these attributes
             continue
-
-

ltr/index.py

@@ -17,8 +17,3 @@ def rebuild(client, index, doc_src):
                            doc_src=doc_src)
 
     print('Done')
-
-
-def rebuild_tmdb(client, enrich=noop):
-    movies=indexable_movies(enrich=enrich)
-    rebuild(client, index='tmdb', doc_src=movies)

ltr/injectTypos.py

@@ -8,7 +8,8 @@
 
 
 def typoIt(judgmentInFile, judgmentOutFile, rounds=100):
-    currJudgments = [judg for judg in judgments_from_file(judgmentInFile)]
+    with open(judgmentInFile) as f:
+        currJudgments = [judg for judg in judgments_from_file(f)]
     lastQid = currJudgments[-1].qid
     judgDict = judgments_by_qid(currJudgments)
 
@@ -32,7 +33,8 @@ def typoIt(judgmentInFile, judgmentOutFile, rounds=100):
                     currJudgments.append(typoJudg)
                 existingTypos.add(keywordsWTypo)
 
-    judgments_to_file(filename=judgmentOutFile, judgmentsList=currJudgments)
+    with open(judgmentOutFile, 'w') as f:
+        judgments_to_file(f, judgmentsList=currJudgments)
 
 
 if __name__ == "__main__":

ltr/judgments.py

@@ -18,6 +18,9 @@ def has_features(self):
     def __str__(self):
         return "grade:%s qid:%s (%s) docid:%s" % (self.grade, self.qid, self.keywords, self.docId)
 
+    def __repr__(self):
+        return "Judgment(grade={grade},qid={qid},keywords={keywords},docId={docId},features={features},weight={weight}".format(**vars(self))
+
     def toRanklibFormat(self):
         featuresAsStrs = ["%s:%s" % (idx+1, feature) for idx, feature in enumerate(self.features)]
         comment = "# %s\t%s" % (self.docId, self.keywords)
@@ -110,30 +113,33 @@ def _judgmentsFromBody(lines):
             #print("Not Recognized as Judgment %s" % line)
 
 
-def judgments_from_file(filename):
-    with open(filename) as f:
-        qidToKeywords = _queriesFromHeader(f)
-    with open(filename) as f:
-        lastQid = -1
-        for grade, qid, docId, features in _judgmentsFromBody(f):
-            if lastQid != qid and qid % 100 == 0:
-                print("Parsing QID %s" % qid)
-            yield Judgment(grade=grade, qid=qid,
-                           keywords=qidToKeywords[qid][0],
-                           weight=qidToKeywords[qid][1],
-                           docId=docId,
-                           features=features)
-            lastQid = qid
-
-
-def judgments_to_file(filename, judgmentsList):
+def judgments_from_file(f):
+    """ Read judgments from a SVMRank File
+        f is a file object
+    """
+    qidToKeywords = _queriesFromHeader(f)
+    lastQid = -1
+    for grade, qid, docId, features in _judgmentsFromBody(f):
+        if lastQid != qid and qid % 100 == 0:
+            print("Parsing QID %s" % qid)
+        yield Judgment(grade=grade, qid=qid,
+                       keywords=qidToKeywords[qid][0],
+                       weight=qidToKeywords[qid][1],
+                       docId=docId,
+                       features=features)
+        lastQid = qid
+
+
+def judgments_to_file(f, judgmentsList):
+    """ Write judgments from a SVMRank File
+        f is a file object
+    """
     judgToQid = judgments_by_qid(judgmentsList) #Pretty hideosly slow stuff
     fileHeader = _queriesToHeader({qid: (judgs[0].keywords, judgs[0].weight) for qid, judgs in judgToQid.items()})
     judgByQid = sorted(judgmentsList, key=lambda j: j.qid)
-    with open(filename, 'w+') as f:
-        f.write(fileHeader)
-        for judg in judgByQid:
-            f.write(judg.toRanklibFormat() + '\n')
+    f.write(fileHeader)
+    for judg in judgByQid:
+        f.write(judg.toRanklibFormat() + '\n')
 
 
 

ltr/log.py

@@ -57,8 +57,10 @@ def log_features(client, index, judgments_by_qid, featureSet):
 def judgments_to_training_set(client, judgmentInFile, featureSet, trainingOutFile='judgments_wfeatures.txt', index='tmdb'):
     from .judgments import judgments_to_file, judgments_from_file, judgments_by_qid
 
-    judgments = judgments_from_file(judgmentInFile)
-    judgments = judgments_by_qid(judgments)
+    judgments = []
+    with open(judgmentInFile) as f:
+        judgments = judgments_from_file(f)
+        judgments = judgments_by_qid(judgments)
     log_features(client, index, judgments, featureSet=featureSet)
 
     judgmentsAsList = []
@@ -71,5 +73,6 @@ def judgments_to_training_set(client, judgmentInFile, featureSet, trainingOutFil
                 discarded.append(judgment)
     print("Discarded %s Keep %s" % (len(discarded), len(judgmentsAsList)))
 
-    judgments_to_file(filename=trainingOutFile, judgmentsList=judgmentsAsList)
+    with open(trainingOutFile, 'w+') as f:
+        judgments_to_file(f, judgmentsList=judgmentsAsList)
     return judgments

ltr/setup.py

@@ -1,5 +1,14 @@
 import os
 from ltr.helpers.ranklib_result import parse_training_log
+from ltr import download
+
+def check_for_rankymcrankface():
+    """ Ensure ranky jar is in a temp dir somewhere..."""
+    ranky_url='http://es-learn-to-rank.labs.o19s.com/RankyMcRankFace.jar'
+    import tempfile
+    tempdir = tempfile.gettempdir()
+    download([ranky_url], dest=tempdir, force=False)
+    return os.path.join(tempdir, 'RankyMcRankFace.jar')
 
 
 def trainModel(training, out, features=None, kcv=None, ranker=6,
@@ -15,8 +24,9 @@ def trainModel(training, out, features=None, kcv=None, ranker=6,
         srate - what proportion of the queries should be examined for each ensemble
     """
 
-    cmd = 'java -jar data/RankyMcRankFace.jar -ranker {} -shrinkage {} -metric2t {} -tree {} -bag {} -leaf {} -frate {} -srate {} -train {} -save {} '.format(
-            ranker, shrinkage, metric2t, trees, bag, leafs, frate, srate, training, out)
+    ranky_loc = check_for_rankymcrankface()
+    cmd = 'java -jar {} -ranker {} -shrinkage {} -metric2t {} -tree {} -bag {} -leaf {} -frate {} -srate {} -train {} -save {} '.format(
+            ranky_loc, ranker, shrinkage, metric2t, trees, bag, leafs, frate, srate, training, out)
 
     if features is not None:
         with open('data/features.txt', 'w') as f:

ltr/train.py

@@ -14,7 +14,10 @@
    "outputs": [],
    "source": [
     "from ltr import download\n",
-    "download();"
+    "corpus='http://es-learn-to-rank.labs.o19s.com/blog.jsonl'\n",
+    "judgments='http://es-learn-to-rank.labs.o19s.com/osc_judgments.txt'\n",
+    "\n",
+    "download([corpus, judgments], dest='data/');"
    ]
   },
   {
@@ -89,6 +92,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "client.reset_ltr(index='tmdb')\n",
+    "\n",
     "config = {\n",
     "    \"featureset\": {\n",
     "        \"features\": [\n",
@@ -226,9 +231,7 @@
     "   }\n",
     "}\n",
     "\n",
-    "\n",
-    "from ltr import setup\n",
-    "setup(client, config=config, index='blog', featureset='test')"
+    "client.create_featureset(index='blog', name='test', ftr_config=config)"
    ]
   },
   {

notebooks/elasticsearch/osc-blog/osc-blog.ipynb

@@ -17,7 +17,10 @@
    "outputs": [],
    "source": [
     "from ltr import download\n",
-    "download();"
+    "corpus='http://es-learn-to-rank.labs.o19s.com/tmdb.json'\n",
+    "judgments='http://es-learn-to-rank.labs.o19s.com/title_judgments.txt'\n",
+    "\n",
+    "download([corpus, judgments], dest='data/');"
    ]
   },
   {
@@ -26,8 +29,11 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "from ltr.index import rebuild_tmdb\n",
-    "rebuild_tmdb(client)"
+    "from ltr.index import rebuild\n",
+    "from ltr.helpers.movies import indexable_movies\n",
+    "\n",
+    "movies=indexable_movies(movies='data/tmdb.json')\n",
+    "rebuild(client, index='tmdb', doc_src=movies)"
    ]
   },
   {
@@ -36,6 +42,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
+    "client.reset_ltr(index='tmdb')\n",
+    "\n",
     "config = {\"validation\": {\n",
     "              \"index\": \"tmdb\",\n",
     "              \"params\": {\n",
@@ -63,10 +71,7 @@
     "    }}\n",
     "\n",
     "\n",
-    "\n",
-    "\n",
-    "from ltr import setup\n",
-    "setup(client, config=config, index='tmdb', featureset='movies')"
+    "client.create_featureset(index='tmdb', name='movies', ftr_config=config)"
    ]
   },
   {

notebooks/elasticsearch/tmdb/es-ltr-basics-project.ipynb

@@ -91,7 +91,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.6.5"
+   "version": "3.6.1"
   }
  },
  "nbformat": 4,