Trying to make pulling indices

Author: mojavad

app.py

@@ -25,11 +25,6 @@
 ids = load('models/ids.joblib')
 qa_model = QA('models')
 
-@strawberry.type
-class User:
-    name: str
-    age: int
-
 
 @strawberry.type
 class Document:
@@ -54,6 +49,17 @@ class QAResult:
     answer: str
     confidence : float
 
+
+@strawberry.type
+class MetaData:
+    tf_idf_len_diff: int
+    bert_len_diff: int
+
+@strawberry.type
+class IndexingResult:
+    status: str
+    metadata: MetaData
+
 @strawberry.type
 class Query:
     @strawberry.field
@@ -80,4 +86,27 @@ def qa(self, q: str) -> QAResult:
         answer = qa_model.predict(" ".join(reference),q)
         return QAResult(answer = answer['answer'], confidence = answer['confidence'], )
 
+    @strawberry.field
+    def pull_updates_from_index_cloud(self) -> IndexingResult:
+        global tfidf_faiss, bert_faiss, ids, qa_model
+        tf_idf_prev_len = tfidf_faiss.ntotal
+        bert_prev_len = bert_faiss.ntotal
+        
+        print("Previous TFIDF length: {}".format(tf_idf_prev_len))
+        pull_indices(True)
+        
+        tfidf_faiss,  bert_faiss = load_faiss(tfidf_model, bert_model)
+        ids = load('models/ids.joblib')
+        qa_model = QA('models')
+
+        metadata = {'tfidf_len_diff': tfidf_faiss.ntotal - tf_idf_prev_len, 'bert_len_diff': bert_faiss.ntotal - bert_prev_len}
+        return IndexingResult(status = "Success", metadata = metadata)
+
+    # @strawberry.field
+    # def reindex(self) -> IndexingResult:
+    #     D, I = vector_search(q, bert_model, bert_faiss)
+    #     reference = [x["description"] for x in collection.find({'_id': {'$in': (np.array(ids)[I[0][:2]]).tolist()}})]
+    #     answer = qa_model.predict(" ".join(reference),q)
+    #     return QAResult(answer = answer['answer'], confidence = answer['confidence'], )
+
 schema = strawberry.Schema(query=Query)

build.py

@@ -135,7 +135,8 @@ def load_bert_model():
 def build_faiss(tfidf_model, bert_model):
     tr = tracker.SummaryTracker()
     print(f"Building indices ...")
-    c = collection.find().count()
+    # c = collection.find().count()
+    c = 1000
     batch_size = 500
     encoder = None
     bert_index =  None
@@ -182,7 +183,7 @@ def build_faiss(tfidf_model, bert_model):
     faiss.write_index(tfidf_index,f"models/tfidf.index")
     dump(ids,'models/ids.joblib')
     print(f"Completed indices.")
-    upload_indices_and_vectors()
+    # upload_indices_and_vectors()
     return [tfidf_index, bert_index]
 
 def load_faiss(tfidf_model, bert_model):

cloud_storage.py

@@ -76,9 +76,9 @@ def download_blob(bucket_name, source_blob_name, destination_file_name):
         )
     )
 
-def pull_indices():
+def pull_indices(run_manually=False):
     # Checks if PULL_INDS environment variable is present, and calls pull function
-    if os.environ.get('PULL_INDS') != None:
+    if run_manually or os.environ.get('PULL_INDS') != None:
         download_blob("symptomizer_indices_bucket-1", "tfidf.index", "models/tfidf.index")
         download_blob("symptomizer_indices_bucket-1", "bert.index", "models/bert.index")
         download_blob("symptomizer_indices_bucket-1", "ids.joblib", "models/ids.joblib")

test.txt

@@ -0,0 +1 @@
+bye fsh