Remove legacy workflow (#33)

Author: latekvo

core/chainables/web.py

@@ -1,25 +1,7 @@
 import datetime
-
-from core.tools.scraper import web_query_google_lookup
-from core.classes.query import WebQuery
 from langchain_core.prompts import ChatPromptTemplate
 
 
-def web_news_lookup(prompt_text: str):
-    query = WebQuery("news", prompt_core=prompt_text)
-    return web_query_google_lookup(query)
-
-
-def web_wiki_lookup(prompt_text: str):
-    query = WebQuery("wiki", prompt_core=prompt_text)
-    return web_query_google_lookup(query)
-
-
-def web_docs_lookup(prompt_text: str):
-    query = WebQuery("docs", prompt_core=prompt_text)
-    return web_query_google_lookup(query)
-
-
 def web_docs_lookup_prompt():
     return ChatPromptTemplate.from_messages(
         [

core/classes/query.py

@@ -1,6 +1,5 @@
 from __future__ import annotations
 
-import datetime
 from typing import Literal
 
 from core.tools import utils

core/lookup.py

@@ -1,18 +1,14 @@
 import requests.exceptions
 import tiktoken
 from googlesearch import search
-from langchain.text_splitter import RecursiveCharacterTextSplitter
 from langchain_community.document_loaders import WebBaseLoader
-from langchain_community.vectorstores.faiss import FAISS
 from langchain_core.documents import Document
 from langchain_core.output_parsers import StrOutputParser
 from colorama import Fore, Style
 from core.tools.model_loader import load_model
 from core.models.configurations import load_llm_config
 from core.tools.utils import purify_name
-from core.tools.dbops import get_vec_db_by_name
 from core.classes.query import WebQuery
-from core.tools.utils import is_text_junk, remove_characters
 
 encoder = tiktoken.get_encoding("cl100k_base")
 output_parser = StrOutputParser()
@@ -40,10 +36,6 @@ def docs_to_context(docs_and_scores: list[Document], token_limit: int) -> str:
     return context_text
 
 
-def rag_query_lookup(prompt_text: str) -> str:
-    pass
-
-
 def query_for_urls(query: WebQuery, url_amount=embed_config.article_limit) -> list[str]:
     print(f"{Fore.CYAN}{Style.BRIGHT}Searching for:{Style.RESET_ALL}", query.web_query)
 
@@ -67,59 +59,3 @@ def download_article(url):
     except requests.exceptions.ConnectionError:
         return None
     return document
-
-
-def populate_db_with_google_search(database: FAISS, query: WebQuery):
-    url_list = query_for_urls(query)
-
-    for url in url_list:
-        document = download_article(url)
-
-        text_splitter = RecursiveCharacterTextSplitter(
-            separators=embed_config.buffer_stops,
-            chunk_size=query.db_chunk_size,
-            chunk_overlap=embed_config.chunk_overlap,
-            keep_separator=False,
-            strip_whitespace=True,
-        )
-
-        chunks = text_splitter.split_documents(document)
-
-        for chunk in chunks:
-            if is_text_junk(chunk.page_content):
-                chunks.remove(chunk)
-                continue
-
-            chunk.page_content = remove_characters(chunk.page_content, ["\n", "`"])
-            chunk.page_content = (
-                query.db_embedding_prefix
-                + chunk.page_content
-                + query.db_embedding_postfix
-            )
-
-        if len(chunks) != 0:
-            database.add_documents(documents=chunks, embeddings=embeddings)
-
-    db_name = embedding_model_safe_name + query.db_save_file_extension
-    database.save_local(folder_path="store/vector", index_name=db_name)
-
-    print(f"{Fore.CYAN}Document vectorization completed.{Fore.RESET}")
-
-
-def web_query_google_lookup(
-    query: WebQuery, token_limit: int = embed_config.model_token_limit
-):
-    db_name = embedding_model_safe_name + query.db_save_file_extension
-    db = get_vec_db_by_name(db_name, embeddings)
-
-    populate_db_with_google_search(db, query)
-
-    # return the document with the highest prompt similarity score (for now only browsing the first search result)
-    embedding_vector = embeddings.embed_query(query.db_embed_query)
-    docs_and_scores = db.similarity_search_by_vector(
-        embedding_vector, k=round(token_limit / 64)
-    )
-
-    print(f"{Fore.CYAN}Database search completed.{Fore.RESET}")
-
-    return docs_to_context(docs_and_scores, llm_config.model_token_limit)

core/tools/scraper.py

@@ -56,35 +56,6 @@ def remove_characters(text: str, wordlist: list[str]):
     return text
 
 
-def timeout_function(task, timeout=2.0):
-    # FIXME: THIS FUNCTION MAY BE BROKEN, TEST THIS
-
-    ctx = multiprocessing.get_context("spawn")
-    q = ctx.Queue()
-
-    def wrapper(q):
-        task_result = task()
-        q.put(task_result)
-
-    thread_loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(thread_loop)
-
-    thread = ctx.Process(target=wrapper, args=(q,))
-
-    thread.start()
-    thread.join(timeout)  # close thread if work is finished
-    if thread.is_alive():
-        thread.kill()
-        return None
-
-    result = q.get()
-
-    thread_loop.run_until_complete(asyncio.sleep(0))
-    thread_loop.close()
-
-    return result
-
-
 def extract_links(text: str):
     return re.findall(r"(https?://\S+\.\S+/)", text)
 

core/tools/utils.py

@@ -1,32 +1,12 @@
-import curses
 import requests
 
 from colorama import init as colorama_init, Fore, Style
-from terminal_gui import user_input, select_input
-from core.lookup import web_lookup
 
 
 colorama_init()
 
 try:
-    try:
-        mode_input = curses.wrapper(select_input)
-        text_input = curses.wrapper(user_input)
-        print(f"{Fore.GREEN}{Style.BRIGHT}Mode: {Fore.RESET}{mode_input}")
-        print(f"{Fore.GREEN}{Style.BRIGHT}Input: {Fore.RESET}{text_input}")
-    except curses.error:
-        # terminal is not present,
-        # user likely tries running through IDE
-        print(
-            f"{Fore.YELLOW}Terminal not detected, full functionality may not be available.{Fore.RESET}"
-        )
-        mode_input = "Wiki"
-        text_input = input(f"{Fore.GREEN}{Style.BRIGHT}(user){Fore.RESET} ")
-
-    chain_output = web_lookup.invoke({"input": text_input, "mode": mode_input})
-    print(f"{Fore.GREEN}{Style.BRIGHT}(llm){Fore.RESET} ", end="")
-    print(chain_output, end="", flush=True)
-    print(end="\n")
+    pass
 except requests.exceptions.ConnectionError:
     print(
         f"{Fore.RED}{Style.BRIGHT}Connection error, make sure Ollama server is running...{Fore.RESET}{Style.RESET_ALL}"

main.py

@@ -1,4 +1,3 @@
-import argparse
 import curses
 
 HIGHLIGHTED_COLOR_ID = 1

terminal_gui.py

@@ -1,5 +1,4 @@
 from core.databases.db_crawl_tasks import (
-    db_are_tasks_completed,
     db_are_crawl_tasks_fully_embedded,
 )
 from core.databases.db_embeddings import (