fix: replace bligfire with nltk sent_tokenize

see: microsoft/BlingFire#171
wandb · Sep 27, 2024 · 1ddf621 · 1ddf621
1 parent 2f3e7ba
commit 1ddf621
Show file tree

Hide file tree

Showing 2 changed files with 2 additions and 3 deletions.
diff --git a/rag-advanced/notebooks/scripts/chunking.py b/rag-advanced/notebooks/scripts/chunking.py
@@ -6,7 +6,7 @@
 from typing import Callable, List, Optional
 
 import numpy as np
-from blingfire import text_to_sentences
+from nltk import sent_tokenize
 from sklearn.metrics.pairwise import cosine_distances
 from tqdm.notebook import tqdm
 
@@ -26,7 +26,7 @@ def sentence_splitter(text: str) -> List[str]:
     Returns:
         List[str]: A list of sentences.
     """
-    return text_to_sentences(text).split("\n")
+    return sent_tokenize(text)
 
 
 def split_into_chunks(

diff --git a/rag-advanced/requirements.txt b/rag-advanced/requirements.txt
@@ -1,7 +1,6 @@
 weave>=0.51.2
 cohere>=5.9.4
 beautifulsoup4>=4.12.3
-blingfire>=0.1.8
 levenshtein>=0.25.1
 markdown-it-py>=3.0.0
 nltk>=3.8.1