fix(language-server): corrent handling of multi-word terms with unicode characters (fixes #90)

Author: chrissimon-au

src/core/Contextive.Core/CandidateTerms.fs

@@ -24,7 +24,9 @@ let private (|Regex|_|) pattern input =
     | _ -> res |> Some
 
 let private Default: CandidateTerms = Seq.empty<CandidateTerm>
-let private TokenSplitterRegex = "([A-Z]+(?![a-z])|[A-Z][a-z]+|[0-9]+|[a-z]+)"
+
+let private TokenSplitterRegex =
+    "([A-Z\\p{Lu}]+(?![a-z\\p{Ll}])|[A-Z\\p{Lu}][a-z\\p{Ll}]+|[0-9]+|[a-z\\p{Ll}]+)"
 
 let candidateTermsFromToken =
     function

src/language-server/Contextive.LanguageServer.Tests/E2e/CompletionTests.fs

@@ -130,58 +130,70 @@ let tests =
 
                   test <@ (completionLabels, expectedCompletionLabels) ||> Seq.compareWith compare = 0 @>
 
-          [ ("Multi Word",
-             Some "",
-             seq {
-                 "multiWord"
-                 "MultiWord"
-                 "multi_word"
-                 "multi-word"
-             })
-            ("Multi Word",
-             Some "m",
-             seq {
-                 "multiWord"
-                 "multi_word"
-             })
-            ("Multi Word",
-             Some "M",
-             seq {
-                 "MultiWord"
-                 "MULTI_WORD"
-             })
-            ("Multi Word",
-             Some "MU",
-             seq {
-                 "MULTIWORD"
-                 "MULTI_WORD"
-             })
-            ("multi word",
-             Some "",
-             seq {
-                 "multiWord"
-                 "MultiWord"
-                 "multi_word"
-                 "multi-word"
-             })
-            ("multi word",
-             Some "m",
-             seq {
-                 "multiWord"
-                 "multi_word"
-             })
-            ("multi word",
-             Some "M",
-             seq {
-                 "MultiWord"
-                 "MULTI_WORD"
-             })
-            ("multi word",
-             Some "MU",
-             seq {
-                 "MULTIWORD"
-                 "MULTI_WORD"
-             }) ]
+          [ "Multi Word",
+            Some "",
+            seq {
+                "multiWord"
+                "MultiWord"
+                "multi_word"
+                "multi-word"
+            }
+            "Multi Word",
+            Some "m",
+            seq {
+                "multiWord"
+                "multi_word"
+            }
+            "Multi Word",
+            Some "M",
+            seq {
+                "MultiWord"
+                "MULTI_WORD"
+            }
+            "Multi Word",
+            Some "MU",
+            seq {
+                "MULTIWORD"
+                "MULTI_WORD"
+            }
+            "multi word",
+            Some "",
+            seq {
+                "multiWord"
+                "MultiWord"
+                "multi_word"
+                "multi-word"
+            }
+            "multi word",
+            Some "m",
+            seq {
+                "multiWord"
+                "multi_word"
+            }
+            "multi word",
+            Some "M",
+            seq {
+                "MultiWord"
+                "MULTI_WORD"
+            }
+            "multi word",
+            Some "MU",
+            seq {
+                "MULTIWORD"
+                "MULTI_WORD"
+            }
+            "Один Два",
+            Some "О",
+            seq {
+                "ОдинДва"
+                "ОДИН_ДВА"
+            }
+            "Один Два",
+            Some "о",
+            seq {
+                "одинДва"
+                "один_два"
+            } ]
           |> List.map multiWordCompletion
           |> testList "Multi Word Completion"
 

src/vscode/contextive/test/single-root/fixtures/simple_workspace/UnicodeDemo/test.txt

@@ -0,0 +1,3 @@
+РУС_ENG
+один_два
+eng_рус

src/vscode/contextive/test/single-root/fixtures/simple_workspace/UnicodeDemo/unicode.glossary.yml

@@ -0,0 +1,5 @@
+contexts:
+  - terms:
+    - name: Один Два
+    - name: Рус Eng
+    - name: Eng Рус