[init] Regex Support: Converter to EBNF (#21)

This PR provides a util to convert regex to EBNF. This is the first stage to support regex. Preliminary profiling results show XGrammar have good performance for regex:

- Model: Llama-3-8B (128k token)
- Preprocess time(ms):
  - IPv4: 203
  - Date-time: 187
  - Date: 85
  - Time: 124
  - Email: 156
- Max time for finding token mask (us):
  - IPv4: 626
  - Date-time, Date, Time: 52
  - Email: 1512

Further performance optimization is planned.

Author: Ubospica

.gitignore

@@ -65,6 +65,7 @@ coverage.xml
 *.py,cover
 .hypothesis/
 .pytest_cache/
+/Testing/
 
 # Translations
 *.mo
@@ -137,7 +138,7 @@ venv.bak/
 *.pyc
 *~
 config.mk
-config.cmake
+/config.cmake
 Win32
 *.dir
 perf

cpp/grammar_parser.cc

@@ -136,9 +136,7 @@ std::string EBNFParserImpl::ParseName(bool accept_empty) {
 // Character class should not contain newlines.
 int32_t EBNFParserImpl::ParseCharacterClass() {
   static constexpr TCodepoint kUnknownUpperBound = -4;
-  static const std::unordered_map<std::string, TCodepoint> kCustomEscapeMap = {
-      {"\\-", '-'}, {"\\]", ']'}
-  };
+  static const std::unordered_map<char, TCodepoint> CUSTOM_ESCAPE_MAP = {{'-', '-'}, {']', ']'}};
 
   std::vector<BNFGrammarBuilder::CharacterClassElement> elements;
 
@@ -160,14 +158,14 @@ int32_t EBNFParserImpl::ParseCharacterClass() {
       continue;
     }
 
-    auto [codepoint, new_cur] = ParseNextUTF8OrEscaped(cur_, kCustomEscapeMap);
+    auto [codepoint, len] = ParseNextUTF8OrEscaped(cur_, CUSTOM_ESCAPE_MAP);
     if (codepoint == CharHandlingError::kInvalidUTF8) {
       ThrowParseError("Invalid UTF8 sequence");
     }
     if (codepoint == CharHandlingError::kInvalidEscape) {
       ThrowParseError("Invalid escape sequence");
     }
-    Consume(new_cur - cur_);
+    Consume(len);
     if (past_is_hyphen) {
       XGRAMMAR_ICHECK(!elements.empty());
       if (elements.back().lower > codepoint) {
@@ -199,14 +197,14 @@ int32_t EBNFParserImpl::ParseString() {
       ThrowParseError("There should be no newline character in a string literal");
     }
 
-    auto [codepoint, new_cur] = ParseNextUTF8OrEscaped(cur_);
+    auto [codepoint, len] = ParseNextUTF8OrEscaped(cur_);
     if (codepoint == CharHandlingError::kInvalidUTF8) {
       ThrowParseError("Invalid utf8 sequence");
     }
     if (codepoint == CharHandlingError::kInvalidEscape) {
       ThrowParseError("Invalid escape sequence");
     }
-    Consume(new_cur - cur_);
+    Consume(len);
     codepoints.push_back(codepoint);
   }
   if (codepoints.empty()) {
@@ -221,7 +219,7 @@ int32_t EBNFParserImpl::ParseString() {
   // convert str to int32_t vector
   std::vector<int32_t> bytes;
   for (auto c : str) {
-    bytes.push_back(static_cast<int32_t>(c));
+    bytes.push_back(static_cast<int32_t>(static_cast<uint8_t>(c)));
   }
   return builder_.AddByteString(bytes);
 }

cpp/grammar_serializer.cc

@@ -55,7 +55,7 @@ std::string BNFGrammarPrinter::PrintByteString(const RuleExpr& rule_expr) {
   for (int i = 0; i < rule_expr.data_len; ++i) {
     internal_str += static_cast<char>(rule_expr[i]);
   }
-  auto codepoints = ParseUTF8(internal_str.c_str(), UTF8ErrorPolicy::kReturnByte);
+  auto codepoints = ParseUTF8(internal_str.c_str(), true);
   std::string result;
   for (auto codepoint : codepoints) {
     result += PrintAsEscapedUTF8(codepoint);

cpp/json_schema_converter.cc

@@ -20,15 +20,6 @@
 
 namespace xgrammar {
 
-// TODO(yixin): test it
-// EMCC somehow cannot pickup operator overload from picojson.h, so we copy here.
-// #ifdef COMPILE_MLC_WASM_RUNTIME
-// inline std::ostream& operator<<(std::ostream& os, const picojson::value& x) {
-//   x.serialize(std::ostream_iterator<char>(os));
-//   return os;
-// }
-// #endif
-
 /*!
  * \brief Manage the indent and separator for the generation of EBNF grammar.
  * \param indent The number of spaces for each indent. If it is std::nullopt, there will be no
@@ -621,7 +612,8 @@ std::string JSONSchemaToEBNFConverter::VisitEnum(
 
 std::string JSONSchemaToEBNFConverter::JSONStrToPrintableStr(const std::string& json_str) {
   static const std::vector<std::pair<std::string, std::string>> kReplaceMapping = {
-      {"\\", "\\\\"}, {"\"", "\\\""}};
+      {"\\", "\\\\"}, {"\"", "\\\""}
+  };
   std::string result = json_str;
   for (const auto& [k, v] : kReplaceMapping) {
     size_t pos = 0;

cpp/pybind/pybind.cc

@@ -23,7 +23,8 @@ PYBIND11_MODULE(xgrammar_bindings, m) {
   auto pyBuiltinGrammar = py::class_<BuiltinGrammar>(m, "BuiltinGrammar");
   pyBuiltinGrammar.def_static("json", &BuiltinGrammar::JSON)
       .def_static("json_schema", &BuiltinGrammar::JSONSchema)
-      .def_static("_json_schema_to_ebnf", &BuiltinGrammar::_JSONSchemaToEBNF);
+      .def_static("_json_schema_to_ebnf", &BuiltinGrammar::_JSONSchemaToEBNF)
+      .def_static("_regex_to_ebnf", &BuiltinGrammar::_RegexToEBNF);
 
   auto pyTokenizerInfo = py::class_<TokenizerInfo>(m, "TokenizerInfo");
   pyTokenizerInfo.def(py::init(&TokenizerInfo_Init))