Skip to content

Commit 7fcdfd8

Browse files
UbospicaUbospica
authored
[init] Enhance cmakelists and pybind (#3)
1 parent ba96e17 commit 7fcdfd8

12 files changed

+469
-27
lines changed

‎.cmake-format.yaml

+7
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
format:
2+
line_width: 100
3+
tab_size: 2
4+
dangle_parens: true
5+
command_case: lower
6+
keyword_case: upper
7+
autosort: true

‎3rdparty/picojson/picojson.h

+1-1
Original file line numberDiff line numberDiff line change
@@ -492,7 +492,7 @@ GET(array, *u_.array_)
492492
GET(object, *u_.object_)
493493
#ifdef PICOJSON_USE_INT64
494494
GET(double, (type_ == int64_type && (const_cast<value*>(this)->type_ = number_type,
495-
const_cast<value*>(this)->u_.number_ = u_.int64_),
495+
(const_cast<value*>(this)->u_.number_ = u_.int64_)),
496496
u_.number_))
497497
GET(int64_t, u_.int64_)
498498
#else

‎CMakeLists.txt

+22-21
Original file line numberDiff line numberDiff line change
@@ -1,43 +1,44 @@
11
cmake_minimum_required(VERSION 3.15)
22
project(xgrammar LANGUAGES CXX)
33

4+
OPTION(BUILD_BINDINGS "Build Python bindings" ON)
5+
46
set(CMAKE_CXX_STANDARD 17)
57
set(CMAKE_CXX_STANDARD_REQUIRED ON)
68
set(CMAKE_POSITION_INDEPENDENT_CODE ON)
79

810
if(MSVC)
9-
set(CMAKE_CXX_FLAGS "/Wall /WX ${CMAKE_CXX_FLAGS}")
11+
set(CMAKE_CXX_FLAGS "/Wall /WX ${CMAKE_CXX_FLAGS}")
1012
else()
11-
set(CMAKE_CXX_FLAGS "-Wall -Wextra -Werror -pedantic -Wno-unused-parameter ${CMAKE_CXX_FLAGS}")
13+
set(CMAKE_CXX_FLAGS "-Wall -Wextra -Werror -pedantic -Wno-unused-parameter ${CMAKE_CXX_FLAGS}")
1214
endif()
1315

1416
if(NOT CMAKE_BUILD_TYPE)
15-
message(STATUS "No build type specified; defaulting to CMAKE_BUILD_TYPE=Debug.")
16-
set(CMAKE_BUILD_TYPE "Debug" CACHE STRING "The build type" FORCE)
17+
message(STATUS "No build type specified; defaulting to CMAKE_BUILD_TYPE=Debug.")
18+
set(CMAKE_BUILD_TYPE
19+
"Debug"
20+
CACHE STRING "The build type" FORCE
21+
)
1722
endif()
1823

19-
# Set pybind11
20-
find_package(Python COMPONENTS Interpreter Development REQUIRED)
21-
find_package(pybind11 CONFIG REQUIRED)
22-
23-
set(XGRAMMAR_INCLUDE
24-
${PROJECT_SOURCE_DIR}/3rdparty/picojson
25-
${PROJECT_SOURCE_DIR}/3rdparty/dlpack/include
26-
${PROJECT_SOURCE_DIR}/include
24+
set(XGRAMMAR_INCLUDE_PATH ${PROJECT_SOURCE_DIR}/3rdparty/picojson
25+
${PROJECT_SOURCE_DIR}/3rdparty/dlpack/include ${PROJECT_SOURCE_DIR}/include
2726
)
2827

2928
if(CMAKE_BUILD_TYPE STREQUAL "Debug")
30-
set(XGRAMMAR_COMPILE_DEFINITIONS XGRAMMAR_ENABLE_LOG_DEBUG=1)
29+
set(XGRAMMAR_COMPILE_DEFINITIONS XGRAMMAR_ENABLE_LOG_DEBUG=1)
3130
else()
32-
set(XGRAMMAR_COMPILE_DEFINITIONS XGRAMMAR_ENABLE_LOG_DEBUG=0)
31+
set(XGRAMMAR_COMPILE_DEFINITIONS XGRAMMAR_ENABLE_LOG_DEBUG=0)
3332
endif()
3433

35-
file(GLOB_RECURSE SOURCES "${PROJECT_SOURCE_DIR}/cpp/*.cc")
36-
add_library(xgrammar_objs OBJECT ${SOURCES})
37-
target_include_directories(xgrammar_objs PUBLIC ${XGRAMMAR_INCLUDE})
38-
target_compile_definitions(xgrammar_objs PUBLIC ${XGRAMMAR_COMPILE_DEFINITIONS})
34+
file(GLOB_RECURSE XGRAMMAR_SOURCES_PATH "${PROJECT_SOURCE_DIR}/cpp/*.cc")
35+
list(FILTER XGRAMMAR_SOURCES_PATH EXCLUDE REGEX "${PROJECT_SOURCE_DIR}/cpp/pybind/.*\\.cc")
3936

40-
add_library(xgrammar_lib $<TARGET_OBJECTS:xgrammar_objs>)
37+
add_library(xgrammar SHARED ${XGRAMMAR_SOURCES_PATH})
38+
target_include_directories(xgrammar PUBLIC ${XGRAMMAR_INCLUDE_PATH})
39+
target_compile_definitions(xgrammar PUBLIC ${XGRAMMAR_COMPILE_DEFINITIONS})
4140

42-
pybind11_add_module(xgrammar_bindings $<TARGET_OBJECTS:xgrammar_objs>)
43-
install(TARGETS xgrammar_bindings DESTINATION .)
41+
if(BUILD_BINDINGS)
42+
message(STATUS "XGrammar Python bindings will be built")
43+
add_subdirectory(${PROJECT_SOURCE_DIR}/cpp/pybind)
44+
endif()

‎cpp/json_schema_converter.cc

+1
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ class IndentManager {
6565
* indent_manager.GetSep(); // get the start separator: "\"\n \""
6666
* indent_manager.GetSep(); // get the middle separator: "\",\n \""
6767
* indent_manager.GetSep(true); // get the end separator: "\"\n\""
68+
* indent_manager.EndIndent();
6869
* \endcode
6970
*/
7071
std::string NextSeparator(bool is_end = false);

‎cpp/pybind/CMakeLists.txt

+23
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
# Set Pybind11
2+
find_package(
3+
Python3
4+
COMPONENTS Interpreter Development
5+
REQUIRED
6+
)
7+
execute_process(
8+
COMMAND ${Python3_EXECUTABLE} "-c" "import pybind11 as pb11; print(pb11.get_cmake_dir(),end='');"
9+
RESULT_VARIABLE PYBIND_CMAKE_DIR_RET
10+
OUTPUT_VARIABLE PYBIND_CMAKE_DIR
11+
)
12+
if(PYBIND_CMAKE_DIR_RET MATCHES 0)
13+
list(APPEND CMAKE_PREFIX_PATH "${PYBIND_CMAKE_DIR}")
14+
else()
15+
message(FATAL_ERROR "pybind11 is not installed. Please install pybind11 with pip or conda first")
16+
endif()
17+
find_package(pybind11 CONFIG REQUIRED)
18+
19+
file(GLOB_RECURSE XGRAMMAR_BINDINGS_PATH ${PROJECT_SOURCE_DIR}/cpp/pybind/*.cc)
20+
pybind11_add_module(xgrammar_bindings ${XGRAMMAR_BINDINGS_PATH})
21+
target_include_directories(xgrammar_bindings PRIVATE ${XGRAMMAR_INCLUDE_PATH})
22+
target_compile_definitions(xgrammar_bindings PRIVATE ${XGRAMMAR_COMPILE_DEFINITIONS})
23+
target_link_libraries(xgrammar_bindings PRIVATE xgrammar)

‎cpp/pybind/pybind.cc

+177
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,177 @@
1+
/*!
2+
* Copyright (c) 2024 by Contributors
3+
* \file xgrammar/pybind.cc
4+
*/
5+
6+
#include <pybind11/pybind11.h>
7+
#include <xgrammar/grammar.h>
8+
#include <xgrammar/json_schema_converter.h>
9+
10+
namespace py = pybind11;
11+
using namespace xgrammar;
12+
13+
// PYBIND11_MODULE(xgrammar_bindings, m) {
14+
// auto pyBNFGrammar = py::class_<BNFGrammar>(m, "BNFGrammar");
15+
// pyBNFGrammar.def_static("from_ebnf_string", &BNFGrammar::FromEBNFString)
16+
// .def("to_string", &BNFGrammar::ToString)
17+
18+
// .def_static("from_json", &BNFGrammar::FromJSON)
19+
// .def_static("from_schema", &BNFGrammar::FromSchema)
20+
// .def_static("get_grammar_of_json", &BNFGrammar::GetGrammarOfJSON);
21+
// }
22+
23+
// namespace xgrammar {
24+
25+
// std::ostream& operator<<(std::ostream& os, const BNFGrammar& grammar) {
26+
// os << BNFGrammarPrinter(grammar).ToString();
27+
// return os;
28+
// }
29+
30+
// BNFGrammar BNFGrammar::FromEBNFString(
31+
// const std::string& ebnf_string, const std::string& main_rule
32+
// ) {
33+
// auto grammar = EBNFParser::Parse(ebnf_string, main_rule);
34+
// // Normalize the grammar by default
35+
// grammar = BNFGrammarNormalizer().Apply(grammar);
36+
// return grammar;
37+
// }
38+
39+
// // TVM_REGISTER_GLOBAL("mlc.grammar.BNFGrammarFromEBNFString")
40+
// // .set_body_typed([](String ebnf_string, String main_rule) {
41+
// // return BNFGrammar::FromEBNFString(ebnf_string, main_rule);
42+
// // });
43+
44+
// // Parse the EBNF string but not normalize it
45+
// BNFGrammar DebugFromEBNFStringNoNormalize(
46+
// const std::string& ebnf_string, const std::string& main_rule
47+
// ) {
48+
// return EBNFParser::Parse(ebnf_string, main_rule);
49+
// }
50+
51+
// // TVM_REGISTER_GLOBAL("mlc.grammar.BNFGrammarDebugFromEBNFStringNoNormalize")
52+
// // .set_body_typed([](String ebnf_string, String main_rule) {
53+
// // return DebugFromEBNFStringNoNormalize(ebnf_string, main_rule);
54+
// // });
55+
56+
// BNFGrammar BNFGrammar::FromSchema(
57+
// const std::string& schema,
58+
// std::optional<int> indent,
59+
// std::optional<std::pair<std::string, std::string>> separators,
60+
// bool strict_mode
61+
// ) {
62+
// return FromEBNFString(JSONSchemaToEBNF(schema, indent, separators, strict_mode));
63+
// }
64+
65+
// // TVM_REGISTER_GLOBAL("mlc.grammar.BNFGrammarFromSchema").set_body([](TVMArgs args, TVMRetValue*
66+
// // rv) {
67+
// // std::optional<int> indent;
68+
// // if (args[1].type_code() != kTVMNullptr) {
69+
// // indent = args[1];
70+
// // } else {
71+
// // indent = std::nullopt;
72+
// // }
73+
74+
// // std::optional<std::pair<std::string, std::string>> separators;
75+
// // if (args[2].type_code() != kTVMNullptr) {
76+
// // Array<String> separators_arr = args[2];
77+
// // XGRAMMAR_CHECK(separators_arr.size() == 2);
78+
// // separators = std::make_pair(separators_arr[0], separators_arr[1]);
79+
// // } else {
80+
// // separators = std::nullopt;
81+
// // }
82+
83+
// // *rv = BNFGrammar::FromSchema(args[0], indent, separators, args[3]);
84+
// // });
85+
86+
// // Optimized json grammar for the speed of the grammar state matcher
87+
// const std::string kJSONGrammarString = R"(
88+
// main ::= (
89+
// "{" [ \n\t]* members_and_embrace |
90+
// "[" [ \n\t]* elements_or_embrace
91+
// )
92+
// value_non_str ::= (
93+
// "{" [ \n\t]* members_and_embrace |
94+
// "[" [ \n\t]* elements_or_embrace |
95+
// "0" fraction exponent |
96+
// [1-9] [0-9]* fraction exponent |
97+
// "-" [0-9] fraction exponent |
98+
// "-" [1-9] [0-9]* fraction exponent |
99+
// "true" |
100+
// "false" |
101+
// "null"
102+
// ) (= [ \n\t,}\]])
103+
// members_and_embrace ::= ("\"" characters_and_colon [ \n\t]* members_suffix | "}") (= [ \n\t,}\]])
104+
// members_suffix ::= (
105+
// value_non_str [ \n\t]* member_suffix_suffix |
106+
// "\"" characters_and_embrace |
107+
// "\"" characters_and_comma [ \n\t]* "\"" characters_and_colon [ \n\t]* members_suffix
108+
// ) (= [ \n\t,}\]])
109+
// member_suffix_suffix ::= (
110+
// "}" |
111+
// "," [ \n\t]* "\"" characters_and_colon [ \n\t]* members_suffix
112+
// ) (= [ \n\t,}\]])
113+
// elements_or_embrace ::= (
114+
// "{" [ \n\t]* members_and_embrace elements_rest [ \n\t]* "]" |
115+
// "[" [ \n\t]* elements_or_embrace elements_rest [ \n\t]* "]" |
116+
// "\"" characters_item elements_rest [ \n\t]* "]" |
117+
// "0" fraction exponent elements_rest [ \n\t]* "]" |
118+
// [1-9] [0-9]* fraction exponent elements_rest [ \n\t]* "]" |
119+
// "-" "0" fraction exponent elements_rest [ \n\t]* "]" |
120+
// "-" [1-9] [0-9]* fraction exponent elements_rest [ \n\t]* "]" |
121+
// "true" elements_rest [ \n\t]* "]" |
122+
// "false" elements_rest [ \n\t]* "]" |
123+
// "null" elements_rest [ \n\t]* "]" |
124+
// "]"
125+
// )
126+
// elements ::= (
127+
// "{" [ \n\t]* members_and_embrace elements_rest |
128+
// "[" [ \n\t]* elements_or_embrace elements_rest |
129+
// "\"" characters_item elements_rest |
130+
// "0" fraction exponent elements_rest |
131+
// [1-9] [0-9]* fraction exponent elements_rest |
132+
// "-" [0-9] fraction exponent elements_rest |
133+
// "-" [1-9] [0-9]* fraction exponent elements_rest |
134+
// "true" elements_rest |
135+
// "false" elements_rest |
136+
// "null" elements_rest
137+
// )
138+
// elements_rest ::= (
139+
// "" |
140+
// [ \n\t]* "," [ \n\t]* elements
141+
// )
142+
// characters_and_colon ::= (
143+
// "\"" [ \n\t]* ":" |
144+
// [^"\\\x00-\x1F] characters_and_colon |
145+
// "\\" escape characters_and_colon
146+
// ) (=[ \n\t]* [\"{[0-9tfn-])
147+
// characters_and_comma ::= (
148+
// "\"" [ \n\t]* "," |
149+
// [^"\\\x00-\x1F] characters_and_comma |
150+
// "\\" escape characters_and_comma
151+
// ) (=[ \n\t]* "\"")
152+
// characters_and_embrace ::= (
153+
// "\"" [ \n\t]* "}" |
154+
// [^"\\\x00-\x1F] characters_and_embrace |
155+
// "\\" escape characters_and_embrace
156+
// ) (=[ \n\t]* [},])
157+
// characters_item ::= (
158+
// "\"" |
159+
// [^"\\\x00-\x1F] characters_item |
160+
// "\\" escape characters_item
161+
// ) (= [ \n\t]* [,\]])
162+
// escape ::= ["\\/bfnrt] | "u" [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9] [A-Fa-f0-9]
163+
// fraction ::= "" | "." [0-9] [0-9]*
164+
// exponent ::= "" | "e" sign [0-9] [0-9]* | "E" sign [0-9] [0-9]*
165+
// sign ::= "" | "+" | "-"
166+
// )";
167+
168+
// BNFGrammar BNFGrammar::GetGrammarOfJSON() {
169+
// static const BNFGrammar grammar = BNFGrammar::FromEBNFString(kJSONGrammarString, "main");
170+
// return grammar;
171+
// }
172+
173+
// // TVM_REGISTER_GLOBAL("mlc.grammar.BNFGrammarGetGrammarOfJSON").set_body_typed([]() {
174+
// // return BNFGrammar::GetGrammarOfJSON();
175+
// // });
176+
177+
// } // namespace xgrammar

‎pyproject.toml

+1
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
requires = [
1919
"setuptools>=42",
2020
"pybind11>=2.10.0",
21+
"ninja",
2122
]
2223
build-backend = "setuptools.build_meta"
2324

‎python/pyproject.toml

+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing,
12+
# software distributed under the License is distributed on an
13+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
# KIND, either express or implied. See the License for the
15+
# specific language governing permissions and limitations
16+
# under the License.
17+
[build-system]
18+
requires = [
19+
"setuptools>=42",
20+
"pybind11>=2.10.0",
21+
"ninja",
22+
]

‎python/requirements.txt

+3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
ninja==1.11.1.1
2+
setuptools==68.2.2
3+
transformers==4.42.3

0 commit comments

Comments
 (0)