feat: simdjson and PythonModuleLoader refactor

Author: jahan-addison

README.md

@@ -2,10 +2,13 @@
 
 ### B Compiler to z80 in C++
 
-* The frontend is built with an [easy-to-read LALR(1) grammar and parser generator in python](https://github.com/jahan-addison/xion/tree/master), that interfaces with C++ via CPython
+* The frontend (Lexer, Parser) and first-pass is built with an [easy-to-use LALR(1) grammar and parser generator in python](https://github.com/jahan-addison/xion/tree/master), that interfaces with C++ via libpython or json via `simdjson`
 * The backend will focus on modern work in SSA, Sea of Nodes, and compiler optimizations through IR breakthroughs in LLVM, gcc, V8, and related toolchains
 
 
+<img src="docs/images/roxas-2.png" width="800px" alt="sunil sapkota twitter" > </img>
+
+
 ### Installation
 
 * Install git submodules
@@ -38,19 +41,13 @@ make
 
 * [Simple and Efficient Construction of Static Single
 Assignment Form](https://c9x.me/compile/bib/braun13cc.pdf)
+* [Engineering a compiler](https://shop.elsevier.com/books/engineering-a-compiler/cooper/978-0-12-815412-0)
 
 Check the [project board](https://github.com/users/jahan-addison/projects/3/views/1) for development and feature progress.
 
-<table border="0">
-	<td width="350px">
-		<b>Apache License 2.0</b>
-		<br>Grammar under CC0 1.0<br>
-		<i>roxas</i>
-	</td>
-	<td border="0"><img src="docs/images/roxas.jpg" width="400px" alt="sunil sapkota twitter" > </img></td>
-</table>
+### License
 
+Apache 2 License.
 
-#### Special Thanks
 
-* [Engineering a compiler](https://shop.elsevier.com/books/engineering-a-compiler/cooper/978-0-12-815412-0)
+![img2](docs/roxas-xion-axel.jpg)

cppcheck.suppress

@@ -19,5 +19,6 @@ unusedFunction
 // libpython uses this a lot, unfortunately
 cstyleCast
 
-// Suppress python directories
-*:*python*
+// ignore dependencies
+*:*python*
+*:*_deps*

docs/images/roxas-2.png

@@ -16,9 +16,19 @@
 
 #include <cstdlib>
 #include <iostream>
-#include <roxas/ast.h>
 #include <roxas/python_module.h>
+#include <roxas/util.h>
 
+/**
+ * @brief
+ *  Arguments
+ *  Loads a Parser via a python module frontend for the first pass.
+ *
+ *  1) Python module importable name
+ *  2) Directory to python module
+ *  3) Path to B program
+ *  4) Optional path to a python venv
+ */
 int main(int argc, const char* argv[])
 {
     if (argc < 4) {
@@ -32,15 +42,15 @@ int main(int argc, const char* argv[])
         return EXIT_FAILURE;
     }
     try {
-        // Dependency venv Example:
-        // ~/.cache/pypoetry/virtualenvs/xion-I1_4RUhc-py3.11/lib/python3.11/site-packages
-        auto parse_tree =
-            argc == 5 ? roxas::ParseTreeModuleLoader(
-                            argv[2], argv[1], argv[3], argv[4])
-                      : roxas::ParseTreeModuleLoader(argv[2], argv[1], argv[3]);
-        std::cout << parse_tree.call_method_on_module(
-                         "get_source_program_ast_as_string")
+        auto source = roxas::util::read_file_from_path(argv[3]);
+        auto python_module =
+            argc == 5 ? roxas::PythonModuleLoader(argv[2], argv[1], argv[4])
+                      : roxas::PythonModuleLoader(argv[2], argv[1]);
+
+        std::cout << python_module.call_method_on_module(
+                         "get_source_program_ast_as_json", { source })
                   << std::endl;
+
     } catch (std::runtime_error& e) {
         std::cerr << "Runtime Exception :: " << e.what() << std::endl;
         return EXIT_FAILURE;

docs/images/roxas-xion-axel.jpg

@@ -14,8 +14,6 @@
  * limitations under the License.
  */
 
-#include <filesystem>
-#include <fstream>
 #include <roxas/python_module.h>
 #include <sstream>
 #include <stdexcept>
@@ -24,25 +22,6 @@
 
 namespace roxas {
 
-namespace fs = std::filesystem;
-
-/**
- * @brief read source file
- *
- * @param path path to file
- * @return std::string
- */
-std::string read_source_file(fs::path path)
-{
-    std::ifstream f(path, std::ios::in | std::ios::binary);
-    const auto sz = fs::file_size(path);
-
-    std::string result(sz, '\0');
-    f.read(result.data(), sz);
-
-    return result;
-}
-
 /**
  * @brief PythonModuleLoader constructor
  *
@@ -83,14 +62,15 @@ PythonModuleLoader::PythonModuleLoader(std::string_view module_path,
  * @param args initializer list of arguments to pass to the python method
  * @return std::string result of method call
  */
-std::string call_method_on_module(std::string_view method_name,
-                                  std::initializer_list<std::string> args);
+std::string PythonModuleLoader::call_method_on_module(
+    std::string_view method_name,
+    std::initializer_list<std::string> args)
 {
     std::string ret{};
     PyObject *pModule, *pDict, *pFunc, *vModule;
 
     // Load the module object
-    pModule = PyImport_ImportModule(module_name_.c_str());
+    pModule = PyImport_ImportModule(module_name_.data());
 
     if (pModule == NULL) {
         throw std::runtime_error("memory error: failed to allocate pModule");
@@ -114,7 +94,7 @@ std::string call_method_on_module(std::string_view method_name,
     }
 
     // pFunc is also a borrowed reference
-    pFunc = PyDict_GetItemString(pDict, method_name.c_str());
+    pFunc = PyDict_GetItemString(pDict, method_name.data());
 
     if (pFunc == NULL) {
         Py_DECREF(pModule);
@@ -124,16 +104,17 @@ std::string call_method_on_module(std::string_view method_name,
 
     if (PyCallable_Check(pFunc)) {
         PyObject *pValue, *pArgs;
-        int pIndex = 0;
+        int index = 0;
         pArgs = PyTuple_New(args.size());
         // construct the arguments from the initializer list `args'
-        for (std::string arg const& : args) {
+        for (std::string const& arg : args) {
             if (arg == "true") {
-                PyTuple_SetItem(pArgs, pIndex, Py_True);
+                PyTuple_SetItem(pArgs, index, Py_True);
             } else if (arg == "false") {
-                PyTuple_SetItem(pArgs, pIndex, Py_False);
+                PyTuple_SetItem(pArgs, index, Py_False);
             } else {
-                PyTuple_SetItem(pArgs, index, PyUnicode_FromString(arg));
+                PyTuple_SetItem(
+                    pArgs, index, PyUnicode_FromString(arg.c_str()));
             }
             index++;
         }

external/xion

@@ -0,0 +1,43 @@
+/*
+ * Copyright (c) Jahan Addison
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <fstream>
+#include <roxas/util.h>
+
+namespace roxas {
+
+namespace util {
+
+/**
+ * @brief read source file
+ *
+ * @param path path to file
+ * @return std::string
+ */
+std::string read_file_from_path(fs::path path)
+{
+    std::ifstream f(path, std::ios::in | std::ios::binary);
+    const auto sz = fs::file_size(path);
+
+    std::string result(sz, '\0');
+    f.read(result.data(), sz);
+
+    return result;
+}
+
+} // namespace util
+
+} // namespace roxas

roxas/main.cc

@@ -0,0 +1,37 @@
+/*
+ * Copyright (c) Jahan Addison
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#pragma once
+
+#include <filesystem>
+
+namespace roxas {
+
+namespace util {
+
+namespace fs = std::filesystem;
+
+/**
+ * @brief read a file from a fs::path
+ *
+ * @param path path to file
+ * @return std::string
+ */
+std::string read_file_from_path(fs::path path);
+
+} // namespace util
+
+} // namespace roxas