diff --git a/.gitignore b/.gitignore index 9f5ce17..0f77964 100644 --- a/.gitignore +++ b/.gitignore @@ -33,3 +33,4 @@ .idea cmake-build-* +llvm-project diff --git a/CMakeLists.txt b/CMakeLists.txt index dbb617f..f4f0c79 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1,7 +1,18 @@ cmake_minimum_required(VERSION 3.12) project(fuzz VERSION 0.1) -add_library(fuzz src/test.cpp src/test.h src/test_main.h src/test_main.cpp) +list(APPEND CMAKE_MODULE_PATH + "${CMAKE_CURRENT_SOURCE_DIR}/llvm-project/llvm/cmake/modules" + "${CMAKE_CURRENT_SOURCE_DIR}/llvm-project/clang/cmake/modules") + +if(CMAKE_PROJECT_NAME STREQUAL PROJECT_NAME) + set(LLVM_TARGETS_TO_BUILD "X86") +endif() + +include(AddLLVM) +include(AddClang) + +add_library(fuzz src/test.cpp src/test.h src/test_main.h src/test_main.cpp src/printer.cpp src/subprocess.cpp src/subprocess.h) add_library(fuzz::fuzz ALIAS fuzz) target_include_directories(fuzz PUBLIC src/) @@ -9,7 +20,6 @@ target_compile_features(fuzz PUBLIC cxx_std_17) add_subdirectory(examples) - set(LLVM_LINK_COMPONENTS support) set(CMAKE_CXX_STANDARD 17) @@ -26,4 +36,4 @@ target_link_libraries(test-builder clangASTMatchers ) -include_directories(src) \ No newline at end of file +include_directories(src) diff --git a/TestBuilder.cpp b/TestBuilder.cpp index d01689c..9593bbf 100644 --- a/TestBuilder.cpp +++ b/TestBuilder.cpp @@ -112,28 +112,11 @@ class FunctionTestBuilder : public MatchFinder::MatchCallback } }; -std::vector tests; -void construct_tests() -{ - std::random_device rd; - - std::mt19937 generator(rd()); - - - for(const auto &i: test_signatures) - { - tests.emplace_back(Test::generate(generator, i.name, i)); - } -} - - -[[nodiscard]] auto generate_tests_and_signatures(auto Tool) +[[nodiscard]] std::vector generate_test_signatures(auto Tool) { DeclarationMatcher Matcher = functionDecl().bind("FunctionDecl"); - - // FunctionArgsPrinter Printer; FunctionTestBuilder Builder; MatchFinder Finder; @@ -142,9 +125,7 @@ void construct_tests() assert(! Tool.run(newFrontendActionFactory(&Finder).get())); - construct_tests(); - - return std::pair{test_signatures, tests}; + return test_signatures; } } // namespace GenerateTest @@ -160,25 +141,13 @@ int main(int argc, char **argv) OptionsParser.getSourcePathList()); - auto [s, t] = GenerateTest::generate_tests_and_signatures(Tool); - - auto tst = t.at(0); - auto sign = s.at(0); - - // std::cout << tst.print() << std::endl << std::endl; + auto signatures = GenerateTest::generate_test_signatures(Tool); TestMain test_main(1, argv); - test_main.fuzz(sign, 5).run(); - - - // // test_main.add(tst); - // for(auto tst: t) - // test_main.add(tst); - - // test_main.run(); - - + for (auto sign : signatures) { + test_main.fuzz(std::make_shared(sign), 5).run(); + } return 0; -} \ No newline at end of file +} diff --git a/examples/fib_generator.cpp b/examples/fib_generator.cpp index f63cc08..3db9dae 100644 --- a/examples/fib_generator.cpp +++ b/examples/fib_generator.cpp @@ -1,12 +1,12 @@ #include "test_main.h" int main(int argc, char **argv) { - auto n = primitiveType(PrimitiveType::INT, 0, 10); + auto n = primitiveType(PrimitiveType::INT, 0, 20); auto tInt = primitiveType(PrimitiveType::INT); - TestSignature fib{ "fib", { n }, tInt }; + TestSignature fib{ "fib", { n }, tInt, "../examples/fib.cpp" }; TestMain(argc, argv) - .fuzz(fib, 5) + .fuzz(&fib, 5) .run(); return 0; } diff --git a/src/printer.cpp b/src/printer.cpp new file mode 100644 index 0000000..9f0d719 --- /dev/null +++ b/src/printer.cpp @@ -0,0 +1,69 @@ +#include "test.h" + +struct TypePrinter { + std::string s; + + void visitPrimitiveType(PrimitiveType x) { + s = printPrimitiveType(x); + } + + void visitPointerTo(const PointerTo& x) { + s = printType(*x.type) + "*"; + } + + void visitInRange(const InRange& x) { + s = printType(*x.type); + } +}; + +std::string printType(const Type &type) { + TypePrinter typePrinter; + type.accept(typePrinter); + return typePrinter.s; +} + +struct ValuePrinter { + std::string value, s; + + void visitPrimitiveType(PrimitiveType x) { + s = "static_cast<" + printPrimitiveType(x) + ">(" + value + ")"; + } + + void visitPointerTo(const PointerTo& x) { + s = "new " + printType(*x.type) + "{" + printValue(*x.type, value) + "}"; + } + + void visitInRange(const InRange& x) { + s = printValue(*x.type, value); + } +}; + +std::string printValue(const Type &type, const std::string &value) { + ValuePrinter valuePrinter; + valuePrinter.value = value; + type.accept(valuePrinter); + return valuePrinter.s; +} + +struct ValueSerializer { + std::string value, s; + + void visitPrimitiveType(PrimitiveType x) { + s = "std::cout << static_cast<" + printPrimitiveType(x) + ">(" + value + ");\n"; + } + + void visitPointerTo(const PointerTo& x) { + s = printValueSerializer(*x.type, value); + } + + void visitInRange(const InRange& x) { + s = printValueSerializer(*x.type, value); + } +}; + +std::string printValueSerializer(const Type &type, const std::string &value) { + ValueSerializer valueSerializer; + valueSerializer.value = value; + type.accept(valueSerializer); + return valueSerializer.s; +} diff --git a/src/subprocess.cpp b/src/subprocess.cpp new file mode 100644 index 0000000..3f9dfce --- /dev/null +++ b/src/subprocess.cpp @@ -0,0 +1,97 @@ +#include "subprocess.h" +#include +#include +#include +#include +#include +#include +#include + +Subprocess::Subprocess(std::vector cmd) : cmd(std::move(cmd)), exitCode(0) {} + +Subprocess &Subprocess::run(const std::string& input) { + exitCode = -1; + capturedStdout.clear(); + + std::array buf; + + int inputPipe[2]; + int outputPipe[2]; + int statusPipe[2]; + + if (pipe(inputPipe) != 0) { + throw std::system_error(errno, std::generic_category()); + } + + if (pipe(outputPipe) != 0) { + throw std::system_error(errno, std::generic_category()); + } + + if (pipe(statusPipe) != 0) { + throw std::system_error(errno, std::generic_category()); + } + + pid_t pid = fork(); + + if (pid < 0) { + throw std::system_error(errno, std::generic_category()); + } + + int ready = 0; + + if (pid == 0) { + // child + close(inputPipe[1]); + close(outputPipe[0]); + close(statusPipe[1]); + + read(statusPipe[0], &ready, sizeof(ready)); + close(statusPipe[0]); + + dup2(inputPipe[0], 0); // remap stdin + dup2(outputPipe[1], 1); // remap stdout + + std::vector argv; + for (auto s : cmd) { + argv.push_back(strdup(s.c_str())); + } + + execvp(cmd[0].c_str(), argv.data()); + } + + // parent + + close(outputPipe[1]); + close(inputPipe[0]); + close(statusPipe[0]); + write(inputPipe[1], input.data(), input.size()); + close(inputPipe[1]); + write(statusPipe[1], &ready, sizeof(ready)); + close(statusPipe[1]); + + int stat; + waitpid(pid, &stat, 0); + + if (WIFEXITED(stat)) { + exitCode = WEXITSTATUS(stat); + } else if (WIFSIGNALED(stat)) { + exitCode = -WTERMSIG(stat); + } + + int rd; + while ((rd = read(outputPipe[0], buf.data(), buf.size())) > 0) { + std::copy(buf.begin(), buf.begin() + rd, std::back_inserter(capturedStdout)); + } + + close(outputPipe[0]); + + return *this; +} + +std::string Subprocess::output() const { + return capturedStdout; +} + +Subprocess::operator bool() const { + return exitCode == 0; +} diff --git a/src/subprocess.h b/src/subprocess.h new file mode 100644 index 0000000..13a9ff4 --- /dev/null +++ b/src/subprocess.h @@ -0,0 +1,18 @@ +#pragma once + +#include +#include + +class Subprocess { +private: + std::vector cmd; + std::string capturedStdout; + int exitCode; + +public: + explicit Subprocess(std::vector cmd); + explicit operator bool() const; + + Subprocess &run(const std::string &input); + [[nodiscard]] std::string output() const; +}; diff --git a/src/test.cpp b/src/test.cpp index 6ee1545..380e6c6 100644 --- a/src/test.cpp +++ b/src/test.cpp @@ -1,33 +1,14 @@ #include "test.h" +#include "subprocess.h" +#include +#include +#include +#include #include std::string Value::print() const { - return type->printValue(std::to_string(value)); -} - -std::string Type::print() const { - if (auto primitiveType = std::get_if(&type)) { - return printPrimitiveType(*primitiveType); - } else if (auto pointerTo = std::get_if(&type)) { - return pointerTo->type->print() + "*"; - } else if (auto inRange = std::get_if(&type)) { - return inRange->type->print(); - } - - return ""; -} - -std::string Type::printValue(const std::string& value) const { - if (auto primitiveType = std::get_if(&type)) { - return "static_cast<" + print() + ">(" + value + ")"; - } else if (auto pointerTo = std::get_if(&type)) { - return "new " + pointerTo->type->print() + "{" + pointerTo->type->printValue(value) + "}"; - } else if (auto inRange = std::get_if(&type)) { - return inRange->type->printValue(value); - } - - return ""; + return printValue(*type, value); } std::pair Type::getRange() const { @@ -45,48 +26,19 @@ std::pair Type::getRange() const { return { 0, 0 }; } -std::string quote(const std::string& s) { - std::stringstream ss; - - ss << '"'; - bool inEscape = false; - - for (char c : s) { - if (c == '\n') ss << "\\n"; - else if (c == '\t') ss << "\\t"; - else if (c == '"') ss << "\\\""; - else if (c == '\0') { - inEscape = !inEscape; - if (inEscape) { - ss << "\" << "; - } else { - ss << " << \""; - } - } else ss << c; - } - - ss << '"'; - - return ss.str(); -} - std::string Test::print() const { std::stringstream ss; auto call = printFunctionCall(); - ss << "TEST(" << signature.name << "Test, " << name << ") {\n"; + ss << "TEST(" << signature->name << "Test, " << name << ") {\n"; ss << " ASSERT_EQ(" << call; - ss << ", " << signature.returnType->printValue('\0' + call + '\0') << ");\n"; + ss << ", " << signature->call(arguments).print() << ");\n"; ss << "}\n"; return ss.str(); } -std::string Test::printGenerator() const { - return "std::cout << " + quote(print()) + ";\n"; -} - std::string Test::printFunctionCall() const { std::string res; - res += signature.name + "("; + res += signature->name + "("; bool first = true; @@ -131,7 +83,7 @@ Type::ptr pointerTo(Type::ptr type) { } std::string TestSignature::print() const { - std::string res = returnType->print() + " " + name; + std::string res = printType(*returnType) + " " + name; res += "("; bool first = true; @@ -141,9 +93,211 @@ std::string TestSignature::print() const { res += ", "; } first = false; - res += type->print(); + res += printType(*type); } res += ");\n"; return res; } + +std::string TestSignature::printInvoker() const { + std::string res = "#include\n"; + res += print(); + res += "int main() {\n"; + + int i = 0; + Test test; + test.signature = this; + + for (const auto& param : parameterTypes) { + std::string paramName = "arg" + std::to_string(i); + res += "long " + paramName + "; std::cin >> " + paramName + ";\n"; + test.arguments.push_back(Value { param, paramName }); + } + + res += "auto retval = " + test.printFunctionCall() + ";\n"; + res += printValueSerializer(*returnType, "retval"); + res += "return 0;\n}\n"; + + return res; +} + +std::string TestSignature::getInvoker() const { + if (!pathToInvoker.empty()) { + return pathToInvoker; + } + + std::string path = "/tmp/UnitTestFuzzerInvoker_" + name; + + Subprocess compiler{{"clang++", "-std=c++17", "-o", path, linkWith, "-x", "c++", "-"}}; + if (!compiler.run(printInvoker())) { + throw std::runtime_error("Failed to compile invoker"); + } + + return pathToInvoker = path; +} + +std::string TestSignature::callSerialized(const std::string& args) const { + std::string path = getInvoker(); + Subprocess prog{{path}}; + + if (!prog.run(args)) { + throw std::runtime_error("Failed to run invoker"); + } + + return prog.output(); +} + +Value TestSignature::call(const std::vector& args) const { + std::string serialized; + for (const auto &val : args) { + serialized += val.value; + serialized += ' '; + } + serialized += '\n'; + + auto ret = callSerialized(serialized); + return Value { returnType, ret }; +} + +TestSignature::TestSignature(TestSignature &&that) noexcept { + swap(that); +} + +TestSignature &TestSignature::operator=(TestSignature &&that) noexcept { + swap(that); + return *this; +} + +void TestSignature::swap(TestSignature &that) noexcept { + std::swap(name, that.name); + std::swap(parameterTypes, that.parameterTypes); + std::swap(returnType, that.returnType); + std::swap(linkWith, that.linkWith); + std::swap(pathToInvoker, that.pathToInvoker); +} + +TestSignature::~TestSignature() { + if (!pathToInvoker.empty()) { + unlink(pathToInvoker.c_str()); + } +} + +TestSignature::TestSignature(std::string name, + std::vector parameterTypes, + Type::ptr returnType, + std::string linkWith) : + name(std::move(name)), + parameterTypes(std::move(parameterTypes)), + returnType(std::move(returnType)), + linkWith(std::move(linkWith)) +{} + +std::string TestSignature::printStruct(const std::string &structName) const { + std::stringstream ss; + + ss << "struct " << structName << " {\n"; + + int i = 0; + for (const auto& param : parameterTypes) { + std::string paramName = "arg" + std::to_string(i++); + ss << "long " << paramName << ";\n"; + } + + ss << "};\n"; + + return ss.str(); +} + +std::string TestSignature::printFuzzer(int nTests, int interval) const { + std::stringstream ss; + + ss << "#include \n"; + ss << "#include \n"; + ss << "#include \n"; + ss << print(); + ss << printStruct("FuzzArgs"); + ss << "extern \"C\" int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) {\n"; + ss << "static long cnt, tests;\n"; + ss << "FuzzArgs args;\n"; + ss << "if (size != sizeof(args)) return 0;\n"; + ss << "memcpy(&args, data, size);\n"; + + int i = 0; + Test test; + test.signature = this; + + for (const auto& param : parameterTypes) { + std::string paramName = "args.arg" + std::to_string(i); + test.arguments.push_back(Value { param, paramName }); + + auto [lo, hi] = param->getRange(); + if (lo != std::numeric_limits::min() || hi != std::numeric_limits::max()) { + PrimitiveIntegerU domain = 1 + hi - lo; + ss << paramName << " = " << "(" << paramName << " % " << domain << " + " << domain << ") % "; + ss << domain << " + " << lo << ";\n"; + } + } + + ss << test.printFunctionCall() << ";\n"; + ss << "if (++cnt % " << interval << " == 0) {\n"; + i = 0; + for (const auto& param : parameterTypes) { + std::string paramName = "args.arg" + std::to_string(i); + ss << "std::cout << \"test \" << " << paramName << " << ' ';\n"; + } + ss << "std::cout << std::endl;\n"; + ss << "if (++tests >= " << nTests << ") {\n"; + ss << "std::cout << \"exit\" << std::endl;\n"; + ss << "exit(0);\n"; + ss << "}\n"; + ss << "}\n"; + ss << "return 0;\n"; + ss << "}\n"; + + return ss.str(); +} + +std::string TestSignature::runFuzzer(int nTests) const { + std::string path = "/tmp/UnitTestFuzzerFuzzer_" + name; + + Subprocess compiler{{"clang++", "-g", "-fsanitize=address,fuzzer", "-std=c++17", "-o", path, linkWith, "-x", "c++", "-"}}; + if (!compiler.run(printFuzzer(nTests, 100))) { + throw std::runtime_error("Failed to compile fuzzer"); + } + + Subprocess fuzzer{{path, "-verbosity=0"}}; + auto res = fuzzer.run("").output(); + unlink(path.c_str()); + return res; +} + +[[nodiscard]] std::vector TestSignature::fuzz(int nTests) const { + std::stringstream ss(runFuzzer(nTests)); + std::vector res; + + int i = 0; + + std::string cmd; + while (ss >> cmd) { + if (cmd == "exit") { + break; + } + + if (cmd == "test") { + Test test; + test.signature = this; + test.name = "test_" + std::to_string(i++); + for (const auto& param : parameterTypes) { + Value value; + value.type = param; + ss >> value.value; + test.arguments.push_back(value); + } + + res.push_back(test); + } + } + + return res; +} diff --git a/src/test.h b/src/test.h index 95df47b..d3e5c41 100644 --- a/src/test.h +++ b/src/test.h @@ -1,15 +1,16 @@ #pragma once +#include #include #include #include #include #include -#include struct Type; using PrimitiveInteger = long; +using PrimitiveIntegerU = unsigned long; enum class PrimitiveType { CHAR, @@ -37,60 +38,75 @@ struct Type { using ptr = std::shared_ptr; - [[nodiscard]] std::string print() const; - [[nodiscard]] std::string printValue(const std::string &value) const; [[nodiscard]] std::pair getRange() const; + + template + void accept(Visitor &visitor) const { + if (auto primitiveType = std::get_if(&type)) { + visitor.visitPrimitiveType(*primitiveType); + } else if (auto pointerTo = std::get_if(&type)) { + visitor.visitPointerTo(*pointerTo); + } else if (auto inRange = std::get_if(&type)) { + visitor.visitInRange(*inRange); + } + } }; +std::string printType(const Type &type); +std::string printValue(const Type &type, const std::string &value); +std::string printValueSerializer(const Type &type, const std::string &value); + [[nodiscard]] Type::ptr primitiveType(PrimitiveType type); [[nodiscard]] Type::ptr primitiveType(PrimitiveType type, PrimitiveInteger min, PrimitiveInteger max); [[nodiscard]] Type::ptr pointerTo(Type::ptr type); -struct TestSignature { - std::string name; - std::vector parameterTypes; - Type::ptr returnType; - - [[nodiscard]] std::string print() const; -}; - struct Value { Type::ptr type; - PrimitiveInteger value; + std::string value; [[nodiscard]] std::string print() const; - - template - static Value generate(Generator& gen, const Type::ptr& type) { - auto [a, b] = type->getRange(); - std::uniform_int_distribution distribution{a, b}; - return Value { type, distribution(gen) }; - } }; +struct TestSignature; + struct Test { std::string name; - TestSignature signature; + const TestSignature *signature; std::vector arguments; - // after test launch - std::optional returnValue; - [[nodiscard]] std::string print() const; - [[nodiscard]] std::string printGenerator() const; [[nodiscard]] std::string printFunctionCall() const; +}; - template - static Test generate(Generator &gen, std::string name, TestSignature signature) { - Test test; - test.name = std::move(name); - test.signature = std::move(signature); - for (const auto &type : test.signature.parameterTypes) { - test.arguments.push_back(Value::generate(gen, type)); - } +struct TestSignature { + std::string name; + std::vector parameterTypes; + Type::ptr returnType; + std::string linkWith; - return test; - } + [[nodiscard]] std::string print() const; + [[nodiscard]] std::string printStruct(const std::string &structName) const; + [[nodiscard]] std::string callSerialized(const std::string& args) const; + [[nodiscard]] Value call(const std::vector& args) const; + [[nodiscard]] std::vector fuzz(int nTests) const; + + TestSignature() = default; + TestSignature(std::string name, + std::vector parameterTypes, + Type::ptr returnType, + std::string linkWith); + TestSignature(TestSignature &&that) noexcept; + TestSignature &operator=(TestSignature &&that) noexcept; + ~TestSignature(); + void swap(TestSignature &that) noexcept; + +private: + mutable std::string pathToInvoker; + [[nodiscard]] std::string printInvoker() const; + [[nodiscard]] std::string getInvoker() const; + + [[nodiscard]] std::string printFuzzer(int nTests, int interval) const; + [[nodiscard]] std::string runFuzzer(int nTests) const; }; diff --git a/src/test_main.cpp b/src/test_main.cpp index 2e5fda5..fbb398c 100644 --- a/src/test_main.cpp +++ b/src/test_main.cpp @@ -3,13 +3,14 @@ #include #include -TestMain::TestMain(int argc, char **argv) : gen { std::random_device{}() } { +TestMain::TestMain(int argc, char **argv) { for (int i = 0; i < argc; ++i) { args.emplace_back(argv[i]); } } TestMain &TestMain::add(Test test) { + signatures.insert(test.signature); tests.push_back(std::move(test)); return *this; } @@ -24,20 +25,20 @@ void TestMain::run() { usage(); } - std::cout << "#include \n"; - for (const auto& test : tests) { - std::cout << test.signature.print(); + std::cout << "#include \n"; + + for (auto signature : signatures) { + std::cout << signature->print(); } - std::cout << "int main() {\n"; + for (const auto& test : tests) { - std::cout << test.printGenerator(); + std::cout << test.print(); } - std::cout << "}\n"; } -TestMain &TestMain::fuzz(const TestSignature& testSignature, int n) { - for (int i = 0; i < n; ++i) { - add(Test::generate(gen, "test_" + std::to_string(i), testSignature)); +TestMain &TestMain::fuzz(TestSignature *testSignature, int n) { + for (const auto &test : testSignature->fuzz(n)) { + add(test); } return *this; diff --git a/src/test_main.h b/src/test_main.h index ee0427c..283173a 100644 --- a/src/test_main.h +++ b/src/test_main.h @@ -3,12 +3,13 @@ #include "test.h" #include #include +#include class TestMain { private: std::vector args; std::vector tests; - std::mt19937 gen; + std::unordered_set signatures; private: void usage(); @@ -17,6 +18,6 @@ class TestMain { TestMain(int argc, char **argv); TestMain &add(Test test); - TestMain &fuzz(const TestSignature& testSignature, int n); + TestMain &fuzz(TestSignature *testSignature, int n); void run(); };