test: exercise the clar itself

While we have had the "clar_test" binary for a very long time already,
it was more for demonstration purposes than for real testing of the
clar. It could serve as sort of a smoke test if the developer remembered
to execute it, but that's about it.

In the preceding commits we have moved the "clar_test" into a separate
directory, renamed it to "selftest_suite" and made its output
deterministic. We can thus now wire up proper testing of the clar by
exercising the "selftest_suite" binary via a new "selftest" binary.
These tests run the "selftest_test" binary with various different
arguments and check both its return code as well as its output.

test/CMakeLists.txt

@@ -1 +1,56 @@
 add_subdirectory(selftest_suite)
+
+find_package(Python COMPONENTS Interpreter REQUIRED)
+
+add_custom_command(OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/clar.suite"
+	COMMAND "${Python_EXECUTABLE}" "${CMAKE_SOURCE_DIR}/generate.py" --output "${CMAKE_CURRENT_BINARY_DIR}"
+	DEPENDS main.c selftest.c
+	WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+)
+
+add_executable(selftest)
+set_target_properties(selftest PROPERTIES
+	C_STANDARD 90
+	C_STANDARD_REQUIRED ON
+	C_EXTENSIONS OFF
+)
+
+# MSVC generates all kinds of warnings. We may want to fix these in the future
+# and then unconditionally treat warnings as errors.
+if (NOT MSVC)
+	set_target_properties(selftest PROPERTIES
+		COMPILE_WARNING_AS_ERROR ON
+	)
+endif()
+
+target_sources(selftest PRIVATE
+	main.c
+	selftest.c
+	"${CMAKE_CURRENT_BINARY_DIR}/clar.suite"
+)
+target_compile_definitions(selftest PRIVATE
+	CLAR_FIXTURE_PATH="${CMAKE_CURRENT_SOURCE_DIR}/expected/"
+	_DARWIN_C_SOURCE
+	_POSIX_C_SOURCE=200809L
+)
+target_compile_options(selftest PRIVATE
+	$<IF:$<CXX_COMPILER_ID:MSVC>,/W4,-Wall>
+)
+target_include_directories(selftest PRIVATE
+	"${CMAKE_SOURCE_DIR}"
+	"${CMAKE_CURRENT_BINARY_DIR}"
+)
+target_link_libraries(selftest clar)
+
+add_test(NAME build_selftest_suite
+	COMMAND "${CMAKE_COMMAND}" --build "${CMAKE_BINARY_DIR}" --config "$<CONFIG>" --target selftest_suite
+)
+set_tests_properties(build_selftest_suite PROPERTIES FIXTURES_SETUP clar_test_fixture)
+
+add_test(NAME build_selftest
+	COMMAND "${CMAKE_COMMAND}" --build "${CMAKE_BINARY_DIR}" --config "$<CONFIG>" --target selftest
+)
+set_tests_properties(build_selftest PROPERTIES FIXTURES_SETUP clar_test_fixture)
+
+add_test(NAME selftest COMMAND "${CMAKE_CURRENT_BINARY_DIR}/selftest" "$<TARGET_FILE:selftest_suite>")
+set_tests_properties(selftest PROPERTIES FIXTURES_REQUIRED clar_test_fixture)

test/expected/help

@@ -0,0 +1,12 @@
+Usage: selftest [options]
+
+Options:
+  -sname        Run only the suite with `name` (can go to individual test name)
+  -iname        Include the suite with `name`
+  -xname        Exclude the suite with `name`
+  -v            Increase verbosity (show suite names)
+  -q            Only report tests that had an error
+  -Q            Quit as soon as a test fails
+  -t            Display results in tap format
+  -l            Print suite names
+  -r[filename]  Write summary file (to the optional filename)

test/expected/quiet

@@ -0,0 +1,80 @@
+Loaded 1 suites: 
+Started (test status codes: OK='.' FAILURE='F' SKIPPED='S')
+  1) Failure:
+selftest::suite::1 [file:42]
+  Function call failed: -1
+
+  1) Failure:
+selftest::suite::2 [file:42]
+  Expression is not true: 100 == 101
+
+  1) Failure:
+selftest::suite::strings [file:42]
+  String mismatch: "mismatched" != actual ("this one fails")
+  'mismatched' != 'expected' (at byte 0)
+
+  1) Failure:
+selftest::suite::strings_with_length [file:42]
+  String mismatch: "exactly" != actual ("this one fails")
+  'exa' != 'exp' (at byte 2)
+
+  1) Failure:
+selftest::suite::int [file:42]
+  101 != value ("extra note on failing test")
+  101 != 100
+
+  1) Failure:
+selftest::suite::int_fmt [file:42]
+  022 != value
+  0022 != 0144
+
+  1) Failure:
+selftest::suite::bool [file:42]
+  0 != value
+  0 != 1
+
+  1) Failure:
+selftest::suite::ptr [file:42]
+  Pointer mismatch: p1 != p2
+  0x1 != 0x2
+
+
+
+  1) Failure:
+selftest::suite::1 [file:42]
+  Function call failed: -1
+
+  2) Failure:
+selftest::suite::2 [file:42]
+  Expression is not true: 100 == 101
+
+  3) Failure:
+selftest::suite::strings [file:42]
+  String mismatch: "mismatched" != actual ("this one fails")
+  'mismatched' != 'expected' (at byte 0)
+
+  4) Failure:
+selftest::suite::strings_with_length [file:42]
+  String mismatch: "exactly" != actual ("this one fails")
+  'exa' != 'exp' (at byte 2)
+
+  5) Failure:
+selftest::suite::int [file:42]
+  101 != value ("extra note on failing test")
+  101 != 100
+
+  6) Failure:
+selftest::suite::int_fmt [file:42]
+  022 != value
+  0022 != 0144
+
+  7) Failure:
+selftest::suite::bool [file:42]
+  0 != value
+  0 != 1
+
+  8) Failure:
+selftest::suite::ptr [file:42]
+  Pointer mismatch: p1 != p2
+  0x1 != 0x2
+

test/expected/specific_test

@@ -0,0 +1,9 @@
+Loaded 1 suites: 
+Started (test status codes: OK='.' FAILURE='F' SKIPPED='S')
+F
+
+  1) Failure:
+selftest::suite::bool [file:42]
+  0 != value
+  0 != 1
+

test/expected/stop_on_failure

@@ -0,0 +1,8 @@
+Loaded 1 suites: 
+Started (test status codes: OK='.' FAILURE='F' SKIPPED='S')
+F
+
+  1) Failure:
+selftest::suite::1 [file:42]
+  Function call failed: -1
+

test/expected/suite_names

@@ -0,0 +1,2 @@
+Test suites (use -s<name> to run just one):
+   0: selftest::suite

test/expected/summary.xml

@@ -0,0 +1,36 @@
+<testsuites>
+	<testsuite id="0" name="selftest" hostname="localhost" timestamp="2024-09-06T10:04:08" tests="8" failures="8" errors="0">
+		<testcase name="1" classname="selftest" time="0.00">
+			<failure type="assert"><![CDATA[Function call failed: -1
+(null)]]></failure>
+		</testcase>
+		<testcase name="2" classname="selftest" time="0.00">
+			<failure type="assert"><![CDATA[Expression is not true: 100 == 101
+(null)]]></failure>
+		</testcase>
+		<testcase name="strings" classname="selftest" time="0.00">
+			<failure type="assert"><![CDATA[String mismatch: "mismatched" != actual ("this one fails")
+'mismatched' != 'expected' (at byte 0)]]></failure>
+		</testcase>
+		<testcase name="strings_with_length" classname="selftest" time="0.00">
+			<failure type="assert"><![CDATA[String mismatch: "exactly" != actual ("this one fails")
+'exa' != 'exp' (at byte 2)]]></failure>
+		</testcase>
+		<testcase name="int" classname="selftest" time="0.00">
+			<failure type="assert"><![CDATA[101 != value ("extra note on failing test")
+101 != 100]]></failure>
+		</testcase>
+		<testcase name="int_fmt" classname="selftest" time="0.00">
+			<failure type="assert"><![CDATA[022 != value
+0022 != 0144]]></failure>
+		</testcase>
+		<testcase name="bool" classname="selftest" time="0.00">
+			<failure type="assert"><![CDATA[0 != value
+0 != 1]]></failure>
+		</testcase>
+		<testcase name="ptr" classname="selftest" time="0.00">
+			<failure type="assert"><![CDATA[Pointer mismatch: p1 != p2
+0x1 != 0x2]]></failure>
+		</testcase>
+	</testsuite>
+</testsuites>

test/expected/summary_with_filename

@@ -0,0 +1,43 @@
+Loaded 1 suites: 
+Started (test status codes: OK='.' FAILURE='F' SKIPPED='S')
+FFFFFFFF
+
+  1) Failure:
+selftest::suite::1 [file:42]
+  Function call failed: -1
+
+  2) Failure:
+selftest::suite::2 [file:42]
+  Expression is not true: 100 == 101
+
+  3) Failure:
+selftest::suite::strings [file:42]
+  String mismatch: "mismatched" != actual ("this one fails")
+  'mismatched' != 'expected' (at byte 0)
+
+  4) Failure:
+selftest::suite::strings_with_length [file:42]
+  String mismatch: "exactly" != actual ("this one fails")
+  'exa' != 'exp' (at byte 2)
+
+  5) Failure:
+selftest::suite::int [file:42]
+  101 != value ("extra note on failing test")
+  101 != 100
+
+  6) Failure:
+selftest::suite::int_fmt [file:42]
+  022 != value
+  0022 != 0144
+
+  7) Failure:
+selftest::suite::bool [file:42]
+  0 != value
+  0 != 1
+
+  8) Failure:
+selftest::suite::ptr [file:42]
+  Pointer mismatch: p1 != p2
+  0x1 != 0x2
+
+written summary file to different.xml

test/expected/summary_without_filename

@@ -0,0 +1,43 @@
+Loaded 1 suites: 
+Started (test status codes: OK='.' FAILURE='F' SKIPPED='S')
+FFFFFFFF
+
+  1) Failure:
+selftest::suite::1 [file:42]
+  Function call failed: -1
+
+  2) Failure:
+selftest::suite::2 [file:42]
+  Expression is not true: 100 == 101
+
+  3) Failure:
+selftest::suite::strings [file:42]
+  String mismatch: "mismatched" != actual ("this one fails")
+  'mismatched' != 'expected' (at byte 0)
+
+  4) Failure:
+selftest::suite::strings_with_length [file:42]
+  String mismatch: "exactly" != actual ("this one fails")
+  'exa' != 'exp' (at byte 2)
+
+  5) Failure:
+selftest::suite::int [file:42]
+  101 != value ("extra note on failing test")
+  101 != 100
+
+  6) Failure:
+selftest::suite::int_fmt [file:42]
+  022 != value
+  0022 != 0144
+
+  7) Failure:
+selftest::suite::bool [file:42]
+  0 != value
+  0 != 1
+
+  8) Failure:
+selftest::suite::ptr [file:42]
+  Pointer mismatch: p1 != p2
+  0x1 != 0x2
+
+written summary file to summary.xml

test/expected/tap

@@ -0,0 +1,81 @@
+TAP version 13
+# start of suite 1: selftest::suite
+not ok 1 - selftest::suite::1
+    ---
+    reason: |
+      Function call failed: -1
+    at:
+      file: 'file'
+      line: 42
+      function: 'func'
+    ---
+not ok 2 - selftest::suite::2
+    ---
+    reason: |
+      Expression is not true: 100 == 101
+    at:
+      file: 'file'
+      line: 42
+      function: 'func'
+    ---
+not ok 3 - selftest::suite::strings
+    ---
+    reason: |
+      String mismatch: "mismatched" != actual ("this one fails")
+      'mismatched' != 'expected' (at byte 0)
+    at:
+      file: 'file'
+      line: 42
+      function: 'func'
+    ---
+not ok 4 - selftest::suite::strings_with_length
+    ---
+    reason: |
+      String mismatch: "exactly" != actual ("this one fails")
+      'exa' != 'exp' (at byte 2)
+    at:
+      file: 'file'
+      line: 42
+      function: 'func'
+    ---
+not ok 5 - selftest::suite::int
+    ---
+    reason: |
+      101 != value ("extra note on failing test")
+      101 != 100
+    at:
+      file: 'file'
+      line: 42
+      function: 'func'
+    ---
+not ok 6 - selftest::suite::int_fmt
+    ---
+    reason: |
+      022 != value
+      0022 != 0144
+    at:
+      file: 'file'
+      line: 42
+      function: 'func'
+    ---
+not ok 7 - selftest::suite::bool
+    ---
+    reason: |
+      0 != value
+      0 != 1
+    at:
+      file: 'file'
+      line: 42
+      function: 'func'
+    ---
+not ok 8 - selftest::suite::ptr
+    ---
+    reason: |
+      Pointer mismatch: p1 != p2
+      0x1 != 0x2
+    at:
+      file: 'file'
+      line: 42
+      function: 'func'
+    ---
+1..8