New test runner design (#130)

Closes #110

Author: mk-mxp

.dockerignore

@@ -1,22 +1,11 @@
-# Ignore git details
-.git
-.gitignore
-.github
+# Disallow everything
+*
 
-# Ignore run artifacts
-error_log
-metadata.json
-output
-output.json
-results.xml
-results.json
-
-# Ignore smoke test files
-test/
-
-junit-to-json/node_modules/
-junit-to-json/**/*.js
-.appends
-.gitattributes
-.dockerignore
-Dockerfile
+# Allow list
+!bin/run.sh
+!CODE_OF_CONDUCT.md
+!composer.json
+!LICENSE
+!phpunit.xml
+!README.md
+!src

.github/CODEOWNERS

@@ -1,2 +1,2 @@
-* @neenjaw
-.github/CODEOWNERS @exercism/maintainers-admin
+* @mk-mxp
+.github/CODEOWNERS @exercism/maintainers-admin

.github/workflows/smoketest.yml

@@ -18,7 +18,7 @@ jobs:
       - name: Build Docker image and store in cache
         uses: docker/build-push-action@16ebe778df0e7752d2cfcbd924afdbbd89c1a755
         env:
-          DOCKER_BUILD_NO_SUMMARY: true
+          DOCKER_BUILD_SUMMARY: false
         with:
           context: .
           push: false

.gitignore

@@ -11,3 +11,6 @@ results.xml
 results.json
 
 *.scratch
+
+/vendor/
+composer.lock

Dockerfile

@@ -1,38 +1,40 @@
-FROM php:8.3.4-cli-alpine3.19 AS build
+FROM php:8.3.10-cli-alpine3.20 AS build
 
-RUN apk update && \
-  apk add --no-cache ca-certificates curl jo zip unzip
+RUN apk add --no-cache ca-certificates curl jo zip unzip
 
 WORKDIR /usr/local/bin
 
-RUN curl -L -o install-php-extensions https://github.com/mlocati/docker-php-extension-installer/releases/latest/download/install-php-extensions && \
-  chmod +x install-php-extensions && \
-  install-php-extensions ds-1.5.0 intl
+RUN curl -L -o install-php-extensions \
+    https://github.com/mlocati/docker-php-extension-installer/releases/latest/download/install-php-extensions \
+    && chmod +x install-php-extensions \
+    && install-php-extensions ds-^1@stable intl
 
-RUN curl -L -o phpunit-10.phar https://phar.phpunit.de/phpunit-10.phar && \
-  chmod +x phpunit-10.phar
+COPY --from=composer:2.7.7 /usr/bin/composer /usr/local/bin/composer
 
-WORKDIR /usr/local/bin/junit-handler/
-COPY --from=composer:2.7.2 /usr/bin/composer /usr/local/bin/composer
-COPY junit-handler/ .
-# We need PHPUnit from junit-handler/ to run test-runner tests in CI / locally
-# composer warns about missing a "root version" to resolve dependencies. Fake to stop warning
-RUN COMPOSER_ROOT_VERSION=1.0.0 composer install --no-interaction
-
-FROM php:8.3.4-cli-alpine3.19 AS runtime
+WORKDIR /opt/test-runner
+COPY . .
+# composer warns about missing a "root version" to resolve dependencies. Fake to stop warning.
+# composer warns about running as root. Silence it, we know what we are doing.
+RUN mv "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini" \
+    && php --modules \
+    && COMPOSER_ALLOW_SUPERUSER=1 \
+        composer --version \
+    && COMPOSER_ROOT_VERSION=1.0.0 \
+        COMPOSER_ALLOW_SUPERUSER=1 \
+        composer install --no-cache --no-dev --no-interaction --no-progress
+
+FROM php:8.3.10-cli-alpine3.20 AS runtime
 
 COPY --from=build /usr/bin/jo /usr/bin/jo
 COPY --from=build /usr/local/lib/php/extensions /usr/local/lib/php/extensions
-COPY --from=build /usr/local/bin/phpunit-10.phar /opt/test-runner/bin/phpunit-10.phar
-COPY --from=build /usr/local/bin/junit-handler /opt/test-runner/junit-handler
+COPY --from=build /opt/test-runner /opt/test-runner
 
 # Use the default production configuration
-RUN mv "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini"
-RUN apk add --no-cache bash
-RUN adduser -Ds /bin/bash appuser
+RUN mv "$PHP_INI_DIR/php.ini-production" "$PHP_INI_DIR/php.ini" \
+    && apk add --no-cache bash \
+    && adduser -Ds /bin/bash appuser
 
 WORKDIR /opt/test-runner
-COPY . .
 
 USER appuser
 

README.md

@@ -10,7 +10,7 @@ It meets the complete spec for testing all exercises.
 ### Docker image
 
 The website uses isolated Docker images to run untrusted code in a sandbox.
-The image provided by this repository consists of PHP 8.3.4 (PHPUnit 10).
+The image provided by this repository consists of PHP 8.3.10 (PHPUnit 10).
 All final assets are built into the image, because the image does not have network access once in use.
 
 Includes PHP extensions: ds, intl
@@ -28,15 +28,23 @@ This is what runs inside the production Docker image when students submit their
 ### Testing the test runner
 
 In `./tests/` are golden tests to verify that the test runner behaves as expected.
-The CI uses `bin/run-tests.sh` to execute them.
+The CI uses `bin/run-tests.sh` to execute them in the Docker image.
+
+### Running tests locally
+
+Recommended to easily test new test cases during development.
+Use `bin/run-locally.sh <test-slug>` to run PHPUnit in your current shell and compare the resulting JSON to `expected_results.json`.
+Make sure you have at least PHP 8.1 installed.
 
 ### Running tests in Docker locally
 
-This is the recommended way to use this locally.
-Use `bin/run-in-docker.sh <test-slug> <directory path to solution> <directory path for output>` and `bin/run-tests-in-docker.sh` to locally build and run the Docker image.
+This is the recommended way to run all tests locally as they would run in production.
+Use `bin/run-tests-in-docker.sh` to locally build and run all tests in the Docker image.
+Use `bin/run-in-docker.sh <test-slug> <directory path to solution> <directory path for output>` for a single test run in the Docker image.
 
-### JUnit to JSON
+### PHPUnit extension
 
-PHPUnit can natively output tests run to JUnit XML format, but Exercism requires output in JSON format.
-A PHP-based app is located in the `junit-handler` folder.
-It provides a translation layer from one format to the other incorporating `task_id` identification and test code inclusion.
+PHPUnit can natively output test results to various formats, but Exercism requires output in a special JSON format.
+An extension to PHPUnit is located in `src` folder and registered in `phpunit.xml`.
+It provides a tracer for PHPUnit events to produce the required JSON result.
+The tracer incorporates `task_id` identification, test code inclusion, and user output dumping.

bin/run-locally.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+EXERCISM_RESULT_FILE="${PWD%/}/results.json" \
+EXERCISM_EXERCISE_DIR="${PWD%/}/tests/${1}" \
+vendor/bin/phpunit --do-not-cache-result tests/"${1}"/*Test.php
+
+# Sync'ed from run-tests.sh - Normalize the object ID of `var_dump(new stdClass())`
+sed -i -E \
+    -e 's/(object\(stdClass\))(#[[:digit:]]+)/\1#79/g' \
+    results.json
+diff results.json tests/"${1}"/expected_results.json

bin/run-tests-in-docker.sh

@@ -23,8 +23,8 @@ docker run \
     --rm \
     --network none \
     --read-only \
-    --mount type=bind,src="${PWD}/tests",dst=/opt/test-runner/tests \
     --mount type=tmpfs,dst=/tmp \
+    --volume "${PWD}/tests:/opt/test-runner/tests" \
     --volume "${PWD}/bin/run-tests.sh:/opt/test-runner/bin/run-tests.sh" \
     --workdir /opt/test-runner \
     --entrypoint /opt/test-runner/bin/run-tests.sh \

bin/run.sh

@@ -2,9 +2,7 @@
 
 set -euo pipefail
 
-PHPUNIT_BIN="./bin/phpunit-10.phar"
-JUNIT_RESULTS='results.xml'
-TEAMCITY_RESULTS='teamcity.txt'
+PHPUNIT_BIN="./vendor/bin/phpunit"
 EXERCISM_RESULTS='results.json'
 # shellcheck disable=SC2034 # Modifies XDebug behaviour when invoking PHP
 XDEBUG_MODE='off'
@@ -25,19 +23,16 @@ function main {
     return 0;
   fi
 
-  # JUnit results contain the unit test failures only. But they contain `@testdox` - readable test names.
-  # Teamcity results contain user output in addition to failures as "failed risky tests"
-  #   (command line arguments --disallow-test-output and --fail-on-risky).
-  # At the moment we require both logs to provide all information to the website.
-  output=$("${PHPUNIT_BIN}" \
-    -d memory_limit=300M \
-    --log-junit "${output_dir%/}/${JUNIT_RESULTS}" \
-    --log-teamcity "${output_dir%/}/${TEAMCITY_RESULTS}" \
-    --no-configuration \
-    --do-not-cache-result \
-    --disallow-test-output \
-    --fail-on-risky \
-    "${test_files%%*( )}" 2>&1)
+  # Our PHPUnit extension writes directly to ${EXERCISM_RESULT_FILE}
+  # Our PHPUnit extension requires ${EXERCISM_EXERCISE_DIR} before PHPUnit provides it
+  output=$( \
+    EXERCISM_RESULT_FILE="${output_dir%/}/${EXERCISM_RESULTS}" \
+    EXERCISM_EXERCISE_DIR="${solution_dir%/}" \
+    "${PHPUNIT_BIN}" \
+      -d memory_limit=300M \
+      --do-not-cache-result \
+      "${test_files%%*( )}" 2>&1 \
+  )
   phpunit_exit_code=$?
   set -e
 
@@ -48,20 +43,6 @@ function main {
     jo version=3 status=error message="${output//"$solution_dir/"/""}" tests="[]" > "${output_dir%/}/${EXERCISM_RESULTS}"
     return 0;
   fi
-
-  # This catches runtime errors in "global student code" (during `require_once`)
-  if [[ "${phpunit_exit_code}" -eq 2 ]]; then
-    if ! grep -q '<testcase' "${output_dir%/}/${JUNIT_RESULTS}"; then
-        output="${output#*" MB"}"
-        jo version=3 status=error message="${output//"$solution_dir/"/""}" tests="[]" > "${output_dir%/}/${EXERCISM_RESULTS}"
-        return 0;
-    fi
-  fi
-
-  php junit-handler/run.php \
-    "${output_dir%/}/${EXERCISM_RESULTS}" \
-    "${output_dir%/}/${JUNIT_RESULTS}" \
-    "${output_dir%/}/${TEAMCITY_RESULTS}"
 }
 
 function installed {

composer.json

@@ -0,0 +1,13 @@
+{
+    "name": "exercism/php-test-runner",
+    "type": "project",
+    "license": "AGPL-3.0-only",
+    "autoload": {
+        "psr-4": {
+            "Exercism\\PhpTestRunner\\": "src/"
+        }
+    },
+    "require": {
+        "phpunit/phpunit": "^10.5.29"
+    }
+}