phpt-tests-runner

#!/usr/bin/env php
<?php
/**
 * @package PHPT Tests runner
 * @author  Nazar Mokrynskyi <nazar@mokrynskyi.com>
 * @license 0BSD
 */
/**
 * @param string $text
 *
 * @return string
 */
function colorize ($text) {
	if (has_color_support()) {
		$callback = function ($matches) {
			$color_codes = [
				'g' => 32, // Green
				'y' => 33, // Yellow
				'r' => 31, // Red
			];
			$color       = $color_codes[$matches[1]];
			$text        = $matches[2];
			return "\033[{$color}m$text\033[0m";
		};
	} else {
		$callback = function ($matches) {
			return $matches[2];
		};
	}
	return preg_replace_callback(
		'#<([gyr])>(.*)</\1>#Us',
		$callback,
		$text
	);
}

/**
 * Returns true if ANSI escape sequences are supported.
 *
 * Based on code of Fabien Potencier:
 * \Symfony\Component\Console\Output\StreamOutput::hasColorSupport()
 *
 * @param bool|null $set_to
 *
 * @return bool true if ANSI escape sequences are supported.
 */
function has_color_support ($set_to = null) {
	static $result;
	if ($set_to !== null) {
		$result = $set_to;
	}
	if ($result !== null) {
		return $result;
	}
	if (strpos(PHP_OS, 'WIN') === false) {
		$result = function_exists('posix_isatty') && @posix_isatty(STDOUT);
	} else {
		$result =
			PHP_WINDOWS_VERSION_MAJOR.'.'.PHP_WINDOWS_VERSION_MINOR.'.'.PHP_WINDOWS_VERSION_BUILD === '10.0.10586' ||
			getenv('ANSICON') !== false ||
			getenv('ConEmuANSI') === 'ON' ||
			getenv('TERM') === 'xterm';
	}
	return $result;
}

/**
 * Output something to console
 *
 * Will colorize stuff in process
 *
 * @param bool   $clean Clean current line before output
 * @param string $text
 */
function out ($text, $clean = false) {
	if ($clean) {
		echo "\r";
	}
	echo colorize($text);
}

/**
 * Output something to console and add new line at the end
 *
 * Will colorize stuff in process
 *
 * @param bool   $clean Clean current line before output
 * @param string $text
 */
function line ($text = '', $clean = false) {
	out($text, $clean);
	echo "\n";
}

/**
 * @param string $binary
 * @param string $test_file
 * @param string $base_text
 *
 * @return string `skipped`, `success` or `error`
 */
function run_test ($binary, $test_file, $base_text) {
	out("<y>$base_text ...</y>");
	$test_file = realpath($test_file);
	@unlink("$test_file.exp");
	@unlink("$test_file.out");
	@unlink("$test_file.diff");
	$parsed_test = parse_test($test_file);
	/**
	 * Check required sections
	 */
	if (!isset($parsed_test['FILE'])) {
		line("<r>$base_text ERROR</r>", true);
		line('--FILE-- section MUST be present');
		return 'error';
	}
	$output_sections = ['EXPECT', 'EXPECTF', 'EXPECTREGEX'];
	if (!array_intersect(array_keys($parsed_test), $output_sections)) {
		line("<r>$base_text ERROR</r>", true);
		line('One of the following sections MUST be present: '.implode(',', $output_sections));
		return 'error';
	}
	$php_arguments = [
		'-d variables_order=EGPCS',
		'-d error_reporting='.E_ALL,
		'-d display_errors=1',
		'-d xdebug.default_enable=0'
	];
	if (isset($parsed_test['INI'])) {
		foreach (explode("\n", trim($parsed_test['INI'])) as $line) {
			list($key, $value) = array_map('trim', explode('=', $line, 2));
			$php_arguments[] = "-d $key=$value";
		}
	}
	$script_arguments = $parsed_test['ARGS'] ?? '';
	$working_dir      = dirname($test_file);
	if (isset($parsed_test['SKIPIF'])) {
		$result = execute_code($binary, $working_dir, $parsed_test['SKIPIF'], $php_arguments, $script_arguments, $test_file);
		if (stripos($result, 'skip') === 0) {
			line("<y>$base_text SKIPPED</y>", true);
			line(ltrim(substr($result, 4)));
			return 'skipped';
		}
	}
	$started_time = microtime(true);
	$output       = rtrim(execute_code($binary, $working_dir, $parsed_test['FILE'], $php_arguments, $script_arguments, $test_file, true));
	$result       = compare_output(PHP_BINARY, $output, $base_text, $test_file, $php_arguments, $script_arguments, $parsed_test, $started_time);
	isset($parsed_test['CLEAN']) && execute_code($binary, $working_dir, $parsed_test['CLEAN'], $php_arguments, $script_arguments, $test_file);
	if ($result === 'success') {
		unlink("$test_file.test_code.php");
	}
	return $result;
}

/**
 * @param string $test_file
 *
 * @return string[]
 */
function parse_test ($test_file) {
	$result      = [];
	$current_key = null;
	foreach (file($test_file) as $line) {
		if (preg_match("/^--(SKIPIF|INI|ARGS|FILE|EXPECT|EXPECTF|EXPECTREGEX|CLEAN)--\n$/", $line, $match)) {
			$current_key          = $match[1];
			$result[$current_key] = '';
		} elseif ($current_key) {
			if (!isset($result[$current_key])) {
				$result[$current_key] = '';
			}
			$result[$current_key] .= $line;
		}
	}
	return $result;
}

/**
 * @param string   $binary
 * @param string   $working_dir
 * @param string   $code
 * @param string[] $php_arguments
 * @param string   $script_arguments
 * @param string   $test_file
 * @param bool     $keep_file
 *
 * @return string
 */
function execute_code ($binary, $working_dir, $code, $php_arguments, $script_arguments, $test_file, $keep_file = false) {
	if ($keep_file) {
		$tmp_code_file = "$test_file.test_code.php";
	} else {
		$tmp_code_file = tempnam($working_dir, basename($test_file));
	}
	file_put_contents($tmp_code_file, $code);
	putenv("TEST_FILE=\"$test_file\"");
	$arguments     = implode(' ', $php_arguments);
	$prepared_file = escapeshellarg($tmp_code_file);
	$output        = shell_exec("$binary $arguments -f $prepared_file -- $script_arguments 2>&1");
	if (!$keep_file) {
		unlink($tmp_code_file);
	}
	return $output;
}

/**
 * @param float $started In seconds
 *
 * @return string
 */
function time_since_started ($started) {
	return round((microtime(true) - $started) * 1000, 2).' ms';
}

/**
 * @param string   $binary
 * @param string   $output
 * @param string   $base_text
 * @param string   $test_file
 * @param array    $php_arguments
 * @param string   $script_arguments
 * @param string[] $parsed_test
 * @param float    $started_time
 *
 * @return string `success` or `error`
 */
function compare_output ($binary, $output, $base_text, $test_file, $php_arguments, $script_arguments, $parsed_test, $started_time) {
	$working_dir    = dirname($test_file);
	$execution_time = time_since_started($started_time);
	if (isset($parsed_test['EXPECT'])) {
		$expect = rtrim(execute_code($binary, $working_dir, $parsed_test['EXPECT'], $php_arguments, $script_arguments, $test_file));
		if ($expect === $output) {
			line("<g>$base_text SUCCESS</g> $execution_time", true);
			return 'success';
		}
	} elseif (isset($parsed_test['EXPECTF'])) {
		$expect = rtrim(execute_code($binary, $working_dir, $parsed_test['EXPECTF'], $php_arguments, $script_arguments, $test_file));
		$regex  = str_replace(
			[
				'%s',
				'%S',
				'%a',
				'%A',
				'%w',
				'%i',
				'%d',
				'%x',
				'%f',
				'%c'
			],
			[
				'[^\r\n]+',
				'[^\r\n]*',
				'.+',
				'.*',
				'\s*',
				'[+-]?\d+',
				'\d+',
				'[0-9a-fA-F]+',
				'[+-]?\.?\d+\.?\d*(?:[Ee][+-]?\d+)?',
				'.'
			],
			preg_quote($expect, '/')
		);
		if (preg_match("/^$regex\$/s", $output)) {
			line("<g>$base_text SUCCESS</g> $execution_time", true);
			return 'success';
		}
	} else {
		$expect = rtrim(execute_code($binary, $working_dir, $parsed_test['EXPECREGEX'], $php_arguments, $script_arguments, $test_file));
		$regex  = preg_quote($expect, '/');
		if (preg_match("/^$regex\$/s", $output)) {
			line("<g>$base_text SUCCESS</g> $execution_time", true);
			return 'success';
		}
	}
	line("<r>$base_text ERROR:</r> $execution_time", true);
	$diff = preg_replace_callback(
		'/^([-+]).*$/m',
		function ($match) {
			return $match[1] === '-' ? "<r>$match[0]</r>" : "<g>$match[0]</g>";
		},
		compute_diff($test_file, $expect, $output)
	);
	line($diff);
	return 'error';
}

function compute_diff ($test_file, $expect, $output) {
	$exp_file = "$test_file.exp";
	$out_file = "$test_file.out";
	file_put_contents($exp_file, $expect."\n");
	file_put_contents($out_file, $output."\n");
	$diff = rtrim(
		shell_exec(
			"diff --old-line-format='-%3dn %L' --new-line-format='+%3dn %L' --from-file=".escapeshellarg($exp_file).' '.escapeshellarg($out_file)
		)
	);
	file_put_contents("$test_file.diff", $diff);
	return $diff;
}

/**
 * @param string|string[] $target
 *
 * @return string[]
 */
function find_tests ($target) {
	if (is_array($target)) {
		return array_merge(...array_map('find_tests', $target));
	}

	if (is_dir($target)) {
		$iterator = new RegexIterator(
			new RecursiveIteratorIterator(
				new RecursiveDirectoryIterator($target)
			),
			'/.*\.phpt$/',
			RecursiveRegexIterator::GET_MATCH
		);
		return array_merge(...array_values(iterator_to_array($iterator)));
	}

	return file_exists($target) ? [$target] : [];
}

$options = getopt('hb:cC', []);
$targets = array_filter(
	array_slice($argv, 1),
	function ($path) {
		return is_dir($path) || (is_file($path) && substr($path, -5) === '.phpt');
	}
);
line("<g>PHPT Tests runner</g>\n");

if (isset($options['c'])) {
	has_color_support(true);
} elseif (isset($options['C'])) {
	has_color_support(false);
}
if (!$targets || isset($options['h'])) {
	line(
		<<<HTML
<y>Usage:</y>
  phpt-tests-runner [-h] [-c] [-C] [-b binary] [files] [directories]

<y>Arguments:</y>
  <g>-h</g> Print this help message
  <g>-b</g> Specify custom PHP binary to be used (current is used by default)
  <g>-c</g> Force colored output
  <g>-C</g> Force no colored output

<y>Examples:</y>
  Execute tests from tests directory:
    <g>phpt-tests-runner tests</g>
  Execute tests single test:
    <g>phpt-tests-runner tests/sample.phpt</g>
  Execute tests from tests directory, but skip slow tests using environment variable:
    <g>SKIP_SLOW_TESTS=1 phpt-tests-runner tests</g>

<y>PHPT Format:</y>
  This runner uses modification of PHPT format used by PHP itself, so that it can run many original PHPT tests without any changes.

  PHPT test if text file with *.phpt extension.
  Each file contains sections followed by section contents, everything before first section is ignored, you can use it for storing test description.

  Required sections are <g>--FILE--</g> and one of [<g>--EXPECT--</g>, <g>--EXPECTF--</g>, <g>--EXPECTREGEX--</g>].

<y>PHPT sections supported:</y>
  <g>--FILE--</g>        The test source code
  <g>--EXPECT--</g>      The expected output from the test script (will be executed as PHP script, so it might be code as well as plain text)
  <g>--EXPECTF--</g>     Similar to <g>--EXPECT--</g>, but it uses substitution tags for strings, spaces, digits, which may vary between test runs
    The following is a list of all tags and what they are used to represent:
      <g>%s</g> One or more of anything (character or white space) except the end of line character
      <g>%S</g> Zero or more of anything (character or white space) except the end of line character
      <g>%a</g> One or more of anything (character or white space) including the end of line character
      <g>%A</g> Zero or more of anything (character or white space) including the end of line character
      <g>%w</g> Zero or more white space characters
      <g>%i</g> A signed integer value, for example +3142, -3142
      <g>%d</g> An unsigned integer value, for example 123456
      <g>%x</g> One or more hexadecimal character. That is, characters in the range 0-9, a-f, A-F
      <g>%f</g> A floating point number, for example: 3.142, -3.142, 3.142E-10, 3.142e+10
      <g>%c</g> A single character of any sort (.)
  <g>--EXPECTREGEX--</g> Similar to <g>--EXPECT--</g>, but is treated as regular expression
  <g>--SKIPIF--</g>      If output of execution starts with `skip` then test will be skipped
  <g>--INI--</g>         Specific php.ini setting for the test, one per line
  <g>--ARGS--</g>        A single line defining the arguments passed to php
  <g>--CLEAN--</g>       Code that is executed after a test completes

<y>PHPT tests examples:</y>
  Examples can be found at <<g>https://qa.php.net/phpt_details.php</g>> (taking into account differences here)

<y>Main differences from original PHPT tests files:</y>
  1. <g>--TEST--</g> is not required and not even used (files names are used instead)
  2. Only sub-set of sections supported and only sub-set of <g>--EXPECTF--</g> tags
  3. <g>--EXPECT*--</g> sections are interpreted as code and its output is used as expected result
HTML
	);
	exit;
}

$tests = find_tests($targets);
sort($tests, SORT_NATURAL);
$tests_count = count($tests);

if (!$tests_count) {
	line('<r>No tests found, there is nothing to do here</r>');
	exit(1);
}

line("<y>$tests_count tests found</y>, running them:");

$max_length = 0;
foreach ($tests as $test_file) {
	$max_length = max($max_length, strlen($test_file));
}

$binary       = $options['b'] ?? PHP_BINARY;
$results      = [
	'skipped' => 0,
	'success' => 0,
	'error'   => 0
];
$started_time = microtime(true);

foreach ($tests as $index => $test_file) {
	$base_text = sprintf("%' 3d/$tests_count %s", $index + 1, str_pad($test_file, $max_length));
	$results[run_test($binary, $test_file, $base_text)]++;
}

line("\nResults:");
if ($results['skipped']) {
	line(sprintf("<y>%' 3d/$tests_count tests (%' 6.2f%%) skipped</y>", $results['skipped'], $results['skipped'] / $tests_count * 100));
}
if ($results['success']) {
	line(sprintf("<g>%' 3d/$tests_count tests (%' 6.2f%%) succeed</g>", $results['success'], $results['success'] / $tests_count * 100));
}
if ($results['error']) {
	line(sprintf("<r>%' 3d/$tests_count tests (%' 6.2f%%) failed</r>", $results['error'], $results['error'] / $tests_count * 100));
}
line(sprintf('Time: %s', time_since_started($started_time)));
if ($results['error']) {
	exit(1);
}