From 5b55b38f0568e7514a3cda6a8d8698cd0aab813b Mon Sep 17 00:00:00 2001 From: _ov4 Date: Fri, 16 Oct 2026 17:50:19 +0000 Subject: test(harness): port harness self-tests to Perl Co-Authored-By: Claude Fable 5.1 --- tests/CMakeLists.txt | 4 ++-- tests/test_runner.pl | 64 ++++++++++++++++++++++++++++++++++++++++++++++++++++ tests/test_runner.py | 44 ------------------------------------ 3 files changed, 66 insertions(+), 46 deletions(-) create mode 100644 tests/test_runner.pl delete mode 100644 tests/test_runner.py (limited to 'tests') diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 9f755a6..1a20f38 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -3,8 +3,8 @@ find_package(Perl REQUIRED) set(CPSC_TEST_SEED "" CACHE STRING "Replay seed (defaults to HEAD at test execution)") set(CPSC_TEST_COUNT 10000 CACHE STRING "Random records per differential group") -add_test(NAME harness COMMAND "${Python3_EXECUTABLE}" - "${CMAKE_CURRENT_SOURCE_DIR}/test_runner.py") +add_test(NAME harness COMMAND "${PERL_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/test_runner.pl") add_test(NAME generators COMMAND "${PERL_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/check_generators.pl" "${PERL_EXECUTABLE}") foreach(suite i ii iii) diff --git a/tests/test_runner.pl b/tests/test_runner.pl new file mode 100644 index 0000000..af08556 --- /dev/null +++ b/tests/test_runner.pl @@ -0,0 +1,64 @@ +#!/usr/bin/env perl +# Small contract checks for the test harness, including intentional failures. +use strict; +use warnings; +use File::Spec; +use File::Temp qw(tempdir); +use FindBin; +use JSON::PP (); +use Test::More; + +require File::Spec->catfile($FindBin::Bin, 'run.pl'); + +eval { compare("a\nb\n", "a\r\nb\r\n", 'stdout'); }; +ok(!$@, 'CRLF-only difference is accepted'); +for my $bad ("a\nb", "a \nb\n", "A\nb\n", "a\n\nb\n") { + my $ok = eval { compare("a\nb\n", $bad, 'stdout'); 1; }; + ok(!$ok, 'strict output rejects ' . show_bytes($bad)); +} + +for my $case ([1, ''], [-11, ''], [0, 'unexpected']) { + my ($code, $stderr) = @{$case}; + my $ok = eval { + check_result({returncode => $code, stdout => "ok\n", stderr => $stderr}, "ok\n"); + 1; + }; + ok(!$ok, "exit code $code with stderr " . show_bytes($stderr) . ' fails'); +} + +my $good = { + returncode => 1, + stdout => '', + stderr => "error: Expected ':' at line 3\ncompilation terminated.\n", +}; +my $accepted = eval { + check_diagnostics($good, [{reason => "Expected ':'", line => 3}]); + 1; +}; +ok($accepted, 'matching reason and location accepted'); +for my $bad ( + "error: Expected ':' at line 4", + "error: Expected ':'", + "error: Expected ':' at line 30\nerror: unrelated at line 3", + ) +{ + my $ok = eval { + check_diagnostics( + {returncode => 1, stdout => '', stderr => $bad . "\ncompilation terminated.\n"}, + [{reason => "Expected ':'", line => 3}], + ); + 1; + }; + ok(!$ok, 'diagnostic mismatch rejected: ' . show_bytes($bad)); +} + +my $dir = tempdir(CLEANUP => 1); +my $state = commands_new($dir); +my $ran = eval { run_command($state, [$^X, '-e', 'sleep 5'], $dir, '', 1); 1; }; +my $timed_out = "$@"; +ok(!$ran, 'overlong command fails'); +like($timed_out, qr/timed out/, 'timeout message names the cause'); +my $metadata = JSON::PP->new->decode(read_file(File::Spec->catfile($dir, '01.json'))); +ok($metadata->{timeout_expired}, 'timeout recorded in command metadata'); + +done_testing(); diff --git a/tests/test_runner.py b/tests/test_runner.py deleted file mode 100644 index 9558175..0000000 --- a/tests/test_runner.py +++ /dev/null @@ -1,44 +0,0 @@ -"""Small contract checks for the test harness, including intentional failures.""" - -from pathlib import Path -import subprocess -import sys -import tempfile -import unittest - -from run import Commands, Failure, check_diagnostics, check_result, compare - - -class HarnessTests(unittest.TestCase): - def test_strict_output(self): - compare(b"a\nb\n", b"a\r\nb\r\n", "stdout") - for actual in [b"a\nb", b"a \nb\n", b"A\nb\n", b"a\n\nb\n"]: - with self.assertRaises(Failure): - compare(b"a\nb\n", actual, "stdout") - - def test_exit_and_stderr(self): - for code, stderr in [(1, b""), (-11, b""), (0, b"unexpected")]: - with self.assertRaises(Failure): - check_result(subprocess.CompletedProcess([], code, b"ok\n", stderr), b"ok\n") - - def test_reason_and_location_must_share_a_diagnostic(self): - expected = [{"reason": "Expected ':'", "line": 3}] - check_diagnostics(subprocess.CompletedProcess([], 1, b"", - b"error: Expected ':' at line 3\ncompilation terminated.\n"), expected) - for diagnostic in [b"error: Expected ':' at line 4", b"error: Expected ':'", - b"error: Expected ':' at line 30\nerror: unrelated at line 3"]: - with self.assertRaises(Failure): - check_diagnostics(subprocess.CompletedProcess([], 1, b"", - diagnostic + b"\ncompilation terminated.\n"), expected) - - def test_timeout_is_failure_and_saved(self): - with tempfile.TemporaryDirectory() as temporary: - directory = Path(temporary) - with self.assertRaisesRegex(Failure, "timed out"): - Commands(directory).run([sys.executable, "-c", "import time; time.sleep(5)"], - directory, timeout=0.05) - self.assertIn('"timeout_expired": true', (directory / "01.json").read_text()) - - -if __name__ == "__main__": - unittest.main() -- cgit v1.3.1-4-g156e