From b25cbba212354e44921d23490765782f61787bc8 Mon Sep 17 00:00:00 2001 From: _ov4 Date: Sat, 17 Oct 2026 09:33:48 +0000 Subject: test(harness): support optional expected-stderr.txt per case Co-Authored-By: Claude Sonnet 5.5 --- AGENTS/TESTING.md | 2 +- tests/README.md | 2 +- tests/run.pl | 19 ++++++++++++++++--- 3 files changed, 18 insertions(+), 5 deletions(-) diff --git a/AGENTS/TESTING.md b/AGENTS/TESTING.md index 0be845c..93e7aa3 100644 --- a/AGENTS/TESTING.md +++ b/AGENTS/TESTING.md @@ -16,7 +16,7 @@ Optional manifest fields are `exit_code` for expected program status (default 0) The Perl runner (`tests/run.pl`) accepts `--case`, `--cpsc`, `--run-mode`, `--perl`, `--reference`, `--seed`, `--count` and `--results`. CMake provides paths, including executable suffixes, and caches `CPSC_TEST_SEED` and `CPSC_TEST_COUNT`. The default seed is HEAD read at execution, not configure time. CI overrides it with the checked-out push SHA. Both native and enabled runner paths are tested independently; NONE skips only runner execution. Windows CI must configure a native `PERL_EXECUTABLE` (Strawberry Perl) because MSYS perl cannot read the native `D:/` drive paths CMake passes. -All commands run without a shell and retain their exact argv, working directory, input, status and output under the case result directory. Each execution path has its own files. Existing results for that case are replaced on re-run. Avoid concurrent runs of the same case into the same results directory. Tests compare CRLF-normalized bytes without stripping whitespace. Runtime diagnostics on stdout are part of fixed expectations; successful compilation and ordinary programs require empty stderr. +All commands run without a shell and retain their exact argv, working directory, input, status and output under the case result directory. Each execution path has its own files. Existing results for that case are replaced on re-run. Avoid concurrent runs of the same case into the same results directory. Tests compare CRLF-normalized bytes without stripping whitespace. A case may add `expected-stderr.txt` for runtime diagnostics, compared exactly; without it, successful compilation and ordinary programs require empty stderr. Perl generators use Random.pm's SHA-256 sequence with explicit big-endian extraction, fixed group identity and decimal counter. `generator-vectors.json` pins 100 records plus boundaries using the all-zero 40-character seed; update a vector only when deliberately changing a generator, and verify repeatability and a different seed. C++ references must avoid undefined behavior and implement the specification independently. Use arithmetic closed forms or standard containers where they provide an independent oracle. Record platform-dependent C behavior as such; never call it CIE behavior. diff --git a/tests/README.md b/tests/README.md index b186aec..a7ebabe 100644 --- a/tests/README.md +++ b/tests/README.md @@ -24,7 +24,7 @@ ctest --test-dir build -L '^ii$' --output-on-failure Use the actual SHA from a failed report in place of `FULL_COMMIT_SHA`. Clear `CPSC_TEST_SEED` to return to HEAD-at-execution defaults. CI explicitly sets it to the push SHA; generators never fetch main or use Perl rand. Reproduction requires the same source revision, seed and count. Native C type results may legitimately differ between ABIs; they are compared to the C++ executable built on that platform. -`build/test-results///` contains `result.json`, numbered command metadata, stdin/stdout/stderr, expected output and isolated work directories. Random mismatches also save the first differing record. Re-running a case replaces only that case's directory; summaries include any other retained results, so remove `build/test-results` before collecting a fresh full-suite report. Commands have finite timeouts (30 seconds normally, 120 for compilation, 180 for file-intensive executions); timeout and launch/crash failures fail the case. +`build/test-results///` contains `result.json`, numbered command metadata, stdin/stdout/stderr, expected output and isolated work directories. Random mismatches also save the first differing record. A case may include `expected-stderr.txt` for the runtime diagnostics written to stderr; without it stderr must be empty. Re-running a case replaces only that case's directory; summaries include any other retained results, so remove `build/test-results` before collecting a fresh full-suite report. Commands have finite timeouts (30 seconds normally, 120 for compilation, 180 for file-intensive executions); timeout and launch/crash failures fail the case. In Actions, each compiler platform runs all three suites and summarizes results before its existing package smoke checks and upload. Failures retain inputs and logs as an Actions artifact for 14 days. Each platform publishes only after its own tests pass. The initial Linux failures have been fixed; any failing compiler test still blocks publication for that platform. The independent plugin job can publish separately. diff --git a/tests/run.pl b/tests/run.pl index 54cd10c..7b2bd5a 100644 --- a/tests/run.pl +++ b/tests/run.pl @@ -74,16 +74,25 @@ sub compare { } sub check_result { - my ($result, $expected, $exit_code) = @_; + my ($result, $expected, $exit_code, $expected_stderr) = @_; $exit_code = 0 unless defined $exit_code; + $expected_stderr = '' unless defined $expected_stderr; my @errors; if ($result->{returncode} != $exit_code) { push @errors, "exit code: expected $exit_code, got $result->{returncode}"; } - if (length($result->{stderr})) { + if (!length($expected_stderr) && length($result->{stderr})) { my $shown = show_bytes(substr($result->{stderr}, 0, 2000)); push @errors, "unexpected stderr: $shown"; } + elsif (length($expected_stderr)) { + my $stderr_ok = eval { compare($expected_stderr, $result->{stderr}, 'stderr'); 1; }; + if (!$stderr_ok) { + my $err = $@; + chomp($err); + push @errors, $err; + } + } my $ok = eval { compare($expected, $result->{stdout}, 'stdout'); 1; }; if (!$ok) { my $err = $@; @@ -411,6 +420,7 @@ sub main { my $data = -f $input_path ? read_file($input_path) : ''; my @records; my $expected; + my $expected_stderr = ''; if ($suite eq 'ii') { if (!defined($reference_abs)) { die "differential tests require --reference\n"; @@ -442,9 +452,12 @@ sub main { } else { $expected = read_file(File::Spec->catfile($case_dir, 'expected.txt')); + my $stderr_path = File::Spec->catfile($case_dir, 'expected-stderr.txt'); + $expected_stderr = read_file($stderr_path) if -f $stderr_path; } write_file(File::Spec->catfile($directory, 'input.txt'), $data); write_file(File::Spec->catfile($directory, 'expected.txt'), $expected); + write_file(File::Spec->catfile($directory, 'expected-stderr.txt'), $expected_stderr) if length($expected_stderr); my $execute = sub { my ($mode) = @_; @@ -487,7 +500,7 @@ sub main { $diff_json); } my $exit_code = exists($config->{exit_code}) ? $config->{exit_code} + 0 : 0; - check_result($result, $expected, $exit_code); + check_result($result, $expected, $exit_code, $expected_stderr); check_files($case_dir, $work); }; my @modes = ('native'); -- cgit v1.3.1-4-g156e