From b25cbba212354e44921d23490765782f61787bc8 Mon Sep 17 00:00:00 2001
From: _ov4 <ov4@ov4.org>
Date: Sat, 17 Oct 2026 09:33:48 +0000
Subject: test(harness): support optional expected-stderr.txt per case

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
---
 tests/README.md |  2 +-
 tests/run.pl    | 19 ++++++++++++++++---
 2 files changed, 17 insertions(+), 4 deletions(-)

(limited to 'tests')

diff --git a/tests/README.md b/tests/README.md
index b186aec..a7ebabe 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -24,7 +24,7 @@ ctest --test-dir build -L '^ii$' --output-on-failure
 
 Use the actual SHA from a failed report in place of `FULL_COMMIT_SHA`. Clear `CPSC_TEST_SEED` to return to HEAD-at-execution defaults. CI explicitly sets it to the push SHA; generators never fetch main or use Perl rand. Reproduction requires the same source revision, seed and count. Native C type results may legitimately differ between ABIs; they are compared to the C++ executable built on that platform.
 
-`build/test-results/<suite>/<case>/` contains `result.json`, numbered command metadata, stdin/stdout/stderr, expected output and isolated work directories. Random mismatches also save the first differing record. Re-running a case replaces only that case's directory; summaries include any other retained results, so remove `build/test-results` before collecting a fresh full-suite report. Commands have finite timeouts (30 seconds normally, 120 for compilation, 180 for file-intensive executions); timeout and launch/crash failures fail the case.
+`build/test-results/<suite>/<case>/` contains `result.json`, numbered command metadata, stdin/stdout/stderr, expected output and isolated work directories. Random mismatches also save the first differing record. A case may include `expected-stderr.txt` for the runtime diagnostics written to stderr; without it stderr must be empty. Re-running a case replaces only that case's directory; summaries include any other retained results, so remove `build/test-results` before collecting a fresh full-suite report. Commands have finite timeouts (30 seconds normally, 120 for compilation, 180 for file-intensive executions); timeout and launch/crash failures fail the case.
 
 In Actions, each compiler platform runs all three suites and summarizes results before its existing package smoke checks and upload. Failures retain inputs and logs as an Actions artifact for 14 days. Each platform publishes only after its own tests pass. The initial Linux failures have been fixed; any failing compiler test still blocks publication for that platform. The independent plugin job can publish separately.
 
diff --git a/tests/run.pl b/tests/run.pl
index 54cd10c..7b2bd5a 100644
--- a/tests/run.pl
+++ b/tests/run.pl
@@ -74,16 +74,25 @@ sub compare {
 }
 
 sub check_result {
-    my ($result, $expected, $exit_code) = @_;
+    my ($result, $expected, $exit_code, $expected_stderr) = @_;
     $exit_code = 0 unless defined $exit_code;
+    $expected_stderr = '' unless defined $expected_stderr;
     my @errors;
     if ($result->{returncode} != $exit_code) {
         push @errors, "exit code: expected $exit_code, got $result->{returncode}";
     }
-    if (length($result->{stderr})) {
+    if (!length($expected_stderr) && length($result->{stderr})) {
         my $shown = show_bytes(substr($result->{stderr}, 0, 2000));
         push @errors, "unexpected stderr: $shown";
     }
+    elsif (length($expected_stderr)) {
+        my $stderr_ok = eval { compare($expected_stderr, $result->{stderr}, 'stderr'); 1; };
+        if (!$stderr_ok) {
+            my $err = $@;
+            chomp($err);
+            push @errors, $err;
+        }
+    }
     my $ok = eval { compare($expected, $result->{stdout}, 'stdout'); 1; };
     if (!$ok) {
         my $err = $@;
@@ -411,6 +420,7 @@ sub main {
             my $data = -f $input_path ? read_file($input_path) : '';
             my @records;
             my $expected;
+            my $expected_stderr = '';
             if ($suite eq 'ii') {
                 if (!defined($reference_abs)) {
                     die "differential tests require --reference\n";
@@ -442,9 +452,12 @@ sub main {
             }
             else {
                 $expected = read_file(File::Spec->catfile($case_dir, 'expected.txt'));
+                my $stderr_path = File::Spec->catfile($case_dir, 'expected-stderr.txt');
+                $expected_stderr = read_file($stderr_path) if -f $stderr_path;
             }
             write_file(File::Spec->catfile($directory, 'input.txt'), $data);
             write_file(File::Spec->catfile($directory, 'expected.txt'), $expected);
+            write_file(File::Spec->catfile($directory, 'expected-stderr.txt'), $expected_stderr) if length($expected_stderr);
 
             my $execute = sub {
                 my ($mode) = @_;
@@ -487,7 +500,7 @@ sub main {
                         $diff_json);
                 }
                 my $exit_code = exists($config->{exit_code}) ? $config->{exit_code} + 0 : 0;
-                check_result($result, $expected, $exit_code);
+                check_result($result, $expected, $exit_code, $expected_stderr);
                 check_files($case_dir, $work);
             };
             my @modes = ('native');
-- 
cgit v1.3.1-4-g156e