From 06529b7995b5e665b0b5cee2bd7ec83a16816eba Mon Sep 17 00:00:00 2001 From: _ov4 Date: Fri, 16 Oct 2026 17:47:27 +0000 Subject: test(harness): port summarize to Perl Co-Authored-By: Claude Fable 5.1 --- .github/workflows/nightly-build.yml | 6 ++-- tests/summarize.pl | 64 +++++++++++++++++++++++++++++++++++++ tests/summarize.py | 41 ------------------------ 3 files changed, 67 insertions(+), 44 deletions(-) create mode 100644 tests/summarize.pl delete mode 100644 tests/summarize.py diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 72c24fe..54c1444 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -79,7 +79,7 @@ jobs: - name: Summarize compiler tests if: ${{ !cancelled() && (steps.compiler-tests.outcome == 'success' || steps.compiler-tests.outcome == 'failure') }} - run: python3 tests/summarize.py build/test-results + run: perl tests/summarize.pl build/test-results - name: Retain failed compiler test inputs and logs if: ${{ failure() && steps.compiler-tests.outcome == 'failure' }} @@ -183,7 +183,7 @@ jobs: - name: Summarize compiler tests if: ${{ !cancelled() && (steps.compiler-tests.outcome == 'success' || steps.compiler-tests.outcome == 'failure') }} - run: python3 tests/summarize.py build/test-results + run: perl tests/summarize.pl build/test-results - name: Retain failed compiler test inputs and logs if: ${{ failure() && steps.compiler-tests.outcome == 'failure' }} @@ -320,7 +320,7 @@ jobs: - name: Summarize compiler tests if: ${{ !cancelled() && (steps.compiler-tests.outcome == 'success' || steps.compiler-tests.outcome == 'failure') }} shell: msys2 {0} - run: python3 tests/summarize.py build/test-results + run: perl tests/summarize.pl build/test-results - name: Retain failed compiler test inputs and logs if: ${{ failure() && steps.compiler-tests.outcome == 'failure' }} diff --git a/tests/summarize.pl b/tests/summarize.pl new file mode 100644 index 0000000..fff4a11 --- /dev/null +++ b/tests/summarize.pl @@ -0,0 +1,64 @@ +#!/usr/bin/env perl +# Summarize retained case results locally and in GitHub Actions. +use strict; +use warnings; +use File::Path qw(make_path); +use File::Spec; +use JSON::PP (); + +die "usage: $0 \n" unless @ARGV == 1; +my ($directory) = @ARGV; + +my $summary = summarize($directory); +print $summary; +make_path($directory); +my $summary_path = File::Spec->catfile($directory, 'summary.md'); +open my $out, '>:raw', $summary_path or die "cannot write $summary_path: $!\n"; +print {$out} $summary; +close $out; +if (defined $ENV{GITHUB_STEP_SUMMARY} && length $ENV{GITHUB_STEP_SUMMARY}) { + open my $step, '>>:raw', $ENV{GITHUB_STEP_SUMMARY} + or die "cannot append to \$GITHUB_STEP_SUMMARY: $!\n"; + print {$step} $summary; + close $step; +} + +sub summarize { + my ($dir) = @_; + my @paths = sort glob(File::Spec->catfile($dir, '*', '*', 'result.json')); + my @results; + for my $path (@paths) { + next unless -f $path; + open my $fh, '<:raw', $path or die "cannot read $path: $!\n"; + my $text = do { local $/; <$fh> }; + close $fh; + push @results, JSON::PP->new->decode($text); + } + if (!@results) { + return "No compiler case results were produced; inspect the build/CTest log.\n"; + } + my @failures = grep { !$_->{passed} } @results; + my @lines = ( + sprintf('Compiler cases: %d/%d passed.', scalar(@results) - scalar(@failures), scalar(@results)), + '', + '| Group | Passed | Total |', + '|---|---:|---:|', + ); + for my $suite (qw(i ii iii)) { + my @subset = grep { $_->{case} =~ m{^\Q$suite\E/} } @results; + my $passed = grep { $_->{passed} } @subset; + push @lines, sprintf('| %s | %d | %d |', $suite, $passed, scalar(@subset)); + } + push @lines, '', + sprintf('Commit: `%s`. Seed: `%s`.', $results[0]{commit}, $results[0]{seed}), + 'Fixed expectations: pending human review.'; + if (@failures) { + push @lines, '', 'Failures:', ''; + for my $result (@failures) { + my $stages = join(', ', map { $_->{stage} } @{$result->{failures}}); + push @lines, sprintf('- `%s` (%s); see its `result.json` and command logs.', + $result->{case}, $stages); + } + } + return join("\n", @lines) . "\n"; +} diff --git a/tests/summarize.py b/tests/summarize.py deleted file mode 100644 index 5006dc0..0000000 --- a/tests/summarize.py +++ /dev/null @@ -1,41 +0,0 @@ -#!/usr/bin/env python3 -"""Summarize retained case results locally and in GitHub Actions.""" - -import argparse -import json -import os -from pathlib import Path - - -def summarize(directory): - results = [json.loads(path.read_text(encoding="utf-8")) - for path in sorted(directory.glob("*/*/result.json"))] - if not results: - return "No compiler case results were produced; inspect the build/CTest log.\n" - failures = [result for result in results if not result["passed"]] - lines = [f"Compiler cases: {len(results) - len(failures)}/{len(results)} passed.", "", - "| Group | Passed | Total |", "|---|---:|---:|"] - for suite in ["i", "ii", "iii"]: - subset = [result for result in results if result["case"].startswith(suite + "/")] - lines.append(f"| {suite} | {sum(result['passed'] for result in subset)} | {len(subset)} |") - lines += ["", f"Commit: `{results[0]['commit']}`. Seed: `{results[0]['seed']}`.", - "Fixed expectations: pending human review."] - if failures: - lines += ["", "Failures:", ""] - for result in failures: - stages = ", ".join(failure["stage"] for failure in result["failures"]) - lines.append(f"- `{result['case']}` ({stages}); see its `result.json` and command logs.") - return "\n".join(lines) + "\n" - - -if __name__ == "__main__": - parser = argparse.ArgumentParser(description=__doc__) - parser.add_argument("directory", type=Path) - args = parser.parse_args() - summary = summarize(args.directory) - print(summary, end="") - args.directory.mkdir(parents=True, exist_ok=True) - (args.directory / "summary.md").write_text(summary, encoding="utf-8") - if os.environ.get("GITHUB_STEP_SUMMARY"): - with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as output: - output.write(summary) -- cgit v1.3.1-4-g156e