From 670b0d8a4669d939b09b4990be89173805831215 Mon Sep 17 00:00:00 2001 From: _ov4 Date: Fri, 16 Oct 2026 20:54:00 +0800 Subject: ci(compiler): gate platform packages on reproducible compiler tests Run all three suites on Linux, macOS and Windows with the push SHA as seed. Retain failure evidence and document the 12 existing failures, pending human review and remaining coverage gaps. Co-Authored-By: Claude Fable 5.1 --- .github/workflows/nightly-build.yml | 67 +++++++++++++++++++++++++++++++++++-- AGENTS/ARCH.md | 6 ++-- AGENTS/PITFALLS.md | 9 ++--- AGENTS/TESTING.md | 15 +++++++++ README.md | 2 ++ doc/build.md | 3 ++ tests/CMakeLists.txt | 3 ++ tests/FAILURES.md | 32 ++++++++++++++++++ tests/README.md | 31 +++++++++++++++++ tests/check_generators.py | 26 ++++++++++++++ tests/ii/arithmetic/notes.md | 2 ++ tests/ii/arithmetic/program.cps | 18 +++++++++- tests/ii/arithmetic/reference.cc | 5 ++- tests/ii/generator-vectors.json | 9 +++++ tests/summarize.py | 41 +++++++++++++++++++++++ 15 files changed, 259 insertions(+), 10 deletions(-) create mode 100644 AGENTS/TESTING.md create mode 100644 tests/FAILURES.md create mode 100644 tests/README.md create mode 100644 tests/check_generators.py create mode 100644 tests/ii/generator-vectors.json create mode 100644 tests/summarize.py diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml index 5a3ec67..72c24fe 100644 --- a/.github/workflows/nightly-build.yml +++ b/.github/workflows/nightly-build.yml @@ -60,17 +60,37 @@ jobs: sudo apt-get update sudo apt-get install -y --no-install-recommends \ build-essential cmake clang-23 llvm-23-dev lld-23 \ - libclang-23-dev liblld-23-dev libpolly-23-dev python3 \ + libclang-23-dev liblld-23-dev libpolly-23-dev python3 perl \ zlib1g-dev libzstd-dev libncurses-dev libedit-dev pandoc - name: Build the embedded release package run: | cmake -S . -B build \ -DCMAKE_BUILD_TYPE=Release -DCPSC_RUN_MODE=EMBEDDED \ + -DBUILD_TESTING=ON -DCPSC_TEST_SEED="$GITHUB_SHA" \ -DCMAKE_C_COMPILER=clang-23 -DCMAKE_CXX_COMPILER=clang++-23 \ -DLLVM_DIR=/usr/lib/llvm-23/lib/cmake/llvm + cmake --build build --parallel 4 cmake --build build --target release --parallel 4 + - name: Test compiler semantics, differential output and diagnostics + id: compiler-tests + run: ctest --test-dir build --output-on-failure --parallel 2 + + - name: Summarize compiler tests + if: ${{ !cancelled() && (steps.compiler-tests.outcome == 'success' || steps.compiler-tests.outcome == 'failure') }} + run: python3 tests/summarize.py build/test-results + + - name: Retain failed compiler test inputs and logs + if: ${{ failure() && steps.compiler-tests.outcome == 'failure' }} + uses: actions/upload-artifact@v7.0.2 + with: + name: compiler-tests-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} + path: | + build/test-results/ + build/Testing/Temporary/ + retention-days: 14 + - name: Smoke test the packaged compiler run: | packages=(build/cpsc-*-Linux-x86_64.tar.xz) @@ -124,7 +144,7 @@ jobs: run: | test "$(uname -m)" = arm64 brew update - brew install llvm@23 pandoc zlib zstd ncurses python + brew install llvm@23 pandoc zlib zstd ncurses python perl brew upgrade llvm@23 llvm_prefix=$(brew --prefix llvm@23) test -x "$llvm_prefix/bin/clang++" @@ -147,14 +167,34 @@ jobs: ncurses_prefix=$(brew --prefix ncurses) cmake -S . -B build \ -DCMAKE_BUILD_TYPE=Release -DCPSC_RUN_MODE=EMBEDDED \ + -DBUILD_TESTING=ON -DCPSC_TEST_SEED="$GITHUB_SHA" \ -DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_OSX_DEPLOYMENT_TARGET=15.0 \ -DCMAKE_OSX_SYSROOT="$(xcrun --show-sdk-path)" \ -DCMAKE_C_COMPILER="$llvm_prefix/bin/clang" \ -DCMAKE_CXX_COMPILER="$llvm_prefix/bin/clang++" \ -DLLVM_DIR="$llvm_prefix/lib/cmake/llvm" \ -DCMAKE_PREFIX_PATH="$llvm_prefix;$zlib_prefix;$zstd_prefix;$ncurses_prefix" + cmake --build build --parallel 3 cmake --build build --target release --parallel 3 + - name: Test compiler semantics, differential output and diagnostics + id: compiler-tests + run: ctest --test-dir build --output-on-failure --parallel 2 + + - name: Summarize compiler tests + if: ${{ !cancelled() && (steps.compiler-tests.outcome == 'success' || steps.compiler-tests.outcome == 'failure') }} + run: python3 tests/summarize.py build/test-results + + - name: Retain failed compiler test inputs and logs + if: ${{ failure() && steps.compiler-tests.outcome == 'failure' }} + uses: actions/upload-artifact@v7.0.2 + with: + name: compiler-tests-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} + path: | + build/test-results/ + build/Testing/Temporary/ + retention-days: 14 + - name: Smoke test the packaged compiler run: | packages=(build/cpsc-*-Darwin-arm64.tar.xz) @@ -217,6 +257,7 @@ jobs: install: >- curl git + perl mingw-w64-x86_64-cmake mingw-w64-x86_64-make mingw-w64-x86_64-clang @@ -243,10 +284,12 @@ jobs: run: | cmake -S . -B build -G "MinGW Makefiles" \ -DCMAKE_BUILD_TYPE=Release -DCPSC_RUN_MODE=EMBEDDED \ + -DBUILD_TESTING=ON -DCPSC_TEST_SEED="$GITHUB_SHA" \ -DCMAKE_C_COMPILER="$(cygpath -m /mingw64/bin/clang.exe)" \ -DCMAKE_CXX_COMPILER="$(cygpath -m /mingw64/bin/clang++.exe)" \ -DLLVM_DIR="$(cygpath -m /mingw64/lib/cmake/llvm)" \ -DCMAKE_PREFIX_PATH="$(cygpath -m /mingw64)" + cmake --build build --parallel 4 cmake --build build --target release --parallel 4 - name: Check Windows runtime dependencies @@ -269,6 +312,26 @@ jobs: esac done <<< "$imports" + - name: Test compiler semantics, differential output and diagnostics + id: compiler-tests + shell: msys2 {0} + run: ctest --test-dir build --output-on-failure --parallel 2 + + - name: Summarize compiler tests + if: ${{ !cancelled() && (steps.compiler-tests.outcome == 'success' || steps.compiler-tests.outcome == 'failure') }} + shell: msys2 {0} + run: python3 tests/summarize.py build/test-results + + - name: Retain failed compiler test inputs and logs + if: ${{ failure() && steps.compiler-tests.outcome == 'failure' }} + uses: actions/upload-artifact@v7.0.2 + with: + name: compiler-tests-${{ runner.os }}-${{ runner.arch }}-${{ github.sha }} + path: | + build/test-results/ + build/Testing/Temporary/ + retention-days: 14 + - name: Smoke test the packaged compiler shell: pwsh run: | diff --git a/AGENTS/ARCH.md b/AGENTS/ARCH.md index ffde252..04edd24 100644 --- a/AGENTS/ARCH.md +++ b/AGENTS/ARCH.md @@ -245,8 +245,10 @@ Windows system directories, excluding MSYS2 and LLVM. The macOS check also verifies arm64 and rejects non-system dynamic libraries. The plugin uses Node.js 22, `npm ci --ignore-scripts` and `npm run package`, then checks VSIX contents. Each job uploads directly after -its own checks pass; failure in one does not block the others. No Actions -artifacts are uploaded. +its own checks pass; failure in one does not block the others. Compiler jobs enable +BUILD_TESTING, install Perl and run all CTest cases with the push SHA as seed before +their package smoke checks and uploads. Failed tests retain inputs and logs as +Actions artifacts for 14 days; test summaries appear even on failure. See TESTING.md. Cleanup and uploads check main's current SHA to avoid modifying nightly for a superseded push. The workflow uses `GITHUB_TOKEN` with `contents: write`. Compiler package names retain the existing commit-based version; the VSIX uses diff --git a/AGENTS/PITFALLS.md b/AGENTS/PITFALLS.md index 58934ce..16fa16d 100644 --- a/AGENTS/PITFALLS.md +++ b/AGENTS/PITFALLS.md @@ -108,7 +108,8 @@ random files (OPENFILE...FOR RANDOM/SEEK/GETRECORD/PUTRECORD — TYPE is done, s are unblocked now; the RANDOM keyword is reserved and gives a dedicated parse error), DATE, INPUT into array elements, string escape sequences, array-typed record fields, whole-array RETURN, class methods/constructors/destructors, -NEW and class inheritance. There is no dedicated compiler test suite; nightly CI -smoke-tests the packaged EMBEDDED compiler and runs the existing plugin tests. -Verify compiler changes by compiling sample programs end-to-end (command -sequence in `ARCH.md`). +NEW and class inheritance. Compiler fixtures now live in `tests/` (see TESTING.md). +Nightly CI runs the compiler suites, packaged EMBEDDED smoke checks and existing +plugin tests. The initial compiler suite intentionally exposes existing standard +conflicts and missing diagnostic locations; see `tests/FAILURES.md` before interpreting +a failing baseline. Keep those failures visible until their implementations are fixed. diff --git a/AGENTS/TESTING.md b/AGENTS/TESTING.md new file mode 100644 index 0000000..7403aa1 --- /dev/null +++ b/AGENTS/TESTING.md @@ -0,0 +1,15 @@ +# Compiler testing + +Compiler tests live in `tests/`; run instructions and findings are in its README and FAILURES files. `BUILD_TESTING` defaults to OFF and adds a Perl dependency only when enabled. Python is already a compiler build dependency. CMake builds independent C++ reference executables with the same host ABI and registers one CTest entry per `case.json`, plus harness and generator checks. + +Each case directory has `program.cps` and `case.json`. Suite i adds `expected.txt`, optional `input.txt`, input files under `files/`, expected text-file contents under `expected-files/`, and independently written trace notes with human-review status. Suite ii adds `reference.cc` and `generate.pl`; the generator accepts seed and random count, emits the total record count followed by one ASCII record per line, and each program emits one result line per record. Suite iii stores an `errors` list of `{reason: regex, line: integer}` in its manifest. A cause and its line must appear on the same error diagnostic. EOF after a final newline is the following source line. + +Optional manifest fields are `exit_code` for expected program status (default 0) and `timeout` for execution/reference seconds (default 30; file-intensive cases use 180). Compilation has a 120-second timeout. Do not make a known compiler defect pass by changing the expectation, adding an exemption or comparing only a permissive substring of program output. + +The Python runner accepts `--case`, `--cpsc`, `--run-mode`, `--perl`, `--reference`, `--seed`, `--count` and `--results`. CMake provides paths, including executable suffixes, and caches `CPSC_TEST_SEED` and `CPSC_TEST_COUNT`. The default seed is HEAD read at execution, not configure time. CI overrides it with the checked-out push SHA. Both native and enabled runner paths are tested independently; NONE skips only runner execution. + +All commands run without a shell and retain their exact argv, working directory, input, status and output under the case result directory. Each execution path has its own files. Existing results for that case are replaced on re-run. Avoid concurrent runs of the same case into the same results directory. Tests compare CRLF-normalized bytes without stripping whitespace. Runtime diagnostics on stdout are part of fixed expectations; successful compilation and ordinary programs require empty stderr. + +Perl generators use Random.pm's SHA-256 sequence with explicit big-endian extraction, fixed group identity and decimal counter. `generator-vectors.json` pins 100 records plus boundaries using the all-zero 40-character seed; update a vector only when deliberately changing a generator, and verify repeatability and a different seed. C++ references must avoid undefined behavior and implement the specification independently. Use arithmetic closed forms or standard containers where they provide an independent oracle. Record platform-dependent C behavior as such; never call it CIE behavior. + +Before committing a test change, build the affected references and run the affected CTest cases. Compiler defects may fail: inspect retained diagnostics and document the failure accurately. Before committing workflow changes, run actionlint and retain the existing release API simulation/package smoke checks without contacting the real release. No compiler implementation changes belong in a tests-only task. diff --git a/README.md b/README.md index 4c20861..e17897c 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,8 @@ A compiler for CIE Pseudocode (A Level 9618 Computer Science) [Usage quick reference](doc/cpsc.1.md) · [Build guide](doc/build.md) · [Installation and releases](doc/install.md) +[Compiler tests](tests/README.md) · [Known test failures](tests/FAILURES.md) + Download the Linux x86_64, Windows x86_64 or macOS 15+ Apple Silicon compiler package and VS Code extension from [Nightly Build](https://github.com/ovfor4/cie-pseudocode-compiler/releases/tag/nightly-build). The compiler uses EMBEDDED mode and needs no LLVM installation. Each push to `main` removes the previous attachments before rebuilding all four packages; each package appears when its build and checks finish. Details can be found here: [Pseudocode Guide for Teachers](https://www.cambridgeinternational.org/Images/697401-2026-pseudocode-guide-for-teachers.pdf) diff --git a/doc/build.md b/doc/build.md index 5aaff51..65390ed 100644 --- a/doc/build.md +++ b/doc/build.md @@ -31,6 +31,9 @@ If you use a different CMake build directory, download into its `deps/packages/` instead. Removing the build directory also removes its downloaded dependencies. All three execution modes need LLVM, Clang and LLD 23 development libraries, Python 3.9 or newer, and the native C/C++ development libraries to build, including `NONE`. Users of an `EMBEDDED` executable do not need LLVM or `lli`. + +Compiler tests are optional (`-DBUILD_TESTING=ON`) and additionally need Perl with its core Digest::SHA module. See [the test instructions](../tests/README.md) for running CTest and replaying random inputs. CI enables the complete suite on every compiler platform. + CMake strictly requires LLVM development libraries and both C/C++ compilers to have major version 23. The compilers must be upstream Clang; GCC and AppleClang are rejected. Any 23.x release is accepted; 22 and 24 are rejected. LLVM/Clang diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4d385b9..630d363 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,9 +1,12 @@ +cpsc_find_dependency(PROGRAM PERL_EXECUTABLE NAMES perl REQUIRED) find_package(Perl REQUIRED) set(CPSC_TEST_SEED "" CACHE STRING "Replay seed (defaults to HEAD at test execution)") set(CPSC_TEST_COUNT 10000 CACHE STRING "Random records per differential group") add_test(NAME harness COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/test_runner.py") +add_test(NAME generators COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/check_generators.py" "${PERL_EXECUTABLE}") foreach(suite i ii iii) file(GLOB cases CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${suite}/*/case.json") foreach(manifest IN LISTS cases) diff --git a/tests/FAILURES.md b/tests/FAILURES.md new file mode 100644 index 0000000..3890e3b --- /dev/null +++ b/tests/FAILURES.md @@ -0,0 +1,32 @@ +# Compiler test findings + +Baseline: Linux x86_64, Clang/LLVM 23.1.3, Release EMBEDDED, compiler sources at `e2c6570`. Tests do not modify compiler sources. The full suite has 45 compiler cases plus harness and portable-generator checks. Fixed output expectations are pending human review. + +## Observed failures + +| Case | Required behavior | Current behavior | +|---|---|---| +| i/class-default-public | Omitted access defaults to public (CIE p. 28). | Rejects the field unless PUBLIC/PRIVATE is explicit. | +| i/identifier-case | Identifier lookup is case insensitive (CIE p. 7). | Reports the differently cased variable as unknown. | +| i/standard-while | Accept WHILE condition without DO (CIE p. 21). | Requires DO. | +| iii/argument-count | Wrong argument count, line 4. | Correct cause, no line. | +| iii/builtin-arity | LENGTH arity, line 2. | Correct cause, no line. | +| iii/byref-literal | Non-variable BYREF argument, line 4. | Correct cause, no line. | +| iii/constant-write | Assignment to constant, line 2. | Correct cause, no line. | +| iii/missing-endif | Missing ENDIF at EOF, line 4. | Correct cause, no line. | +| iii/missing-parenthesis | Missing closing parenthesis at EOF, line 3. | Correct cause, no line. | +| iii/missing-then | Missing THEN, line 2. | Correct cause, no line. | +| iii/nominal-type-mismatch | Incompatible named pointers, line 7. | Correct cause, no line. | +| iii/unknown-name | Undeclared name, line 2. | Correct cause, no line. | + +Both native and `-r` compilation expose the three i-group failures. All nine iii-group failures occur with both native/IR output requests, whether the destination was absent or already contained a sentinel. Failed compilation preserves the destination in these cases. The remaining 33 compiler cases passed locally, including seven groups of 10,000 random records plus 36 boundary records (70,036 total inputs, each checked through native and embedded execution). + +The baseline test seed and exact command results are retained in `build/test-results`; subsequent CI reports use their own explicit push SHA. Platform results from Actions are the authority for Windows/macOS verification; local Linux results do not establish their behavior. + +## Coverage gaps + +- Not implemented by the compiler, and not required to pass this initial suite: DATE, SET/DEFINE, random files, class methods/constructors/inheritance, INT/RAND, whole-array assignment and INPUT into array/record elements. Rejection of valid CIE syntax must not be relabeled as a successful negative test. +- The guide's p. 14 pointer example points an INTEGER pointer at an enum; it conflicts with the project's explicit strict-pointee contract. Tests use matching pointees rather than silently choosing a new policy. +- LCASE/UCASE are CHAR-to-CHAR in the guide (p. 17), while the project declares STRING results and also supports whole strings. Output observations cover case conversion but do not prove exact static return-type conformance. +- No requirements are invented for unspecified integer overflow, REAL output spelling, out-of-range substring behavior, Unicode character semantics, or uninitialized CIE values. Native C ABI behavior is checked only against that platform's independent C++ reference. +- This is input differential testing of seven fixed programs, not random-program fuzzing. It cannot establish complete parser or optimizer correctness. New bug reports should become small permanent fixed cases. diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..9024191 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,31 @@ +# Compiler tests + +Enable tests in an existing configured build (Perl with core Digest::SHA is additionally required): + +```sh +cmake -S . -B build -DBUILD_TESTING=ON +cmake --build build --parallel 2 +ctest --test-dir build --output-on-failure --parallel 2 +python3 tests/summarize.py build/test-results +``` + +Use `ctest --test-dir build -L '^ii$' --output-on-failure` for one suite, or `-R '^i/arrays$'` for one case. CTest returns failure if any case fails; it continues through the other cases. [Current failures and coverage gaps](FAILURES.md) are intentionally visible, with no expected-failure exemptions. + +The suites are **i**: independently traced fixed outputs, **ii**: C++ differential tests with Perl inputs, and **iii**: rejected programs with required causes and source lines. Fixed expectations are **pending human review**; each case's notes explain the trace and distinguish CIE rules from project extensions. REAL calculations are observed through BOOLEAN/INTEGER output; REAL text and REAL-to-string formatting remain outside this suite. + +The default is 10,000 random records per differential group, plus fixed boundary records. Both generated native programs and the configured `-r` path are checked; NONE builds only run native programs. Each source is compiled once per execution path and processes its records in one batch. Tests normalize only CRLF to LF; spaces, blank lines, case and final newlines remain significant. Generated text files use the same comparison rule. + +Replay an exact seed, or reduce the random count for a quick local check: + +```sh +cmake -S . -B build -DCPSC_TEST_SEED=FULL_COMMIT_SHA -DCPSC_TEST_COUNT=100 +ctest --test-dir build -L '^ii$' --output-on-failure +``` + +Use the actual SHA from a failed report in place of `FULL_COMMIT_SHA`. Clear `CPSC_TEST_SEED` to return to HEAD-at-execution defaults. CI explicitly sets it to the push SHA; generators never fetch main or use Perl rand. Reproduction requires the same source revision, seed and count. Native C type results may legitimately differ between ABIs; they are compared to the C++ executable built on that platform. + +`build/test-results///` contains `result.json`, numbered command metadata, stdin/stdout/stderr, expected output and isolated work directories. Random mismatches also save the first differing record. Re-running a case replaces only that case's directory; summaries include any other retained results, so remove `build/test-results` before collecting a fresh full-suite report. Commands have finite timeouts (30 seconds normally, 120 for compilation, 180 for file-intensive executions); timeout and launch/crash failures fail the case. + +In Actions, each compiler platform runs all three suites and summarizes results before its existing package smoke checks and upload. Failures retain inputs and logs as an Actions artifact for 14 days. Each platform publishes only after its own tests pass. These new tests expose existing defects, so compiler packages will remain unpublished until those defects are fixed; the independent plugin job can still publish. + +For adding cases and runner interfaces, see [AGENTS/TESTING.md](../AGENTS/TESTING.md). diff --git a/tests/check_generators.py b/tests/check_generators.py new file mode 100644 index 0000000..2b92789 --- /dev/null +++ b/tests/check_generators.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Require identical generated input bytes across platforms and Perl versions.""" + +import hashlib +import json +from pathlib import Path +import subprocess +import sys + + +directory = Path(__file__).resolve().parent / "ii" +vectors = json.loads((directory / "generator-vectors.json").read_text(encoding="utf-8")) +for name, digest in vectors.items(): + command = [sys.argv[1], str(directory / name / "generate.pl")] + outputs = [] + for seed in ["0" * 40, "0" * 40, "1" * 40]: + result = subprocess.run([*command, seed, "100"], check=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=10) + if result.stderr: + raise RuntimeError(f"{name}: unexpected stderr {result.stderr!r}") + outputs.append(result.stdout) + if outputs[0] != outputs[1] or outputs[0] == outputs[2]: + raise RuntimeError(f"{name}: seed reproducibility failed") + if hashlib.sha256(outputs[0]).hexdigest() != digest: + raise RuntimeError(f"{name}: generated input differs from the portable test vector") + print(f"PASS {name}: repeatable, seed-sensitive, portable bytes") diff --git a/tests/ii/arithmetic/notes.md b/tests/ii/arithmetic/notes.md index 947a3fa..d152e29 100644 --- a/tests/ii/arithmetic/notes.md +++ b/tests/ii/arithmetic/notes.md @@ -3,3 +3,5 @@ CIE pp. 15-16, project i64 and truncation-towards-zero DIV/MOD. Full-width extremes are used only in safe division/remainder and comparisons; addition, subtraction and multiplication use A MOD 1000001. B is nonzero, and INT64_MIN / -1 is never generated. Each record produces exactly one output line. Random records are preceded by fixed boundary records; every arithmetic intermediate is bounded. + +Nested IF classifies the sign; CASE tests an inclusive -10..10 range before an overlapping zero label, so zero must select the first range. The oracle computes these results without reproducing the pseudocode control flow. diff --git a/tests/ii/arithmetic/program.cps b/tests/ii/arithmetic/program.cps index 7c4bf52..e13381b 100644 --- a/tests/ii/arithmetic/program.cps +++ b/tests/ii/arithmetic/program.cps @@ -3,10 +3,26 @@ DECLARE Index : INTEGER DECLARE A : INTEGER DECLARE B : INTEGER DECLARE Small : INTEGER +DECLARE Sign : INTEGER +DECLARE Band : INTEGER INPUT Count FOR Index <- 1 TO Count INPUT A INPUT B Small <- A MOD 1000001 - OUTPUT A < B, ":", A DIV B, ":", A MOD B, ":", Small + B, ":", Small - B, ":", Small * B + IF A < 0 THEN + Sign <- -1 + ELSE + IF A = 0 THEN + Sign <- 0 + ELSE + Sign <- 1 + ENDIF + ENDIF + CASE OF Small + -10 TO 10 : Band <- 2 + 0 : Band <- 99 + OTHERWISE : Band <- 3 + ENDCASE + OUTPUT A < B, ":", A DIV B, ":", A MOD B, ":", Small + B, ":", Small - B, ":", Small * B, ":", Sign, ":", Band NEXT Index diff --git a/tests/ii/arithmetic/reference.cc b/tests/ii/arithmetic/reference.cc index 6d3d98f..3e0b27f 100644 --- a/tests/ii/arithmetic/reference.cc +++ b/tests/ii/arithmetic/reference.cc @@ -7,7 +7,10 @@ int main() { std::int64_t a, b; if (!(std::cin >> a >> b) || b == 0) return 1; auto small = a % 1000001; + int sign = (a > 0) - (a < 0); + int band = small >= -10 && small <= 10 ? 2 : 3; std::cout << (a < b ? "TRUE" : "FALSE") << ':' << a / b << ':' << a % b - << ':' << small + b << ':' << small - b << ':' << small * b << '\n'; + << ':' << small + b << ':' << small - b << ':' << small * b + << ':' << sign << ':' << band << '\n'; } } diff --git a/tests/ii/generator-vectors.json b/tests/ii/generator-vectors.json new file mode 100644 index 0000000..8d08da7 --- /dev/null +++ b/tests/ii/generator-vectors.json @@ -0,0 +1,9 @@ +{ + "arithmetic": "fa1b753544b05a783d1291aefea47a3288bbdf1339e912b55c6ed01e16b82588", + "arrays": "be2bad75adcf82776e7de77bb4992c31994b11984eac7b9cb3d225ca0fcf9e9d", + "calls": "1933d62e59a50ab51eca103b012ba69064651de0af06b62ff48a6235c47405ee", + "files": "0addc4652097fed574bdc5c0e9d59f237d49dcb7b92638dde146adde04bc6042", + "native-types": "649e978c2ccc0d4de66e2a69b8f98131015fb77c371ce4749d6278ffd84287f2", + "real": "8ffb0244589eff8a1d873ca6f1e1d66cc84944ab68d8bfafa118e941891ac518", + "strings": "c8b1aa74413d7e2a673baa27e495745b5fdf799ba423e0180620b8427cd1c42a" +} diff --git a/tests/summarize.py b/tests/summarize.py new file mode 100644 index 0000000..5006dc0 --- /dev/null +++ b/tests/summarize.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Summarize retained case results locally and in GitHub Actions.""" + +import argparse +import json +import os +from pathlib import Path + + +def summarize(directory): + results = [json.loads(path.read_text(encoding="utf-8")) + for path in sorted(directory.glob("*/*/result.json"))] + if not results: + return "No compiler case results were produced; inspect the build/CTest log.\n" + failures = [result for result in results if not result["passed"]] + lines = [f"Compiler cases: {len(results) - len(failures)}/{len(results)} passed.", "", + "| Group | Passed | Total |", "|---|---:|---:|"] + for suite in ["i", "ii", "iii"]: + subset = [result for result in results if result["case"].startswith(suite + "/")] + lines.append(f"| {suite} | {sum(result['passed'] for result in subset)} | {len(subset)} |") + lines += ["", f"Commit: `{results[0]['commit']}`. Seed: `{results[0]['seed']}`.", + "Fixed expectations: pending human review."] + if failures: + lines += ["", "Failures:", ""] + for result in failures: + stages = ", ".join(failure["stage"] for failure in result["failures"]) + lines.append(f"- `{result['case']}` ({stages}); see its `result.json` and command logs.") + return "\n".join(lines) + "\n" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory", type=Path) + args = parser.parse_args() + summary = summarize(args.directory) + print(summary, end="") + args.directory.mkdir(parents=True, exist_ok=True) + (args.directory / "summary.md").write_text(summary, encoding="utf-8") + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as output: + output.write(summary) -- cgit v1.3.1-4-g156e