diff options
Diffstat (limited to 'tests')
| -rw-r--r-- | tests/CMakeLists.txt | 3 | ||||
| -rw-r--r-- | tests/FAILURES.md | 32 | ||||
| -rw-r--r-- | tests/README.md | 31 | ||||
| -rw-r--r-- | tests/check_generators.py | 26 | ||||
| -rw-r--r-- | tests/ii/arithmetic/notes.md | 2 | ||||
| -rw-r--r-- | tests/ii/arithmetic/program.cps | 18 | ||||
| -rw-r--r-- | tests/ii/arithmetic/reference.cc | 5 | ||||
| -rw-r--r-- | tests/ii/generator-vectors.json | 9 | ||||
| -rw-r--r-- | tests/summarize.py | 41 |
9 files changed, 165 insertions, 2 deletions
diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 4d385b9..630d363 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -1,9 +1,12 @@ +cpsc_find_dependency(PROGRAM PERL_EXECUTABLE NAMES perl REQUIRED) find_package(Perl REQUIRED) set(CPSC_TEST_SEED "" CACHE STRING "Replay seed (defaults to HEAD at test execution)") set(CPSC_TEST_COUNT 10000 CACHE STRING "Random records per differential group") add_test(NAME harness COMMAND "${Python3_EXECUTABLE}" "${CMAKE_CURRENT_SOURCE_DIR}/test_runner.py") +add_test(NAME generators COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/check_generators.py" "${PERL_EXECUTABLE}") foreach(suite i ii iii) file(GLOB cases CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${suite}/*/case.json") foreach(manifest IN LISTS cases) diff --git a/tests/FAILURES.md b/tests/FAILURES.md new file mode 100644 index 0000000..3890e3b --- /dev/null +++ b/tests/FAILURES.md @@ -0,0 +1,32 @@ +# Compiler test findings + +Baseline: Linux x86_64, Clang/LLVM 23.1.3, Release EMBEDDED, compiler sources at `e2c6570`. Tests do not modify compiler sources. The full suite has 45 compiler cases plus harness and portable-generator checks. Fixed output expectations are pending human review. + +## Observed failures + +| Case | Required behavior | Current behavior | +|---|---|---| +| i/class-default-public | Omitted access defaults to public (CIE p. 28). | Rejects the field unless PUBLIC/PRIVATE is explicit. | +| i/identifier-case | Identifier lookup is case insensitive (CIE p. 7). | Reports the differently cased variable as unknown. | +| i/standard-while | Accept WHILE condition without DO (CIE p. 21). | Requires DO. | +| iii/argument-count | Wrong argument count, line 4. | Correct cause, no line. | +| iii/builtin-arity | LENGTH arity, line 2. | Correct cause, no line. | +| iii/byref-literal | Non-variable BYREF argument, line 4. | Correct cause, no line. | +| iii/constant-write | Assignment to constant, line 2. | Correct cause, no line. | +| iii/missing-endif | Missing ENDIF at EOF, line 4. | Correct cause, no line. | +| iii/missing-parenthesis | Missing closing parenthesis at EOF, line 3. | Correct cause, no line. | +| iii/missing-then | Missing THEN, line 2. | Correct cause, no line. | +| iii/nominal-type-mismatch | Incompatible named pointers, line 7. | Correct cause, no line. | +| iii/unknown-name | Undeclared name, line 2. | Correct cause, no line. | + +Both native and `-r` compilation expose the three i-group failures. All nine iii-group failures occur with both native/IR output requests, whether the destination was absent or already contained a sentinel. Failed compilation preserves the destination in these cases. The remaining 33 compiler cases passed locally, including seven groups of 10,000 random records plus 36 boundary records (70,036 total inputs, each checked through native and embedded execution). + +The baseline test seed and exact command results are retained in `build/test-results`; subsequent CI reports use their own explicit push SHA. Platform results from Actions are the authority for Windows/macOS verification; local Linux results do not establish their behavior. + +## Coverage gaps + +- Not implemented by the compiler, and not required to pass this initial suite: DATE, SET/DEFINE, random files, class methods/constructors/inheritance, INT/RAND, whole-array assignment and INPUT into array/record elements. Rejection of valid CIE syntax must not be relabeled as a successful negative test. +- The guide's p. 14 pointer example points an INTEGER pointer at an enum; it conflicts with the project's explicit strict-pointee contract. Tests use matching pointees rather than silently choosing a new policy. +- LCASE/UCASE are CHAR-to-CHAR in the guide (p. 17), while the project declares STRING results and also supports whole strings. Output observations cover case conversion but do not prove exact static return-type conformance. +- No requirements are invented for unspecified integer overflow, REAL output spelling, out-of-range substring behavior, Unicode character semantics, or uninitialized CIE values. Native C ABI behavior is checked only against that platform's independent C++ reference. +- This is input differential testing of seven fixed programs, not random-program fuzzing. It cannot establish complete parser or optimizer correctness. New bug reports should become small permanent fixed cases. diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..9024191 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,31 @@ +# Compiler tests + +Enable tests in an existing configured build (Perl with core Digest::SHA is additionally required): + +```sh +cmake -S . -B build -DBUILD_TESTING=ON +cmake --build build --parallel 2 +ctest --test-dir build --output-on-failure --parallel 2 +python3 tests/summarize.py build/test-results +``` + +Use `ctest --test-dir build -L '^ii$' --output-on-failure` for one suite, or `-R '^i/arrays$'` for one case. CTest returns failure if any case fails; it continues through the other cases. [Current failures and coverage gaps](FAILURES.md) are intentionally visible, with no expected-failure exemptions. + +The suites are **i**: independently traced fixed outputs, **ii**: C++ differential tests with Perl inputs, and **iii**: rejected programs with required causes and source lines. Fixed expectations are **pending human review**; each case's notes explain the trace and distinguish CIE rules from project extensions. REAL calculations are observed through BOOLEAN/INTEGER output; REAL text and REAL-to-string formatting remain outside this suite. + +The default is 10,000 random records per differential group, plus fixed boundary records. Both generated native programs and the configured `-r` path are checked; NONE builds only run native programs. Each source is compiled once per execution path and processes its records in one batch. Tests normalize only CRLF to LF; spaces, blank lines, case and final newlines remain significant. Generated text files use the same comparison rule. + +Replay an exact seed, or reduce the random count for a quick local check: + +```sh +cmake -S . -B build -DCPSC_TEST_SEED=FULL_COMMIT_SHA -DCPSC_TEST_COUNT=100 +ctest --test-dir build -L '^ii$' --output-on-failure +``` + +Use the actual SHA from a failed report in place of `FULL_COMMIT_SHA`. Clear `CPSC_TEST_SEED` to return to HEAD-at-execution defaults. CI explicitly sets it to the push SHA; generators never fetch main or use Perl rand. Reproduction requires the same source revision, seed and count. Native C type results may legitimately differ between ABIs; they are compared to the C++ executable built on that platform. + +`build/test-results/<suite>/<case>/` contains `result.json`, numbered command metadata, stdin/stdout/stderr, expected output and isolated work directories. Random mismatches also save the first differing record. Re-running a case replaces only that case's directory; summaries include any other retained results, so remove `build/test-results` before collecting a fresh full-suite report. Commands have finite timeouts (30 seconds normally, 120 for compilation, 180 for file-intensive executions); timeout and launch/crash failures fail the case. + +In Actions, each compiler platform runs all three suites and summarizes results before its existing package smoke checks and upload. Failures retain inputs and logs as an Actions artifact for 14 days. Each platform publishes only after its own tests pass. These new tests expose existing defects, so compiler packages will remain unpublished until those defects are fixed; the independent plugin job can still publish. + +For adding cases and runner interfaces, see [AGENTS/TESTING.md](../AGENTS/TESTING.md). diff --git a/tests/check_generators.py b/tests/check_generators.py new file mode 100644 index 0000000..2b92789 --- /dev/null +++ b/tests/check_generators.py @@ -0,0 +1,26 @@ +#!/usr/bin/env python3 +"""Require identical generated input bytes across platforms and Perl versions.""" + +import hashlib +import json +from pathlib import Path +import subprocess +import sys + + +directory = Path(__file__).resolve().parent / "ii" +vectors = json.loads((directory / "generator-vectors.json").read_text(encoding="utf-8")) +for name, digest in vectors.items(): + command = [sys.argv[1], str(directory / name / "generate.pl")] + outputs = [] + for seed in ["0" * 40, "0" * 40, "1" * 40]: + result = subprocess.run([*command, seed, "100"], check=True, stdout=subprocess.PIPE, + stderr=subprocess.PIPE, timeout=10) + if result.stderr: + raise RuntimeError(f"{name}: unexpected stderr {result.stderr!r}") + outputs.append(result.stdout) + if outputs[0] != outputs[1] or outputs[0] == outputs[2]: + raise RuntimeError(f"{name}: seed reproducibility failed") + if hashlib.sha256(outputs[0]).hexdigest() != digest: + raise RuntimeError(f"{name}: generated input differs from the portable test vector") + print(f"PASS {name}: repeatable, seed-sensitive, portable bytes") diff --git a/tests/ii/arithmetic/notes.md b/tests/ii/arithmetic/notes.md index 947a3fa..d152e29 100644 --- a/tests/ii/arithmetic/notes.md +++ b/tests/ii/arithmetic/notes.md @@ -3,3 +3,5 @@ CIE pp. 15-16, project i64 and truncation-towards-zero DIV/MOD. Full-width extremes are used only in safe division/remainder and comparisons; addition, subtraction and multiplication use A MOD 1000001. B is nonzero, and INT64_MIN / -1 is never generated. Each record produces exactly one output line. Random records are preceded by fixed boundary records; every arithmetic intermediate is bounded. + +Nested IF classifies the sign; CASE tests an inclusive -10..10 range before an overlapping zero label, so zero must select the first range. The oracle computes these results without reproducing the pseudocode control flow. diff --git a/tests/ii/arithmetic/program.cps b/tests/ii/arithmetic/program.cps index 7c4bf52..e13381b 100644 --- a/tests/ii/arithmetic/program.cps +++ b/tests/ii/arithmetic/program.cps @@ -3,10 +3,26 @@ DECLARE Index : INTEGER DECLARE A : INTEGER DECLARE B : INTEGER DECLARE Small : INTEGER +DECLARE Sign : INTEGER +DECLARE Band : INTEGER INPUT Count FOR Index <- 1 TO Count INPUT A INPUT B Small <- A MOD 1000001 - OUTPUT A < B, ":", A DIV B, ":", A MOD B, ":", Small + B, ":", Small - B, ":", Small * B + IF A < 0 THEN + Sign <- -1 + ELSE + IF A = 0 THEN + Sign <- 0 + ELSE + Sign <- 1 + ENDIF + ENDIF + CASE OF Small + -10 TO 10 : Band <- 2 + 0 : Band <- 99 + OTHERWISE : Band <- 3 + ENDCASE + OUTPUT A < B, ":", A DIV B, ":", A MOD B, ":", Small + B, ":", Small - B, ":", Small * B, ":", Sign, ":", Band NEXT Index diff --git a/tests/ii/arithmetic/reference.cc b/tests/ii/arithmetic/reference.cc index 6d3d98f..3e0b27f 100644 --- a/tests/ii/arithmetic/reference.cc +++ b/tests/ii/arithmetic/reference.cc @@ -7,7 +7,10 @@ int main() { std::int64_t a, b; if (!(std::cin >> a >> b) || b == 0) return 1; auto small = a % 1000001; + int sign = (a > 0) - (a < 0); + int band = small >= -10 && small <= 10 ? 2 : 3; std::cout << (a < b ? "TRUE" : "FALSE") << ':' << a / b << ':' << a % b - << ':' << small + b << ':' << small - b << ':' << small * b << '\n'; + << ':' << small + b << ':' << small - b << ':' << small * b + << ':' << sign << ':' << band << '\n'; } } diff --git a/tests/ii/generator-vectors.json b/tests/ii/generator-vectors.json new file mode 100644 index 0000000..8d08da7 --- /dev/null +++ b/tests/ii/generator-vectors.json @@ -0,0 +1,9 @@ +{ + "arithmetic": "fa1b753544b05a783d1291aefea47a3288bbdf1339e912b55c6ed01e16b82588", + "arrays": "be2bad75adcf82776e7de77bb4992c31994b11984eac7b9cb3d225ca0fcf9e9d", + "calls": "1933d62e59a50ab51eca103b012ba69064651de0af06b62ff48a6235c47405ee", + "files": "0addc4652097fed574bdc5c0e9d59f237d49dcb7b92638dde146adde04bc6042", + "native-types": "649e978c2ccc0d4de66e2a69b8f98131015fb77c371ce4749d6278ffd84287f2", + "real": "8ffb0244589eff8a1d873ca6f1e1d66cc84944ab68d8bfafa118e941891ac518", + "strings": "c8b1aa74413d7e2a673baa27e495745b5fdf799ba423e0180620b8427cd1c42a" +} diff --git a/tests/summarize.py b/tests/summarize.py new file mode 100644 index 0000000..5006dc0 --- /dev/null +++ b/tests/summarize.py @@ -0,0 +1,41 @@ +#!/usr/bin/env python3 +"""Summarize retained case results locally and in GitHub Actions.""" + +import argparse +import json +import os +from pathlib import Path + + +def summarize(directory): + results = [json.loads(path.read_text(encoding="utf-8")) + for path in sorted(directory.glob("*/*/result.json"))] + if not results: + return "No compiler case results were produced; inspect the build/CTest log.\n" + failures = [result for result in results if not result["passed"]] + lines = [f"Compiler cases: {len(results) - len(failures)}/{len(results)} passed.", "", + "| Group | Passed | Total |", "|---|---:|---:|"] + for suite in ["i", "ii", "iii"]: + subset = [result for result in results if result["case"].startswith(suite + "/")] + lines.append(f"| {suite} | {sum(result['passed'] for result in subset)} | {len(subset)} |") + lines += ["", f"Commit: `{results[0]['commit']}`. Seed: `{results[0]['seed']}`.", + "Fixed expectations: pending human review."] + if failures: + lines += ["", "Failures:", ""] + for result in failures: + stages = ", ".join(failure["stage"] for failure in result["failures"]) + lines.append(f"- `{result['case']}` ({stages}); see its `result.json` and command logs.") + return "\n".join(lines) + "\n" + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("directory", type=Path) + args = parser.parse_args() + summary = summarize(args.directory) + print(summary, end="") + args.directory.mkdir(parents=True, exist_ok=True) + (args.directory / "summary.md").write_text(summary, encoding="utf-8") + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as output: + output.write(summary) |