diff options
| -rw-r--r-- | CMakeLists.txt | 6 | ||||
| -rw-r--r-- | tests/.gitignore | 1 | ||||
| -rw-r--r-- | tests/CMakeLists.txt | 28 | ||||
| -rw-r--r-- | tests/run.py | 250 | ||||
| -rw-r--r-- | tests/test_runner.py | 44 |
5 files changed, 329 insertions, 0 deletions
diff --git a/CMakeLists.txt b/CMakeLists.txt index 965bb0b..e5bae31 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -112,6 +112,12 @@ target_link_libraries(cpsc PRIVATE include(cmake/NativeCompiler.cmake) +option(BUILD_TESTING "Enable compiler conformance tests" OFF) +if(BUILD_TESTING) + enable_testing() + add_subdirectory(tests) +endif() + include(cmake/Manual.cmake) configure_file(cmake/Install.cmake.in "${CMAKE_CURRENT_BINARY_DIR}/Install.cmake" @ONLY) diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000..c18dd8d --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +__pycache__/ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt new file mode 100644 index 0000000..c5a7dc2 --- /dev/null +++ b/tests/CMakeLists.txt @@ -0,0 +1,28 @@ +find_package(Perl REQUIRED) +set(CPSC_TEST_SEED "" CACHE STRING "Replay seed (defaults to HEAD at test execution)") +set(CPSC_TEST_COUNT 10000 CACHE STRING "Random records per differential group") + +add_test(NAME harness COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/test_runner.py") +foreach(suite i ii iii) + file(GLOB cases CONFIGURE_DEPENDS "${CMAKE_CURRENT_SOURCE_DIR}/${suite}/*/case.json") + foreach(manifest IN LISTS cases) + get_filename_component(case_dir "${manifest}" DIRECTORY) + get_filename_component(case_name "${case_dir}" NAME) + set(reference_args) + if(suite STREQUAL "ii") + set(reference_target "cpsc-reference-${case_name}") + add_executable(${reference_target} "${case_dir}/reference.cc") + target_compile_features(${reference_target} PRIVATE cxx_std_17) + list(APPEND reference_args --reference "$<TARGET_FILE:${reference_target}>") + endif() + add_test(NAME "${suite}/${case_name}" COMMAND "${Python3_EXECUTABLE}" + "${CMAKE_CURRENT_SOURCE_DIR}/run.py" + --case "${manifest}" --cpsc "$<TARGET_FILE:cpsc>" + --run-mode "${CPSC_RUN_MODE}" --perl "${PERL_EXECUTABLE}" + --seed "${CPSC_TEST_SEED}" --count "${CPSC_TEST_COUNT}" + --results "${CMAKE_BINARY_DIR}/test-results" ${reference_args}) + set_tests_properties("${suite}/${case_name}" PROPERTIES + LABELS "${suite}" TIMEOUT 600) + endforeach() +endforeach() diff --git a/tests/run.py b/tests/run.py new file mode 100644 index 0000000..432d1b9 --- /dev/null +++ b/tests/run.py @@ -0,0 +1,250 @@ +#!/usr/bin/env python3 +"""Run one compiler fixture. Only CRLF is normalized; failures are never waived.""" + +import argparse +import difflib +import json +import os +from pathlib import Path +import platform +import re +import shutil +import subprocess +import sys + + +ROOT = Path(__file__).resolve().parent.parent + + +class Failure(Exception): + pass + + +def normalized(data): + return data.replace(b"\r\n", b"\n") + + +def compare(expected, actual, label): + expected, actual = normalized(expected), normalized(actual) + if expected == actual: + return + # repr keeps trailing spaces, missing newlines and non-UTF-8 bytes visible. + left = [repr(line) + "\n" for line in expected.splitlines(keepends=True)] + right = [repr(line) + "\n" for line in actual.splitlines(keepends=True)] + diff = "".join(difflib.unified_diff(left, right, "expected", "actual", n=2)) + raise Failure(f"{label} differs:\n{diff[:5000]}") + + +def check_result(result, expected, exit_code=0): + errors = [] + if result.returncode != exit_code: + errors.append(f"exit code: expected {exit_code}, got {result.returncode}") + if result.stderr: + errors.append(f"unexpected stderr: {result.stderr[:2000]!r}") + try: + compare(expected, result.stdout, "stdout") + except Failure as error: + errors.append(str(error)) + if errors: + raise Failure("\n".join(errors)) + + +def check_diagnostics(result, expectations): + errors = [] + if result.returncode != 1: + errors.append(f"compiler exit code: expected 1, got {result.returncode}") + if result.stdout: + errors.append(f"unexpected stdout: {result.stdout[:1000]!r}") + lines = normalized(result.stderr).decode("utf-8", errors="replace").splitlines() + for expectation in expectations: + matching = [line for line in lines if "error:" in line + and re.search(expectation["reason"], line)] + location = rf"\bline {expectation['line']}\b" + if not any(re.search(location, line) for line in matching): + errors.append(f"missing error /{expectation['reason']}/ at line " + f"{expectation['line']}; matching diagnostics: {matching}") + if lines.count("compilation terminated.") != 1: + errors.append("expected exactly one 'compilation terminated.' line") + if errors: + raise Failure("\n".join(errors)) + + +class Commands: + def __init__(self, directory): + self.directory = directory + self.index = 0 + + def run(self, command, cwd, data=b"", timeout=30): + self.index += 1 + stem = self.directory / f"{self.index:02d}" + metadata = {"command": [str(arg) for arg in command], "cwd": str(cwd), + "timeout": timeout} + stem.with_suffix(".stdin").write_bytes(data) + try: + result = subprocess.run(metadata["command"], cwd=cwd, input=data, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + timeout=timeout) + metadata["exit_code"] = result.returncode + except subprocess.TimeoutExpired as error: + metadata["timeout_expired"] = True + stem.with_suffix(".stdout").write_bytes(error.stdout or b"") + stem.with_suffix(".stderr").write_bytes(error.stderr or b"") + raise Failure(f"command timed out after {timeout}s: {command}") from error + except OSError as error: + metadata["launch_error"] = str(error) + raise Failure(f"cannot launch {command}: {error}") from error + finally: + stem.with_suffix(".json").write_text(json.dumps(metadata, indent=2) + "\n", + encoding="utf-8") + stem.with_suffix(".stdout").write_bytes(result.stdout) + stem.with_suffix(".stderr").write_bytes(result.stderr) + return result + + +def workspace(case_dir, directory, name): + work = directory / name + work.mkdir() + if (case_dir / "files").is_dir(): + shutil.copytree(case_dir / "files", work, dirs_exist_ok=True) + return work + + +def check_files(case_dir, work): + for expected in sorted((case_dir / "expected-files").glob("**/*")): + if expected.is_file(): + relative = expected.relative_to(case_dir / "expected-files") + actual = work / relative + if not actual.is_file(): + raise Failure(f"missing output file: {relative}") + compare(expected.read_bytes(), actual.read_bytes(), str(relative)) + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--case", required=True, type=Path) + parser.add_argument("--cpsc", required=True, type=Path) + parser.add_argument("--run-mode", choices=["EMBEDDED", "PATH", "NONE"], required=True) + parser.add_argument("--reference", type=Path) + parser.add_argument("--perl", default="perl") + parser.add_argument("--seed", default="") + parser.add_argument("--count", type=int, default=10000) + parser.add_argument("--results", type=Path, default=ROOT / "build/test-results") + args = parser.parse_args(argv) + if args.count < 1: + parser.error("--count must be positive") + args.case = args.case.resolve() + args.cpsc = args.cpsc.resolve() + if args.reference: + args.reference = args.reference.resolve() + case_dir = args.case.parent + suite, name = case_dir.parent.name, case_dir.name + config = json.loads(args.case.read_text(encoding="utf-8")) + commit = subprocess.check_output(["git", "rev-parse", "HEAD"], cwd=ROOT, + text=True, timeout=10).strip() + seed = args.seed or commit + directory = args.results.resolve() / suite / name + if directory.exists(): + shutil.rmtree(directory) + directory.mkdir(parents=True) + commands = Commands(directory) + report = {"case": f"{suite}/{name}", "commit": commit, "seed": seed, + "random_count": args.count, "platform": platform.platform(), + "run_mode": args.run_mode, "failures": []} + source = case_dir / "program.cps" + + def attempt(label, action): + try: + action() + print(f"PASS {suite}/{name} [{label}]", flush=True) + except (Failure, OSError, ValueError) as error: + report["failures"].append({"stage": label, "message": str(error)}) + print(f"FAIL {suite}/{name} [{label}]: {error}", flush=True) + + def diagnostics(): + for output_format in ["native", "ir"]: + work = workspace(case_dir, directory, output_format) + target = work / ("program.exe" if os.name == "nt" else "program.bin") + flags = ["-S"] if output_format == "ir" else [] + for existing in [False, True]: + sentinel = b"existing output must survive\n" + if existing: + target.write_bytes(sentinel) + result = commands.run([args.cpsc, source, *flags, "-o", target], work, + timeout=120) + # Check output preservation even when the diagnostics are wrong. + preservation_error = None + if existing and (not target.exists() or target.read_bytes() != sentinel): + preservation_error = "compilation failure overwrote existing output" + if not existing and target.exists(): + preservation_error = "compilation failure created output" + target.unlink() + def validate(): + if preservation_error: + raise Failure(preservation_error) + check_diagnostics(result, config["errors"]) + attempt(f"{output_format}/{'existing' if existing else 'absent'}", validate) + + try: + if suite == "iii": + diagnostics() + else: + data = (case_dir / "input.txt").read_bytes() if (case_dir / "input.txt").exists() else b"" + if suite == "ii": + if not args.reference: + raise Failure("differential tests require --reference") + generated = commands.run([args.perl, case_dir / "generate.pl", seed, + str(args.count)], directory) + if generated.returncode != 0 or generated.stderr: + raise Failure(f"generator failed: {generated.stderr!r}") + data = generated.stdout + records = normalized(data).splitlines() + if not records or int(records[0]) != len(records) - 1: + raise Failure("generator must emit record count followed by one record per line") + work = workspace(case_dir, directory, "reference") + reference = commands.run([args.reference], work, data) + if reference.returncode != 0 or reference.stderr: + raise Failure(f"reference failed ({reference.returncode}): {reference.stderr!r}") + expected = reference.stdout + if len(normalized(expected).splitlines()) != len(records) - 1: + raise Failure("reference must emit exactly one output line per input record") + else: + expected = (case_dir / "expected.txt").read_bytes() + (directory / "input.txt").write_bytes(data) + (directory / "expected.txt").write_bytes(expected) + + def execute(mode): + work = workspace(case_dir, directory, mode) + if mode == "native": + target = work / ("program.exe" if os.name == "nt" else "program.bin") + compiled = commands.run([args.cpsc, source, "-o", target], work, timeout=120) + check_result(compiled, b"") + command = [target] + else: + command = [args.cpsc, source, "-r"] + result = commands.run(command, work, data) + if suite == "ii" and normalized(result.stdout) != normalized(expected): + wanted, got = normalized(expected).splitlines(), normalized(result.stdout).splitlines() + index = next((i for i in range(min(len(wanted), len(got))) + if wanted[i] != got[i]), min(len(wanted), len(got))) + record = records[index + 1] if index + 1 < len(records) else b"<extra output>" + print(f"First differing record {index + 1}: {record!r}", flush=True) + (directory / f"{mode}-first-difference.json").write_text(json.dumps({ + "record": index + 1, "input": record.decode("ascii"), + "expected": repr(wanted[index]) if index < len(wanted) else "<missing>", + "actual": repr(got[index]) if index < len(got) else "<missing>"}, indent=2) + "\n", + encoding="utf-8") + check_result(result, expected, config.get("exit_code", 0)) + check_files(case_dir, work) + for mode in ["native"] + (["runner"] if args.run_mode != "NONE" else []): + attempt(mode, lambda mode=mode: execute(mode)) + except (Failure, OSError, ValueError) as error: + report["failures"].append({"stage": "setup", "message": str(error)}) + print(f"FAIL {suite}/{name} [setup]: {error}", flush=True) + report["passed"] = not report["failures"] + (directory / "result.json").write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8") + print(f"Seed: {seed}; results: {directory}", flush=True) + return 0 if report["passed"] else 1 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/test_runner.py b/tests/test_runner.py new file mode 100644 index 0000000..9558175 --- /dev/null +++ b/tests/test_runner.py @@ -0,0 +1,44 @@ +"""Small contract checks for the test harness, including intentional failures.""" + +from pathlib import Path +import subprocess +import sys +import tempfile +import unittest + +from run import Commands, Failure, check_diagnostics, check_result, compare + + +class HarnessTests(unittest.TestCase): + def test_strict_output(self): + compare(b"a\nb\n", b"a\r\nb\r\n", "stdout") + for actual in [b"a\nb", b"a \nb\n", b"A\nb\n", b"a\n\nb\n"]: + with self.assertRaises(Failure): + compare(b"a\nb\n", actual, "stdout") + + def test_exit_and_stderr(self): + for code, stderr in [(1, b""), (-11, b""), (0, b"unexpected")]: + with self.assertRaises(Failure): + check_result(subprocess.CompletedProcess([], code, b"ok\n", stderr), b"ok\n") + + def test_reason_and_location_must_share_a_diagnostic(self): + expected = [{"reason": "Expected ':'", "line": 3}] + check_diagnostics(subprocess.CompletedProcess([], 1, b"", + b"error: Expected ':' at line 3\ncompilation terminated.\n"), expected) + for diagnostic in [b"error: Expected ':' at line 4", b"error: Expected ':'", + b"error: Expected ':' at line 30\nerror: unrelated at line 3"]: + with self.assertRaises(Failure): + check_diagnostics(subprocess.CompletedProcess([], 1, b"", + diagnostic + b"\ncompilation terminated.\n"), expected) + + def test_timeout_is_failure_and_saved(self): + with tempfile.TemporaryDirectory() as temporary: + directory = Path(temporary) + with self.assertRaisesRegex(Failure, "timed out"): + Commands(directory).run([sys.executable, "-c", "import time; time.sleep(5)"], + directory, timeout=0.05) + self.assertIn('"timeout_expired": true', (directory / "01.json").read_text()) + + +if __name__ == "__main__": + unittest.main() |