#!/usr/bin/env python3 """Run selected numbered testcases through the IR backend and IRCompiler.""" from __future__ import annotations import argparse import subprocess import sys from dataclasses import dataclass from pathlib import Path @dataclass(frozen=True) class ExpectedOutput: stdout: bytes returncode: int @dataclass(frozen=True) class Testcase: number: int source: Path input_file: Path | None output_file: Path | None def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser( description=( "Run numbered testcases like 1-5,7 with " "`cargo run -- -I -S -D -o output.ir testcase.c`, then " "`tools/IRCompiler -R output.ir`, and compare with testcase.out." ) ) parser.add_argument("numbers", help="testcase numbers, for example: 1-5,7") parser.add_argument( "--testcases", type=Path, default=Path("testcases"), help="testcase directory (default: testcases)", ) parser.add_argument( "--ir", type=Path, default=Path("output.ir"), help="IR output path reused for each testcase (default: output.ir)", ) parser.add_argument( "--ircompiler", type=Path, default=Path("tools/IRCompiler"), help="IRCompiler path (default: tools/IRCompiler)", ) parser.add_argument( "-v", "--verbose", action="store_true", help="print compiler/interpreter stderr for passing cases too", ) return parser.parse_args() def parse_numbers(spec: str) -> list[int]: numbers: list[int] = [] seen: set[int] = set() for raw_part in spec.split(","): part = raw_part.strip() if not part: continue if "-" in part: bounds = part.split("-", 1) if len(bounds) != 2 or not bounds[0] or not bounds[1]: raise ValueError(f"invalid range: {part!r}") start, end = int(bounds[0]), int(bounds[1]) step = 1 if start <= end else -1 values = range(start, end + step, step) else: values = (int(part),) for value in values: if value <= 0: raise ValueError(f"testcase number must be positive: {value}") if value not in seen: seen.add(value) numbers.append(value) if not numbers: raise ValueError("no testcase numbers specified") return numbers def find_testcase(testcases_dir: Path, number: int) -> Testcase: prefix = f"{number:03d}_" matches = sorted(testcases_dir.glob(f"{prefix}*.c")) if not matches: raise FileNotFoundError(f"no C testcase found for {number} ({prefix}*.c)") if len(matches) > 1: joined = ", ".join(str(path) for path in matches) raise RuntimeError(f"multiple C testcases found for {number}: {joined}") source = matches[0] stem = source.with_suffix("") input_file = stem.with_suffix(".in") output_file = stem.with_suffix(".out") return Testcase( number=number, source=source, input_file=input_file if input_file.is_file() else None, output_file=output_file if output_file.is_file() else None, ) def parse_expected(output_file: Path) -> ExpectedOutput: data = output_file.read_bytes() lines = data.splitlines(keepends=True) if not lines: raise ValueError(f"empty expected output file: {output_file}") returncode_line = lines[-1].strip() try: returncode = int(returncode_line) except ValueError as err: raise ValueError( f"last line of {output_file} is not an integer return code: " f"{returncode_line!r}" ) from err return ExpectedOutput(stdout=b"".join(lines[:-1]), returncode=returncode) def normalize_returncode(returncode: int) -> int: return returncode & 255 def strip_trailing_newlines(data: bytes) -> bytes: return data.rstrip(b"\r\n") def run_command( command: list[str], *, stdin: bytes | None = None, ) -> subprocess.CompletedProcess[bytes]: return subprocess.run( command, input=stdin, stdout=subprocess.PIPE, stderr=subprocess.PIPE, check=False, ) def format_bytes(data: bytes) -> str: if not data: return "" try: return data.decode() except UnicodeDecodeError: return repr(data) def format_bytes_debug(data: bytes) -> str: return f"{data!r} ({len(data)} bytes)" def print_failure_detail( testcase: Testcase, *, expected: ExpectedOutput | None, actual_stdout: bytes, actual_returncode: int | None, stderr: bytes, verbose: bool, ) -> None: print(f" source: {testcase.source}") if testcase.input_file is not None: print(f" input: {testcase.input_file}") if expected is not None: print(" expected stdout:") print(format_bytes(expected.stdout), end="" if expected.stdout.endswith(b"\n") else "\n") print(f" expected stdout bytes: {format_bytes_debug(expected.stdout)}") print(f" expected returncode: {expected.returncode}") print(" actual stdout:") print(format_bytes(actual_stdout), end="" if actual_stdout.endswith(b"\n") else "\n") print(f" actual stdout bytes: {format_bytes_debug(actual_stdout)}") if actual_returncode is not None: print(f" actual returncode: {actual_returncode}") if stderr and (verbose or expected is None): print(" stderr:") print(format_bytes(stderr), end="" if stderr.endswith(b"\n") else "\n") def run_testcase( testcase: Testcase, *, ir_file: Path, ircompiler: Path, verbose: bool, ) -> bool: compile_cmd = [ "cargo", "run", "--", "-I", "-S", "-D", "-o", str(ir_file), str(testcase.source), ] compile_result = run_command(compile_cmd) if compile_result.returncode != 0: print(f"[FAIL] {testcase.number:03d} compile failed") print_failure_detail( testcase, expected=None, actual_stdout=compile_result.stdout, actual_returncode=compile_result.returncode, stderr=compile_result.stderr, verbose=True, ) return False stdin = testcase.input_file.read_bytes() if testcase.input_file is not None else None run_result = run_command([str(ircompiler), "-R", str(ir_file)], stdin=stdin) if testcase.output_file is None: print(f"[SKIP] {testcase.number:03d} missing .out: {testcase.source.name}") print_failure_detail( testcase, expected=None, actual_stdout=run_result.stdout, actual_returncode=run_result.returncode, stderr=run_result.stderr, verbose=verbose, ) return False expected = parse_expected(testcase.output_file) actual_returncode = normalize_returncode(run_result.returncode) ok = ( strip_trailing_newlines(run_result.stdout) == strip_trailing_newlines(expected.stdout) and actual_returncode == expected.returncode ) status = "PASS" if ok else "FAIL" print(f"[{status}] {testcase.number:03d} {testcase.source.name}") if not ok or (verbose and (compile_result.stderr or run_result.stderr)): if verbose and compile_result.stderr: print(" compiler stderr:") print( format_bytes(compile_result.stderr), end="" if compile_result.stderr.endswith(b"\n") else "\n", ) print_failure_detail( testcase, expected=expected, actual_stdout=run_result.stdout, actual_returncode=actual_returncode, stderr=run_result.stderr, verbose=verbose, ) return ok def main() -> int: args = parse_args() try: numbers = parse_numbers(args.numbers) testcases = [find_testcase(args.testcases, number) for number in numbers] except (OSError, RuntimeError, ValueError) as err: print(f"error: {err}", file=sys.stderr) return 2 passed = 0 for testcase in testcases: try: if run_testcase( testcase, ir_file=args.ir, ircompiler=args.ircompiler, verbose=args.verbose, ): passed += 1 except OSError as err: print(f"[FAIL] {testcase.number:03d} failed to run command: {err}") except ValueError as err: print(f"[FAIL] {testcase.number:03d} invalid expected output: {err}") total = len(testcases) failed = total - passed print(f"\nsummary: {passed}/{total} passed, {failed} failed") return 0 if failed == 0 else 1 if __name__ == "__main__": raise SystemExit(main())