302 lines
8.8 KiB
Python
Executable File
302 lines
8.8 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""Run selected numbered testcases through the IR backend and IRCompiler."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import subprocess
|
|
import sys
|
|
from dataclasses import dataclass
|
|
from pathlib import Path
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class ExpectedOutput:
|
|
stdout: bytes
|
|
returncode: int
|
|
|
|
|
|
@dataclass(frozen=True)
|
|
class Testcase:
|
|
number: int
|
|
source: Path
|
|
input_file: Path | None
|
|
output_file: Path | None
|
|
|
|
|
|
def parse_args() -> argparse.Namespace:
|
|
parser = argparse.ArgumentParser(
|
|
description=(
|
|
"Run numbered testcases like 1-5,7 with "
|
|
"`cargo run -- -I -S -D -o output.ir testcase.c`, then "
|
|
"`tools/IRCompiler -R output.ir`, and compare with testcase.out."
|
|
)
|
|
)
|
|
parser.add_argument("numbers", help="testcase numbers, for example: 1-5,7")
|
|
parser.add_argument(
|
|
"--testcases",
|
|
type=Path,
|
|
default=Path("testcases"),
|
|
help="testcase directory (default: testcases)",
|
|
)
|
|
parser.add_argument(
|
|
"--ir",
|
|
type=Path,
|
|
default=Path("output.ir"),
|
|
help="IR output path reused for each testcase (default: output.ir)",
|
|
)
|
|
parser.add_argument(
|
|
"--ircompiler",
|
|
type=Path,
|
|
default=Path("tools/IRCompiler"),
|
|
help="IRCompiler path (default: tools/IRCompiler)",
|
|
)
|
|
parser.add_argument(
|
|
"-v",
|
|
"--verbose",
|
|
action="store_true",
|
|
help="print compiler/interpreter stderr for passing cases too",
|
|
)
|
|
return parser.parse_args()
|
|
|
|
|
|
def parse_numbers(spec: str) -> list[int]:
|
|
numbers: list[int] = []
|
|
seen: set[int] = set()
|
|
|
|
for raw_part in spec.split(","):
|
|
part = raw_part.strip()
|
|
if not part:
|
|
continue
|
|
|
|
if "-" in part:
|
|
bounds = part.split("-", 1)
|
|
if len(bounds) != 2 or not bounds[0] or not bounds[1]:
|
|
raise ValueError(f"invalid range: {part!r}")
|
|
start, end = int(bounds[0]), int(bounds[1])
|
|
step = 1 if start <= end else -1
|
|
values = range(start, end + step, step)
|
|
else:
|
|
values = (int(part),)
|
|
|
|
for value in values:
|
|
if value <= 0:
|
|
raise ValueError(f"testcase number must be positive: {value}")
|
|
if value not in seen:
|
|
seen.add(value)
|
|
numbers.append(value)
|
|
|
|
if not numbers:
|
|
raise ValueError("no testcase numbers specified")
|
|
return numbers
|
|
|
|
|
|
def find_testcase(testcases_dir: Path, number: int) -> Testcase:
|
|
prefix = f"{number:03d}_"
|
|
matches = sorted(testcases_dir.glob(f"{prefix}*.c"))
|
|
if not matches:
|
|
raise FileNotFoundError(f"no C testcase found for {number} ({prefix}*.c)")
|
|
if len(matches) > 1:
|
|
joined = ", ".join(str(path) for path in matches)
|
|
raise RuntimeError(f"multiple C testcases found for {number}: {joined}")
|
|
|
|
source = matches[0]
|
|
stem = source.with_suffix("")
|
|
input_file = stem.with_suffix(".in")
|
|
output_file = stem.with_suffix(".out")
|
|
return Testcase(
|
|
number=number,
|
|
source=source,
|
|
input_file=input_file if input_file.is_file() else None,
|
|
output_file=output_file if output_file.is_file() else None,
|
|
)
|
|
|
|
|
|
def parse_expected(output_file: Path) -> ExpectedOutput:
|
|
data = output_file.read_bytes()
|
|
lines = data.splitlines(keepends=True)
|
|
if not lines:
|
|
raise ValueError(f"empty expected output file: {output_file}")
|
|
|
|
returncode_line = lines[-1].strip()
|
|
try:
|
|
returncode = int(returncode_line)
|
|
except ValueError as err:
|
|
raise ValueError(
|
|
f"last line of {output_file} is not an integer return code: "
|
|
f"{returncode_line!r}"
|
|
) from err
|
|
|
|
return ExpectedOutput(stdout=b"".join(lines[:-1]), returncode=returncode)
|
|
|
|
|
|
def normalize_returncode(returncode: int) -> int:
|
|
return returncode & 255
|
|
|
|
|
|
def strip_trailing_newlines(data: bytes) -> bytes:
|
|
return data.rstrip(b"\r\n")
|
|
|
|
|
|
def run_command(
|
|
command: list[str],
|
|
*,
|
|
stdin: bytes | None = None,
|
|
) -> subprocess.CompletedProcess[bytes]:
|
|
return subprocess.run(
|
|
command,
|
|
input=stdin,
|
|
stdout=subprocess.PIPE,
|
|
stderr=subprocess.PIPE,
|
|
check=False,
|
|
)
|
|
|
|
|
|
def format_bytes(data: bytes) -> str:
|
|
if not data:
|
|
return "<empty>"
|
|
try:
|
|
return data.decode()
|
|
except UnicodeDecodeError:
|
|
return repr(data)
|
|
|
|
|
|
def format_bytes_debug(data: bytes) -> str:
|
|
return f"{data!r} ({len(data)} bytes)"
|
|
|
|
|
|
def print_failure_detail(
|
|
testcase: Testcase,
|
|
*,
|
|
expected: ExpectedOutput | None,
|
|
actual_stdout: bytes,
|
|
actual_returncode: int | None,
|
|
stderr: bytes,
|
|
verbose: bool,
|
|
) -> None:
|
|
print(f" source: {testcase.source}")
|
|
if testcase.input_file is not None:
|
|
print(f" input: {testcase.input_file}")
|
|
if expected is not None:
|
|
print(" expected stdout:")
|
|
print(format_bytes(expected.stdout), end="" if expected.stdout.endswith(b"\n") else "\n")
|
|
print(f" expected stdout bytes: {format_bytes_debug(expected.stdout)}")
|
|
print(f" expected returncode: {expected.returncode}")
|
|
print(" actual stdout:")
|
|
print(format_bytes(actual_stdout), end="" if actual_stdout.endswith(b"\n") else "\n")
|
|
print(f" actual stdout bytes: {format_bytes_debug(actual_stdout)}")
|
|
if actual_returncode is not None:
|
|
print(f" actual returncode: {actual_returncode}")
|
|
if stderr and (verbose or expected is None):
|
|
print(" stderr:")
|
|
print(format_bytes(stderr), end="" if stderr.endswith(b"\n") else "\n")
|
|
|
|
|
|
def run_testcase(
|
|
testcase: Testcase,
|
|
*,
|
|
ir_file: Path,
|
|
ircompiler: Path,
|
|
verbose: bool,
|
|
) -> bool:
|
|
compile_cmd = [
|
|
"cargo",
|
|
"run",
|
|
"--",
|
|
"-I",
|
|
"-S",
|
|
"-D",
|
|
"-o",
|
|
str(ir_file),
|
|
str(testcase.source),
|
|
]
|
|
compile_result = run_command(compile_cmd)
|
|
if compile_result.returncode != 0:
|
|
print(f"[FAIL] {testcase.number:03d} compile failed")
|
|
print_failure_detail(
|
|
testcase,
|
|
expected=None,
|
|
actual_stdout=compile_result.stdout,
|
|
actual_returncode=compile_result.returncode,
|
|
stderr=compile_result.stderr,
|
|
verbose=True,
|
|
)
|
|
return False
|
|
|
|
stdin = testcase.input_file.read_bytes() if testcase.input_file is not None else None
|
|
run_result = run_command([str(ircompiler), "-R", str(ir_file)], stdin=stdin)
|
|
|
|
if testcase.output_file is None:
|
|
print(f"[SKIP] {testcase.number:03d} missing .out: {testcase.source.name}")
|
|
print_failure_detail(
|
|
testcase,
|
|
expected=None,
|
|
actual_stdout=run_result.stdout,
|
|
actual_returncode=run_result.returncode,
|
|
stderr=run_result.stderr,
|
|
verbose=verbose,
|
|
)
|
|
return False
|
|
|
|
expected = parse_expected(testcase.output_file)
|
|
actual_returncode = normalize_returncode(run_result.returncode)
|
|
ok = (
|
|
strip_trailing_newlines(run_result.stdout)
|
|
== strip_trailing_newlines(expected.stdout)
|
|
and actual_returncode == expected.returncode
|
|
)
|
|
|
|
status = "PASS" if ok else "FAIL"
|
|
print(f"[{status}] {testcase.number:03d} {testcase.source.name}")
|
|
if not ok or (verbose and (compile_result.stderr or run_result.stderr)):
|
|
if verbose and compile_result.stderr:
|
|
print(" compiler stderr:")
|
|
print(
|
|
format_bytes(compile_result.stderr),
|
|
end="" if compile_result.stderr.endswith(b"\n") else "\n",
|
|
)
|
|
print_failure_detail(
|
|
testcase,
|
|
expected=expected,
|
|
actual_stdout=run_result.stdout,
|
|
actual_returncode=actual_returncode,
|
|
stderr=run_result.stderr,
|
|
verbose=verbose,
|
|
)
|
|
return ok
|
|
|
|
|
|
def main() -> int:
|
|
args = parse_args()
|
|
|
|
try:
|
|
numbers = parse_numbers(args.numbers)
|
|
testcases = [find_testcase(args.testcases, number) for number in numbers]
|
|
except (OSError, RuntimeError, ValueError) as err:
|
|
print(f"error: {err}", file=sys.stderr)
|
|
return 2
|
|
|
|
passed = 0
|
|
for testcase in testcases:
|
|
try:
|
|
if run_testcase(
|
|
testcase,
|
|
ir_file=args.ir,
|
|
ircompiler=args.ircompiler,
|
|
verbose=args.verbose,
|
|
):
|
|
passed += 1
|
|
except OSError as err:
|
|
print(f"[FAIL] {testcase.number:03d} failed to run command: {err}")
|
|
except ValueError as err:
|
|
print(f"[FAIL] {testcase.number:03d} invalid expected output: {err}")
|
|
|
|
total = len(testcases)
|
|
failed = total - passed
|
|
print(f"\nsummary: {passed}/{total} passed, {failed} failed")
|
|
return 0 if failed == 0 else 1
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|