From bb717c11b760819a89d15ac40fcb3ea97754201a Mon Sep 17 00:00:00 2001 From: Hydrostic Date: Fri, 15 May 2026 11:05:27 +0800 Subject: [PATCH] feat(tool): Add tools to test asm and ir --- tools/run_arm_asm.py | 19 ++- tools/run_asm_tests.py | 321 +++++++++++++++++++++++++++++++++++++++++ tools/run_ir_tests.py | 301 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 638 insertions(+), 3 deletions(-) create mode 100755 tools/run_asm_tests.py create mode 100755 tools/run_ir_tests.py diff --git a/tools/run_arm_asm.py b/tools/run_arm_asm.py index 540bc28..89198aa 100755 --- a/tools/run_arm_asm.py +++ b/tools/run_arm_asm.py @@ -84,6 +84,11 @@ def parse_args() -> argparse.Namespace: action="store_true", help="do not compile tests/std.c with the assembly file", ) + parser.add_argument( + "--stdin-file", + type=Path, + help="file used as stdin when running the compiled program", + ) args = parser.parse_args(tool_args) args.program_args = program_args return args @@ -95,9 +100,13 @@ def require_command(command: str) -> None: sys.exit(127) -def run_command(command: list[str]) -> subprocess.CompletedProcess[bytes]: +def run_command( + command: list[str], + *, + stdin: bytes | None = None, +) -> subprocess.CompletedProcess[bytes]: try: - return subprocess.run(command, capture_output=True, check=False) + return subprocess.run(command, input=stdin, capture_output=True, check=False) except OSError as err: print(f"error: failed to run {command[0]}: {err}", file=sys.stderr) sys.exit(127) @@ -125,6 +134,9 @@ def main() -> int: if not args.no_std_c and not std_c.is_file(): print(f"error: std C file not found: {std_c}", file=sys.stderr) return 2 + if args.stdin_file is not None and not args.stdin_file.is_file(): + print(f"error: stdin file not found: {args.stdin_file}", file=sys.stderr) + return 2 require_command(args.cc) require_command(args.qemu) @@ -164,7 +176,8 @@ def main() -> int: print("gdb: use 'si' or 'c'; do not use 'run'", file=sys.stderr, flush=True) qemu_cmd.extend([str(output), *args.program_args]) - run_result = run_command(qemu_cmd) + stdin = args.stdin_file.read_bytes() if args.stdin_file is not None else None + run_result = run_command(qemu_cmd, stdin=stdin) sys.stdout.buffer.write(run_result.stdout) sys.stderr.buffer.write(run_result.stderr) if run_result.stdout and not run_result.stdout.endswith(b"\n"): diff --git a/tools/run_asm_tests.py b/tools/run_asm_tests.py new file mode 100755 index 0000000..c1a1d8a --- /dev/null +++ b/tools/run_asm_tests.py @@ -0,0 +1,321 @@ +#!/usr/bin/env python3 +"""Run selected numbered testcases through the ARM assembly backend.""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class ExpectedOutput: + stdout: bytes + returncode: int + + +@dataclass(frozen=True) +class Testcase: + number: int + source: Path + input_file: Path | None + output_file: Path | None + + +@dataclass(frozen=True) +class RunOutput: + stdout: bytes + returncode: int + stderr: bytes + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Run numbered testcases like 1-5,7 with " + "`cargo run -- -S -D -o output.s testcase.c`, then " + "`tools/run_arm_asm.py output.s`, and compare with testcase.out." + ) + ) + parser.add_argument("numbers", help="testcase numbers, for example: 1-5,7") + parser.add_argument( + "--testcases", + type=Path, + default=Path("testcases"), + help="testcase directory (default: testcases)", + ) + parser.add_argument( + "--asm", + type=Path, + default=Path("output.s"), + help="assembly output path reused for each testcase (default: output.s)", + ) + parser.add_argument( + "--runner", + type=Path, + default=Path("tools/run_arm_asm.py"), + help="assembly runner path (default: tools/run_arm_asm.py)", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="print compiler/runner stderr for passing cases too", + ) + return parser.parse_args() + + +def parse_numbers(spec: str) -> list[int]: + numbers: list[int] = [] + seen: set[int] = set() + + for raw_part in spec.split(","): + part = raw_part.strip() + if not part: + continue + + if "-" in part: + bounds = part.split("-", 1) + if len(bounds) != 2 or not bounds[0] or not bounds[1]: + raise ValueError(f"invalid range: {part!r}") + start, end = int(bounds[0]), int(bounds[1]) + step = 1 if start <= end else -1 + values = range(start, end + step, step) + else: + values = (int(part),) + + for value in values: + if value <= 0: + raise ValueError(f"testcase number must be positive: {value}") + if value not in seen: + seen.add(value) + numbers.append(value) + + if not numbers: + raise ValueError("no testcase numbers specified") + return numbers + + +def find_testcase(testcases_dir: Path, number: int) -> Testcase: + prefix = f"{number:03d}_" + matches = sorted(testcases_dir.glob(f"{prefix}*.c")) + if not matches: + raise FileNotFoundError(f"no C testcase found for {number} ({prefix}*.c)") + if len(matches) > 1: + joined = ", ".join(str(path) for path in matches) + raise RuntimeError(f"multiple C testcases found for {number}: {joined}") + + source = matches[0] + stem = source.with_suffix("") + input_file = stem.with_suffix(".in") + output_file = stem.with_suffix(".out") + return Testcase( + number=number, + source=source, + input_file=input_file if input_file.is_file() else None, + output_file=output_file if output_file.is_file() else None, + ) + + +def parse_expected(output_file: Path) -> ExpectedOutput: + data = output_file.read_bytes() + lines = data.splitlines(keepends=True) + if not lines: + raise ValueError(f"empty expected output file: {output_file}") + + returncode_line = lines[-1].strip() + try: + returncode = int(returncode_line) + except ValueError as err: + raise ValueError( + f"last line of {output_file} is not an integer return code: " + f"{returncode_line!r}" + ) from err + + return ExpectedOutput(stdout=b"".join(lines[:-1]), returncode=returncode) + + +def normalize_returncode(returncode: int) -> int: + return returncode & 255 + + +def strip_trailing_newlines(data: bytes) -> bytes: + return data.rstrip(b"\r\n") + + +def run_command(command: list[str]) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + command, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + +def format_bytes(data: bytes) -> str: + if not data: + return "" + try: + return data.decode() + except UnicodeDecodeError: + return repr(data) + + +def format_bytes_debug(data: bytes) -> str: + return f"{data!r} ({len(data)} bytes)" + + +def parse_runner_output(result: subprocess.CompletedProcess[bytes]) -> RunOutput: + lines = result.stdout.splitlines(keepends=True) + if not lines: + return RunOutput(stdout=b"", returncode=normalize_returncode(result.returncode), stderr=result.stderr) + + last_line = lines[-1].strip() + marker = b"$? = " + if not last_line.startswith(marker): + return RunOutput(stdout=result.stdout, returncode=normalize_returncode(result.returncode), stderr=result.stderr) + + try: + returncode = int(last_line[len(marker) :]) + except ValueError: + return RunOutput(stdout=result.stdout, returncode=normalize_returncode(result.returncode), stderr=result.stderr) + + return RunOutput(stdout=b"".join(lines[:-1]), returncode=normalize_returncode(returncode), stderr=result.stderr) + + +def print_failure_detail( + testcase: Testcase, + *, + expected: ExpectedOutput | None, + actual_stdout: bytes, + actual_returncode: int | None, + stderr: bytes, + verbose: bool, +) -> None: + print(f" source: {testcase.source}") + if testcase.input_file is not None: + print(f" input: {testcase.input_file}") + if expected is not None: + print(" expected stdout:") + print(format_bytes(expected.stdout), end="" if expected.stdout.endswith(b"\n") else "\n") + print(f" expected stdout bytes: {format_bytes_debug(expected.stdout)}") + print(f" expected returncode: {expected.returncode}") + print(" actual stdout:") + print(format_bytes(actual_stdout), end="" if actual_stdout.endswith(b"\n") else "\n") + print(f" actual stdout bytes: {format_bytes_debug(actual_stdout)}") + if actual_returncode is not None: + print(f" actual returncode: {actual_returncode}") + if stderr and (verbose or expected is None): + print(" stderr:") + print(format_bytes(stderr), end="" if stderr.endswith(b"\n") else "\n") + + +def run_testcase( + testcase: Testcase, + *, + asm_file: Path, + runner: Path, + verbose: bool, +) -> bool: + compile_cmd = [ + "cargo", + "run", + "--", + "-S", + "-D", + "-o", + str(asm_file), + str(testcase.source), + ] + compile_result = run_command(compile_cmd) + if compile_result.returncode != 0: + print(f"[FAIL] {testcase.number:03d} compile failed") + print_failure_detail( + testcase, + expected=None, + actual_stdout=compile_result.stdout, + actual_returncode=compile_result.returncode, + stderr=compile_result.stderr, + verbose=True, + ) + return False + + run_cmd = [str(runner), str(asm_file)] + if testcase.input_file is not None: + run_cmd.extend(["--stdin-file", str(testcase.input_file)]) + run_result = parse_runner_output(run_command(run_cmd)) + + if testcase.output_file is None: + print(f"[SKIP] {testcase.number:03d} missing .out: {testcase.source.name}") + print_failure_detail( + testcase, + expected=None, + actual_stdout=run_result.stdout, + actual_returncode=run_result.returncode, + stderr=run_result.stderr, + verbose=verbose, + ) + return False + + expected = parse_expected(testcase.output_file) + ok = ( + strip_trailing_newlines(run_result.stdout) + == strip_trailing_newlines(expected.stdout) + and run_result.returncode == expected.returncode + ) + + status = "PASS" if ok else "FAIL" + print(f"[{status}] {testcase.number:03d} {testcase.source.name}") + if not ok or (verbose and (compile_result.stderr or run_result.stderr)): + if verbose and compile_result.stderr: + print(" compiler stderr:") + print( + format_bytes(compile_result.stderr), + end="" if compile_result.stderr.endswith(b"\n") else "\n", + ) + print_failure_detail( + testcase, + expected=expected, + actual_stdout=run_result.stdout, + actual_returncode=run_result.returncode, + stderr=run_result.stderr, + verbose=verbose, + ) + return ok + + +def main() -> int: + args = parse_args() + + try: + numbers = parse_numbers(args.numbers) + testcases = [find_testcase(args.testcases, number) for number in numbers] + except (OSError, RuntimeError, ValueError) as err: + print(f"error: {err}", file=sys.stderr) + return 2 + + passed = 0 + for testcase in testcases: + try: + if run_testcase( + testcase, + asm_file=args.asm, + runner=args.runner, + verbose=args.verbose, + ): + passed += 1 + except OSError as err: + print(f"[FAIL] {testcase.number:03d} failed to run command: {err}") + except ValueError as err: + print(f"[FAIL] {testcase.number:03d} invalid expected output: {err}") + + total = len(testcases) + failed = total - passed + print(f"\nsummary: {passed}/{total} passed, {failed} failed") + return 0 if failed == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tools/run_ir_tests.py b/tools/run_ir_tests.py new file mode 100755 index 0000000..486ca69 --- /dev/null +++ b/tools/run_ir_tests.py @@ -0,0 +1,301 @@ +#!/usr/bin/env python3 +"""Run selected numbered testcases through the IR backend and IRCompiler.""" + +from __future__ import annotations + +import argparse +import subprocess +import sys +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class ExpectedOutput: + stdout: bytes + returncode: int + + +@dataclass(frozen=True) +class Testcase: + number: int + source: Path + input_file: Path | None + output_file: Path | None + + +def parse_args() -> argparse.Namespace: + parser = argparse.ArgumentParser( + description=( + "Run numbered testcases like 1-5,7 with " + "`cargo run -- -I -S -D -o output.ir testcase.c`, then " + "`tools/IRCompiler -R output.ir`, and compare with testcase.out." + ) + ) + parser.add_argument("numbers", help="testcase numbers, for example: 1-5,7") + parser.add_argument( + "--testcases", + type=Path, + default=Path("testcases"), + help="testcase directory (default: testcases)", + ) + parser.add_argument( + "--ir", + type=Path, + default=Path("output.ir"), + help="IR output path reused for each testcase (default: output.ir)", + ) + parser.add_argument( + "--ircompiler", + type=Path, + default=Path("tools/IRCompiler"), + help="IRCompiler path (default: tools/IRCompiler)", + ) + parser.add_argument( + "-v", + "--verbose", + action="store_true", + help="print compiler/interpreter stderr for passing cases too", + ) + return parser.parse_args() + + +def parse_numbers(spec: str) -> list[int]: + numbers: list[int] = [] + seen: set[int] = set() + + for raw_part in spec.split(","): + part = raw_part.strip() + if not part: + continue + + if "-" in part: + bounds = part.split("-", 1) + if len(bounds) != 2 or not bounds[0] or not bounds[1]: + raise ValueError(f"invalid range: {part!r}") + start, end = int(bounds[0]), int(bounds[1]) + step = 1 if start <= end else -1 + values = range(start, end + step, step) + else: + values = (int(part),) + + for value in values: + if value <= 0: + raise ValueError(f"testcase number must be positive: {value}") + if value not in seen: + seen.add(value) + numbers.append(value) + + if not numbers: + raise ValueError("no testcase numbers specified") + return numbers + + +def find_testcase(testcases_dir: Path, number: int) -> Testcase: + prefix = f"{number:03d}_" + matches = sorted(testcases_dir.glob(f"{prefix}*.c")) + if not matches: + raise FileNotFoundError(f"no C testcase found for {number} ({prefix}*.c)") + if len(matches) > 1: + joined = ", ".join(str(path) for path in matches) + raise RuntimeError(f"multiple C testcases found for {number}: {joined}") + + source = matches[0] + stem = source.with_suffix("") + input_file = stem.with_suffix(".in") + output_file = stem.with_suffix(".out") + return Testcase( + number=number, + source=source, + input_file=input_file if input_file.is_file() else None, + output_file=output_file if output_file.is_file() else None, + ) + + +def parse_expected(output_file: Path) -> ExpectedOutput: + data = output_file.read_bytes() + lines = data.splitlines(keepends=True) + if not lines: + raise ValueError(f"empty expected output file: {output_file}") + + returncode_line = lines[-1].strip() + try: + returncode = int(returncode_line) + except ValueError as err: + raise ValueError( + f"last line of {output_file} is not an integer return code: " + f"{returncode_line!r}" + ) from err + + return ExpectedOutput(stdout=b"".join(lines[:-1]), returncode=returncode) + + +def normalize_returncode(returncode: int) -> int: + return returncode & 255 + + +def strip_trailing_newlines(data: bytes) -> bytes: + return data.rstrip(b"\r\n") + + +def run_command( + command: list[str], + *, + stdin: bytes | None = None, +) -> subprocess.CompletedProcess[bytes]: + return subprocess.run( + command, + input=stdin, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + +def format_bytes(data: bytes) -> str: + if not data: + return "" + try: + return data.decode() + except UnicodeDecodeError: + return repr(data) + + +def format_bytes_debug(data: bytes) -> str: + return f"{data!r} ({len(data)} bytes)" + + +def print_failure_detail( + testcase: Testcase, + *, + expected: ExpectedOutput | None, + actual_stdout: bytes, + actual_returncode: int | None, + stderr: bytes, + verbose: bool, +) -> None: + print(f" source: {testcase.source}") + if testcase.input_file is not None: + print(f" input: {testcase.input_file}") + if expected is not None: + print(" expected stdout:") + print(format_bytes(expected.stdout), end="" if expected.stdout.endswith(b"\n") else "\n") + print(f" expected stdout bytes: {format_bytes_debug(expected.stdout)}") + print(f" expected returncode: {expected.returncode}") + print(" actual stdout:") + print(format_bytes(actual_stdout), end="" if actual_stdout.endswith(b"\n") else "\n") + print(f" actual stdout bytes: {format_bytes_debug(actual_stdout)}") + if actual_returncode is not None: + print(f" actual returncode: {actual_returncode}") + if stderr and (verbose or expected is None): + print(" stderr:") + print(format_bytes(stderr), end="" if stderr.endswith(b"\n") else "\n") + + +def run_testcase( + testcase: Testcase, + *, + ir_file: Path, + ircompiler: Path, + verbose: bool, +) -> bool: + compile_cmd = [ + "cargo", + "run", + "--", + "-I", + "-S", + "-D", + "-o", + str(ir_file), + str(testcase.source), + ] + compile_result = run_command(compile_cmd) + if compile_result.returncode != 0: + print(f"[FAIL] {testcase.number:03d} compile failed") + print_failure_detail( + testcase, + expected=None, + actual_stdout=compile_result.stdout, + actual_returncode=compile_result.returncode, + stderr=compile_result.stderr, + verbose=True, + ) + return False + + stdin = testcase.input_file.read_bytes() if testcase.input_file is not None else None + run_result = run_command([str(ircompiler), "-R", str(ir_file)], stdin=stdin) + + if testcase.output_file is None: + print(f"[SKIP] {testcase.number:03d} missing .out: {testcase.source.name}") + print_failure_detail( + testcase, + expected=None, + actual_stdout=run_result.stdout, + actual_returncode=run_result.returncode, + stderr=run_result.stderr, + verbose=verbose, + ) + return False + + expected = parse_expected(testcase.output_file) + actual_returncode = normalize_returncode(run_result.returncode) + ok = ( + strip_trailing_newlines(run_result.stdout) + == strip_trailing_newlines(expected.stdout) + and actual_returncode == expected.returncode + ) + + status = "PASS" if ok else "FAIL" + print(f"[{status}] {testcase.number:03d} {testcase.source.name}") + if not ok or (verbose and (compile_result.stderr or run_result.stderr)): + if verbose and compile_result.stderr: + print(" compiler stderr:") + print( + format_bytes(compile_result.stderr), + end="" if compile_result.stderr.endswith(b"\n") else "\n", + ) + print_failure_detail( + testcase, + expected=expected, + actual_stdout=run_result.stdout, + actual_returncode=actual_returncode, + stderr=run_result.stderr, + verbose=verbose, + ) + return ok + + +def main() -> int: + args = parse_args() + + try: + numbers = parse_numbers(args.numbers) + testcases = [find_testcase(args.testcases, number) for number in numbers] + except (OSError, RuntimeError, ValueError) as err: + print(f"error: {err}", file=sys.stderr) + return 2 + + passed = 0 + for testcase in testcases: + try: + if run_testcase( + testcase, + ir_file=args.ir, + ircompiler=args.ircompiler, + verbose=args.verbose, + ): + passed += 1 + except OSError as err: + print(f"[FAIL] {testcase.number:03d} failed to run command: {err}") + except ValueError as err: + print(f"[FAIL] {testcase.number:03d} invalid expected output: {err}") + + total = len(testcases) + failed = total - passed + print(f"\nsummary: {passed}/{total} passed, {failed} failed") + return 0 if failed == 0 else 1 + + +if __name__ == "__main__": + raise SystemExit(main())