feat(tool): Add tools to test asm and ir

tools/run_arm_asm.py

@@ -84,6 +84,11 @@ def parse_args() -> argparse.Namespace:
         action="store_true",
         help="do not compile tests/std.c with the assembly file",
     )
+    parser.add_argument(
+        "--stdin-file",
+        type=Path,
+        help="file used as stdin when running the compiled program",
+    )
     args = parser.parse_args(tool_args)
     args.program_args = program_args
     return args
@@ -95,9 +100,13 @@ def require_command(command: str) -> None:
         sys.exit(127)
 
 
-def run_command(command: list[str]) -> subprocess.CompletedProcess[bytes]:
+def run_command(
+    command: list[str],
+    *,
+    stdin: bytes | None = None,
+) -> subprocess.CompletedProcess[bytes]:
     try:
-        return subprocess.run(command, capture_output=True, check=False)
+        return subprocess.run(command, input=stdin, capture_output=True, check=False)
     except OSError as err:
         print(f"error: failed to run {command[0]}: {err}", file=sys.stderr)
         sys.exit(127)
@@ -125,6 +134,9 @@ def main() -> int:
     if not args.no_std_c and not std_c.is_file():
         print(f"error: std C file not found: {std_c}", file=sys.stderr)
         return 2
+    if args.stdin_file is not None and not args.stdin_file.is_file():
+        print(f"error: stdin file not found: {args.stdin_file}", file=sys.stderr)
+        return 2
 
     require_command(args.cc)
     require_command(args.qemu)
@@ -164,7 +176,8 @@ def main() -> int:
             print("gdb: use 'si' or 'c'; do not use 'run'", file=sys.stderr, flush=True)
         qemu_cmd.extend([str(output), *args.program_args])
 
-        run_result = run_command(qemu_cmd)
+        stdin = args.stdin_file.read_bytes() if args.stdin_file is not None else None
+        run_result = run_command(qemu_cmd, stdin=stdin)
         sys.stdout.buffer.write(run_result.stdout)
         sys.stderr.buffer.write(run_result.stderr)
         if run_result.stdout and not run_result.stdout.endswith(b"\n"):

tools/run_asm_tests.py

@@ -0,0 +1,321 @@
+#!/usr/bin/env python3
+"""Run selected numbered testcases through the ARM assembly backend."""
+
+from __future__ import annotations
+
+import argparse
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class ExpectedOutput:
+    stdout: bytes
+    returncode: int
+
+
+@dataclass(frozen=True)
+class Testcase:
+    number: int
+    source: Path
+    input_file: Path | None
+    output_file: Path | None
+
+
+@dataclass(frozen=True)
+class RunOutput:
+    stdout: bytes
+    returncode: int
+    stderr: bytes
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Run numbered testcases like 1-5,7 with "
+            "`cargo run -- -S -D -o output.s testcase.c`, then "
+            "`tools/run_arm_asm.py output.s`, and compare with testcase.out."
+        )
+    )
+    parser.add_argument("numbers", help="testcase numbers, for example: 1-5,7")
+    parser.add_argument(
+        "--testcases",
+        type=Path,
+        default=Path("testcases"),
+        help="testcase directory (default: testcases)",
+    )
+    parser.add_argument(
+        "--asm",
+        type=Path,
+        default=Path("output.s"),
+        help="assembly output path reused for each testcase (default: output.s)",
+    )
+    parser.add_argument(
+        "--runner",
+        type=Path,
+        default=Path("tools/run_arm_asm.py"),
+        help="assembly runner path (default: tools/run_arm_asm.py)",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="print compiler/runner stderr for passing cases too",
+    )
+    return parser.parse_args()
+
+
+def parse_numbers(spec: str) -> list[int]:
+    numbers: list[int] = []
+    seen: set[int] = set()
+
+    for raw_part in spec.split(","):
+        part = raw_part.strip()
+        if not part:
+            continue
+
+        if "-" in part:
+            bounds = part.split("-", 1)
+            if len(bounds) != 2 or not bounds[0] or not bounds[1]:
+                raise ValueError(f"invalid range: {part!r}")
+            start, end = int(bounds[0]), int(bounds[1])
+            step = 1 if start <= end else -1
+            values = range(start, end + step, step)
+        else:
+            values = (int(part),)
+
+        for value in values:
+            if value <= 0:
+                raise ValueError(f"testcase number must be positive: {value}")
+            if value not in seen:
+                seen.add(value)
+                numbers.append(value)
+
+    if not numbers:
+        raise ValueError("no testcase numbers specified")
+    return numbers
+
+
+def find_testcase(testcases_dir: Path, number: int) -> Testcase:
+    prefix = f"{number:03d}_"
+    matches = sorted(testcases_dir.glob(f"{prefix}*.c"))
+    if not matches:
+        raise FileNotFoundError(f"no C testcase found for {number} ({prefix}*.c)")
+    if len(matches) > 1:
+        joined = ", ".join(str(path) for path in matches)
+        raise RuntimeError(f"multiple C testcases found for {number}: {joined}")
+
+    source = matches[0]
+    stem = source.with_suffix("")
+    input_file = stem.with_suffix(".in")
+    output_file = stem.with_suffix(".out")
+    return Testcase(
+        number=number,
+        source=source,
+        input_file=input_file if input_file.is_file() else None,
+        output_file=output_file if output_file.is_file() else None,
+    )
+
+
+def parse_expected(output_file: Path) -> ExpectedOutput:
+    data = output_file.read_bytes()
+    lines = data.splitlines(keepends=True)
+    if not lines:
+        raise ValueError(f"empty expected output file: {output_file}")
+
+    returncode_line = lines[-1].strip()
+    try:
+        returncode = int(returncode_line)
+    except ValueError as err:
+        raise ValueError(
+            f"last line of {output_file} is not an integer return code: "
+            f"{returncode_line!r}"
+        ) from err
+
+    return ExpectedOutput(stdout=b"".join(lines[:-1]), returncode=returncode)
+
+
+def normalize_returncode(returncode: int) -> int:
+    return returncode & 255
+
+
+def strip_trailing_newlines(data: bytes) -> bytes:
+    return data.rstrip(b"\r\n")
+
+
+def run_command(command: list[str]) -> subprocess.CompletedProcess[bytes]:
+    return subprocess.run(
+        command,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        check=False,
+    )
+
+
+def format_bytes(data: bytes) -> str:
+    if not data:
+        return "<empty>"
+    try:
+        return data.decode()
+    except UnicodeDecodeError:
+        return repr(data)
+
+
+def format_bytes_debug(data: bytes) -> str:
+    return f"{data!r} ({len(data)} bytes)"
+
+
+def parse_runner_output(result: subprocess.CompletedProcess[bytes]) -> RunOutput:
+    lines = result.stdout.splitlines(keepends=True)
+    if not lines:
+        return RunOutput(stdout=b"", returncode=normalize_returncode(result.returncode), stderr=result.stderr)
+
+    last_line = lines[-1].strip()
+    marker = b"$? = "
+    if not last_line.startswith(marker):
+        return RunOutput(stdout=result.stdout, returncode=normalize_returncode(result.returncode), stderr=result.stderr)
+
+    try:
+        returncode = int(last_line[len(marker) :])
+    except ValueError:
+        return RunOutput(stdout=result.stdout, returncode=normalize_returncode(result.returncode), stderr=result.stderr)
+
+    return RunOutput(stdout=b"".join(lines[:-1]), returncode=normalize_returncode(returncode), stderr=result.stderr)
+
+
+def print_failure_detail(
+    testcase: Testcase,
+    *,
+    expected: ExpectedOutput | None,
+    actual_stdout: bytes,
+    actual_returncode: int | None,
+    stderr: bytes,
+    verbose: bool,
+) -> None:
+    print(f"  source: {testcase.source}")
+    if testcase.input_file is not None:
+        print(f"  input:  {testcase.input_file}")
+    if expected is not None:
+        print("  expected stdout:")
+        print(format_bytes(expected.stdout), end="" if expected.stdout.endswith(b"\n") else "\n")
+        print(f"  expected stdout bytes: {format_bytes_debug(expected.stdout)}")
+        print(f"  expected returncode: {expected.returncode}")
+    print("  actual stdout:")
+    print(format_bytes(actual_stdout), end="" if actual_stdout.endswith(b"\n") else "\n")
+    print(f"  actual stdout bytes:   {format_bytes_debug(actual_stdout)}")
+    if actual_returncode is not None:
+        print(f"  actual returncode:   {actual_returncode}")
+    if stderr and (verbose or expected is None):
+        print("  stderr:")
+        print(format_bytes(stderr), end="" if stderr.endswith(b"\n") else "\n")
+
+
+def run_testcase(
+    testcase: Testcase,
+    *,
+    asm_file: Path,
+    runner: Path,
+    verbose: bool,
+) -> bool:
+    compile_cmd = [
+        "cargo",
+        "run",
+        "--",
+        "-S",
+        "-D",
+        "-o",
+        str(asm_file),
+        str(testcase.source),
+    ]
+    compile_result = run_command(compile_cmd)
+    if compile_result.returncode != 0:
+        print(f"[FAIL] {testcase.number:03d} compile failed")
+        print_failure_detail(
+            testcase,
+            expected=None,
+            actual_stdout=compile_result.stdout,
+            actual_returncode=compile_result.returncode,
+            stderr=compile_result.stderr,
+            verbose=True,
+        )
+        return False
+
+    run_cmd = [str(runner), str(asm_file)]
+    if testcase.input_file is not None:
+        run_cmd.extend(["--stdin-file", str(testcase.input_file)])
+    run_result = parse_runner_output(run_command(run_cmd))
+
+    if testcase.output_file is None:
+        print(f"[SKIP] {testcase.number:03d} missing .out: {testcase.source.name}")
+        print_failure_detail(
+            testcase,
+            expected=None,
+            actual_stdout=run_result.stdout,
+            actual_returncode=run_result.returncode,
+            stderr=run_result.stderr,
+            verbose=verbose,
+        )
+        return False
+
+    expected = parse_expected(testcase.output_file)
+    ok = (
+        strip_trailing_newlines(run_result.stdout)
+        == strip_trailing_newlines(expected.stdout)
+        and run_result.returncode == expected.returncode
+    )
+
+    status = "PASS" if ok else "FAIL"
+    print(f"[{status}] {testcase.number:03d} {testcase.source.name}")
+    if not ok or (verbose and (compile_result.stderr or run_result.stderr)):
+        if verbose and compile_result.stderr:
+            print("  compiler stderr:")
+            print(
+                format_bytes(compile_result.stderr),
+                end="" if compile_result.stderr.endswith(b"\n") else "\n",
+            )
+        print_failure_detail(
+            testcase,
+            expected=expected,
+            actual_stdout=run_result.stdout,
+            actual_returncode=run_result.returncode,
+            stderr=run_result.stderr,
+            verbose=verbose,
+        )
+    return ok
+
+
+def main() -> int:
+    args = parse_args()
+
+    try:
+        numbers = parse_numbers(args.numbers)
+        testcases = [find_testcase(args.testcases, number) for number in numbers]
+    except (OSError, RuntimeError, ValueError) as err:
+        print(f"error: {err}", file=sys.stderr)
+        return 2
+
+    passed = 0
+    for testcase in testcases:
+        try:
+            if run_testcase(
+                testcase,
+                asm_file=args.asm,
+                runner=args.runner,
+                verbose=args.verbose,
+            ):
+                passed += 1
+        except OSError as err:
+            print(f"[FAIL] {testcase.number:03d} failed to run command: {err}")
+        except ValueError as err:
+            print(f"[FAIL] {testcase.number:03d} invalid expected output: {err}")
+
+    total = len(testcases)
+    failed = total - passed
+    print(f"\nsummary: {passed}/{total} passed, {failed} failed")
+    return 0 if failed == 0 else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

tools/run_ir_tests.py

@@ -0,0 +1,301 @@
+#!/usr/bin/env python3
+"""Run selected numbered testcases through the IR backend and IRCompiler."""
+
+from __future__ import annotations
+
+import argparse
+import subprocess
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class ExpectedOutput:
+    stdout: bytes
+    returncode: int
+
+
+@dataclass(frozen=True)
+class Testcase:
+    number: int
+    source: Path
+    input_file: Path | None
+    output_file: Path | None
+
+
+def parse_args() -> argparse.Namespace:
+    parser = argparse.ArgumentParser(
+        description=(
+            "Run numbered testcases like 1-5,7 with "
+            "`cargo run -- -I -S -D -o output.ir testcase.c`, then "
+            "`tools/IRCompiler -R output.ir`, and compare with testcase.out."
+        )
+    )
+    parser.add_argument("numbers", help="testcase numbers, for example: 1-5,7")
+    parser.add_argument(
+        "--testcases",
+        type=Path,
+        default=Path("testcases"),
+        help="testcase directory (default: testcases)",
+    )
+    parser.add_argument(
+        "--ir",
+        type=Path,
+        default=Path("output.ir"),
+        help="IR output path reused for each testcase (default: output.ir)",
+    )
+    parser.add_argument(
+        "--ircompiler",
+        type=Path,
+        default=Path("tools/IRCompiler"),
+        help="IRCompiler path (default: tools/IRCompiler)",
+    )
+    parser.add_argument(
+        "-v",
+        "--verbose",
+        action="store_true",
+        help="print compiler/interpreter stderr for passing cases too",
+    )
+    return parser.parse_args()
+
+
+def parse_numbers(spec: str) -> list[int]:
+    numbers: list[int] = []
+    seen: set[int] = set()
+
+    for raw_part in spec.split(","):
+        part = raw_part.strip()
+        if not part:
+            continue
+
+        if "-" in part:
+            bounds = part.split("-", 1)
+            if len(bounds) != 2 or not bounds[0] or not bounds[1]:
+                raise ValueError(f"invalid range: {part!r}")
+            start, end = int(bounds[0]), int(bounds[1])
+            step = 1 if start <= end else -1
+            values = range(start, end + step, step)
+        else:
+            values = (int(part),)
+
+        for value in values:
+            if value <= 0:
+                raise ValueError(f"testcase number must be positive: {value}")
+            if value not in seen:
+                seen.add(value)
+                numbers.append(value)
+
+    if not numbers:
+        raise ValueError("no testcase numbers specified")
+    return numbers
+
+
+def find_testcase(testcases_dir: Path, number: int) -> Testcase:
+    prefix = f"{number:03d}_"
+    matches = sorted(testcases_dir.glob(f"{prefix}*.c"))
+    if not matches:
+        raise FileNotFoundError(f"no C testcase found for {number} ({prefix}*.c)")
+    if len(matches) > 1:
+        joined = ", ".join(str(path) for path in matches)
+        raise RuntimeError(f"multiple C testcases found for {number}: {joined}")
+
+    source = matches[0]
+    stem = source.with_suffix("")
+    input_file = stem.with_suffix(".in")
+    output_file = stem.with_suffix(".out")
+    return Testcase(
+        number=number,
+        source=source,
+        input_file=input_file if input_file.is_file() else None,
+        output_file=output_file if output_file.is_file() else None,
+    )
+
+
+def parse_expected(output_file: Path) -> ExpectedOutput:
+    data = output_file.read_bytes()
+    lines = data.splitlines(keepends=True)
+    if not lines:
+        raise ValueError(f"empty expected output file: {output_file}")
+
+    returncode_line = lines[-1].strip()
+    try:
+        returncode = int(returncode_line)
+    except ValueError as err:
+        raise ValueError(
+            f"last line of {output_file} is not an integer return code: "
+            f"{returncode_line!r}"
+        ) from err
+
+    return ExpectedOutput(stdout=b"".join(lines[:-1]), returncode=returncode)
+
+
+def normalize_returncode(returncode: int) -> int:
+    return returncode & 255
+
+
+def strip_trailing_newlines(data: bytes) -> bytes:
+    return data.rstrip(b"\r\n")
+
+
+def run_command(
+    command: list[str],
+    *,
+    stdin: bytes | None = None,
+) -> subprocess.CompletedProcess[bytes]:
+    return subprocess.run(
+        command,
+        input=stdin,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        check=False,
+    )
+
+
+def format_bytes(data: bytes) -> str:
+    if not data:
+        return "<empty>"
+    try:
+        return data.decode()
+    except UnicodeDecodeError:
+        return repr(data)
+
+
+def format_bytes_debug(data: bytes) -> str:
+    return f"{data!r} ({len(data)} bytes)"
+
+
+def print_failure_detail(
+    testcase: Testcase,
+    *,
+    expected: ExpectedOutput | None,
+    actual_stdout: bytes,
+    actual_returncode: int | None,
+    stderr: bytes,
+    verbose: bool,
+) -> None:
+    print(f"  source: {testcase.source}")
+    if testcase.input_file is not None:
+        print(f"  input:  {testcase.input_file}")
+    if expected is not None:
+        print("  expected stdout:")
+        print(format_bytes(expected.stdout), end="" if expected.stdout.endswith(b"\n") else "\n")
+        print(f"  expected stdout bytes: {format_bytes_debug(expected.stdout)}")
+        print(f"  expected returncode: {expected.returncode}")
+    print("  actual stdout:")
+    print(format_bytes(actual_stdout), end="" if actual_stdout.endswith(b"\n") else "\n")
+    print(f"  actual stdout bytes:   {format_bytes_debug(actual_stdout)}")
+    if actual_returncode is not None:
+        print(f"  actual returncode:   {actual_returncode}")
+    if stderr and (verbose or expected is None):
+        print("  stderr:")
+        print(format_bytes(stderr), end="" if stderr.endswith(b"\n") else "\n")
+
+
+def run_testcase(
+    testcase: Testcase,
+    *,
+    ir_file: Path,
+    ircompiler: Path,
+    verbose: bool,
+) -> bool:
+    compile_cmd = [
+        "cargo",
+        "run",
+        "--",
+        "-I",
+        "-S",
+        "-D",
+        "-o",
+        str(ir_file),
+        str(testcase.source),
+    ]
+    compile_result = run_command(compile_cmd)
+    if compile_result.returncode != 0:
+        print(f"[FAIL] {testcase.number:03d} compile failed")
+        print_failure_detail(
+            testcase,
+            expected=None,
+            actual_stdout=compile_result.stdout,
+            actual_returncode=compile_result.returncode,
+            stderr=compile_result.stderr,
+            verbose=True,
+        )
+        return False
+
+    stdin = testcase.input_file.read_bytes() if testcase.input_file is not None else None
+    run_result = run_command([str(ircompiler), "-R", str(ir_file)], stdin=stdin)
+
+    if testcase.output_file is None:
+        print(f"[SKIP] {testcase.number:03d} missing .out: {testcase.source.name}")
+        print_failure_detail(
+            testcase,
+            expected=None,
+            actual_stdout=run_result.stdout,
+            actual_returncode=run_result.returncode,
+            stderr=run_result.stderr,
+            verbose=verbose,
+        )
+        return False
+
+    expected = parse_expected(testcase.output_file)
+    actual_returncode = normalize_returncode(run_result.returncode)
+    ok = (
+        strip_trailing_newlines(run_result.stdout)
+        == strip_trailing_newlines(expected.stdout)
+        and actual_returncode == expected.returncode
+    )
+
+    status = "PASS" if ok else "FAIL"
+    print(f"[{status}] {testcase.number:03d} {testcase.source.name}")
+    if not ok or (verbose and (compile_result.stderr or run_result.stderr)):
+        if verbose and compile_result.stderr:
+            print("  compiler stderr:")
+            print(
+                format_bytes(compile_result.stderr),
+                end="" if compile_result.stderr.endswith(b"\n") else "\n",
+            )
+        print_failure_detail(
+            testcase,
+            expected=expected,
+            actual_stdout=run_result.stdout,
+            actual_returncode=actual_returncode,
+            stderr=run_result.stderr,
+            verbose=verbose,
+        )
+    return ok
+
+
+def main() -> int:
+    args = parse_args()
+
+    try:
+        numbers = parse_numbers(args.numbers)
+        testcases = [find_testcase(args.testcases, number) for number in numbers]
+    except (OSError, RuntimeError, ValueError) as err:
+        print(f"error: {err}", file=sys.stderr)
+        return 2
+
+    passed = 0
+    for testcase in testcases:
+        try:
+            if run_testcase(
+                testcase,
+                ir_file=args.ir,
+                ircompiler=args.ircompiler,
+                verbose=args.verbose,
+            ):
+                passed += 1
+        except OSError as err:
+            print(f"[FAIL] {testcase.number:03d} failed to run command: {err}")
+        except ValueError as err:
+            print(f"[FAIL] {testcase.number:03d} invalid expected output: {err}")
+
+    total = len(testcases)
+    failed = total - passed
+    print(f"\nsummary: {passed}/{total} passed, {failed} failed")
+    return 0 if failed == 0 else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())