gauntlet

#!/usr/bin/env python3
"""gauntlet.py — does your check actually catch a fault?

A check (a linter, a validator, a test, a verifier) is only worth what it
catches. The quiet failure is a check that passes on input it was supposed to
reject — a green that means nothing. The way you find that out is to break the
input ON PURPOSE, run the check, and see whether it fires. If you damage the
file and the check still says PASS, the check has a hole.

gauntlet automates exactly that, safely:

  1. Copy the target file into a private, disposable sandbox. The original is
     NEVER touched.
  2. Inject ONE typed fault into the sandbox copy (truncate its tail, flip a
     byte, or apply a find/replace regression you name).
  3. Run YOUR check command against the mutated copy.
  4. Report the verdict:
       HELD    -- the check FAILED on the broken input (good: it caught the fault)
       ESCAPED -- the check PASSED on the broken input (bad: your check has a hole)

ONE FAULT PER SHOT, SANDBOX ONLY (the safety rails, kept from the tool this was
stripped from). Never buckshot: one typed fault per run, so a HELD/ESCAPED
verdict names exactly what got through. The sandbox is deleted when the run
ends -- abort is just "delete the temp dir," and the original file is read-only
to this tool by construction (it is only ever copied, never written).

THE VERDICT IS INVERTED ON PURPOSE.
A check that FAILS on broken input is doing its job -- so a non-zero exit from
your check is a gauntlet PASS (HELD). A check that PASSES on broken input has a
hole -- so a zero exit from your check is a gauntlet FAIL (ESCAPED). Read the
exit codes below with that inversion in mind.

EXITS
  0  HELD     -- the check caught the fault (your check FAILED on broken input)
  3  ESCAPED  -- the check missed the fault (your check PASSED on broken input)
  4  NO-FAULT -- the chosen fault could not be injected (e.g. --replace found no
                 match, or the file was too small to truncate); nothing tested
  2  USAGE    -- bad arguments, or the target file is missing

USAGE
  # Truncate the tail 10% and see whether your validator catches it:
  python3 gauntlet.py --target data.json --fault truncate --check "python3 validate.py {}"

  # Flip one byte in the middle:
  python3 gauntlet.py --target data.json --fault bitflip --check "python3 validate.py {}"

  # Apply a well-formed-wrong regression (the file still parses, it just lies):
  python3 gauntlet.py --target config.yaml --fault replace \\
      --from "version: 3" --to "version: 2" --check "python3 validate.py {}"

The check command runs against the mutated SANDBOX copy: `{}` in --check is
replaced with the sandbox path. If your check reads stdin instead, use
`--stdin` and gauntlet pipes the mutated bytes to it.

  python3 gauntlet.py --json     # machine-readable verdict
  python3 gauntlet.py --edge     # print the edge and exit
"""

import argparse
import os
import shutil
import subprocess
import sys
import tempfile

EXIT_HELD = 0
EXIT_USAGE = 2
EXIT_ESCAPED = 3
EXIT_NO_FAULT = 4

EDGE = (
    "gauntlet tests whether a check CATCHES the ONE fault you inject, not "
    "whether the check is correct in general: a HELD proves the check fired on "
    "this one broken input, never that it catches every fault. It runs your "
    "check command, so only point it at a command you trust. It only ever "
    "copies the target -- it never modifies your original file."
)


# --- the fault catalog: each returns (mutated_bytes, note) or (None, why) -----

def _truncate(data):
    """Tail-truncate 10% (byte loss). Well-formed-wrong on top-heavy files."""
    if len(data) < 10:
        return None, "file too small to truncate meaningfully"
    cut = int(len(data) * 0.9) or (len(data) - 1)
    return data[:cut], f"tail truncated: {len(data)}B -> {cut}B"


def _bitflip(data):
    """Flip one byte in the middle. Deterministic (always the midpoint byte)."""
    if not data:
        return None, "file is empty; nothing to flip"
    i = len(data) // 2
    b = bytearray(data)
    b[i] ^= 0xFF
    return bytes(b), f"byte flipped at offset {i}: 0x{data[i]:02x} -> 0x{b[i]:02x}"


def _replace(data, frm, to):
    """A find/replace regression: the file still parses, it just lies."""
    if frm is None or to is None:
        return None, "--replace requires --from and --to"
    fb, tb = frm.encode("utf-8"), to.encode("utf-8")
    if fb not in data:
        return None, f"--from string not found in target: {frm!r}"
    return data.replace(fb, tb, 1), f"replaced {frm!r} -> {to!r} (first match)"


def inject(data, fault, frm=None, to=None):
    if fault == "truncate":
        return _truncate(data)
    if fault == "bitflip":
        return _bitflip(data)
    if fault == "replace":
        return _replace(data, frm, to)
    return None, f"unknown fault: {fault}"


def run(target, fault, check_cmd, use_stdin=False, frm=None, to=None):
    """Copy target to a sandbox, inject one fault, run the check, report verdict.

    Never touches the original target. Returns a result dict.
    """
    result = {
        "target": target, "fault": fault, "status": "held",
        "note": "", "check_exit": None, "edge": EDGE,
    }
    if not os.path.isfile(target):
        result["status"] = "usage"
        result["error"] = f"target file not found: {target}"
        return result

    with open(target, "rb") as fh:
        original = fh.read()

    mutated, note = inject(original, fault, frm, to)
    result["note"] = note
    if mutated is None:
        result["status"] = "no-fault"
        return result
    if mutated == original:
        result["status"] = "no-fault"
        result["note"] = note + " (no change to bytes)"
        return result

    sandbox = tempfile.mkdtemp(prefix="gauntlet.")
    try:
        sandbox_file = os.path.join(sandbox, os.path.basename(target))
        with open(sandbox_file, "wb") as fh:
            fh.write(mutated)

        if use_stdin:
            proc = subprocess.run(
                check_cmd, shell=True, input=mutated,
                capture_output=True, timeout=300,
            )
        else:
            cmd = check_cmd.replace("{}", sandbox_file)
            if cmd == check_cmd and "{}" not in check_cmd:
                # no placeholder given: append the path
                cmd = f"{check_cmd} {sandbox_file}"
            proc = subprocess.run(
                cmd, shell=True, capture_output=True, timeout=300,
            )

        result["check_exit"] = proc.returncode
        # INVERSION: check FAILED (non-zero) on broken input => HELD (good).
        #            check PASSED (zero)  on broken input     => ESCAPED (bad).
        result["status"] = "held" if proc.returncode != 0 else "escaped"
        return result
    finally:
        shutil.rmtree(sandbox, ignore_errors=True)


def main(argv=None):
    ap = argparse.ArgumentParser(
        description="Does your check actually catch a fault you inject?",
        epilog="EDGE: " + EDGE,
    )
    ap.add_argument("--target", help="the file to inject a fault into (copied, never modified)")
    ap.add_argument("--fault", choices=["truncate", "bitflip", "replace"],
                    help="the single typed fault to inject")
    ap.add_argument("--check", help="the check command to run; {} is replaced with the sandbox path")
    ap.add_argument("--from", dest="frm", help="(--fault replace) the string to find")
    ap.add_argument("--to", dest="to", help="(--fault replace) the string to substitute")
    ap.add_argument("--stdin", action="store_true",
                    help="pipe the mutated bytes to the check on stdin instead of a path")
    ap.add_argument("--json", action="store_true", help="machine-readable verdict")
    ap.add_argument("--edge", action="store_true", help="print the edge and exit")
    args = ap.parse_args(argv)

    if args.edge:
        print(EDGE)
        return EXIT_HELD

    if not (args.target and args.fault and args.check):
        print("USAGE ERROR — --target, --fault and --check are all required",
              file=sys.stderr)
        return EXIT_USAGE

    res = run(args.target, args.fault, args.check,
              use_stdin=args.stdin, frm=args.frm, to=args.to)
    status = res["status"]

    if args.json:
        import json
        print(json.dumps(res, indent=2, sort_keys=True))
    else:
        if status == "held":
            print(f"HELD — the check caught the {args.fault} fault "
                  f"(check exited {res['check_exit']} on broken input)")
            print(f"  fault: {res['note']}")
        elif status == "escaped":
            print(f"ESCAPED — the check MISSED the {args.fault} fault "
                  f"(check exited 0 on broken input — it has a hole)")
            print(f"  fault: {res['note']}")
        elif status == "no-fault":
            print(f"NO-FAULT — could not inject: {res['note']}")
        else:
            print(f"USAGE ERROR — {res.get('error', 'bad arguments')}", file=sys.stderr)

    return {
        "held": EXIT_HELD, "escaped": EXIT_ESCAPED,
        "no-fault": EXIT_NO_FAULT, "usage": EXIT_USAGE,
    }[status]


if __name__ == "__main__":
    sys.exit(main())

#!/usr/bin/env python3
"""test_gauntlet.py — mutation-bitten behavior proof for the gauntlet gift.

stdlib only. Each test asserts a distinct BEHAVIOR so deleting the behavior
makes a test fail. Run:  python3 test_gauntlet.py
"""
import os
import shutil
import subprocess
import sys
import tempfile

HERE = os.path.dirname(os.path.abspath(__file__))
GIFT = os.path.join(HERE, "gauntlet.py")

HELD, USAGE, ESCAPED, NO_FAULT = 0, 2, 3, 4

# A check that PASSES (exit 0) only if the file is exactly the 200-byte original.
# Deterministic, stdlib-only, written per-test into the sandbox.
STRICT_CHECK = (
    "python3 -c \"import sys; d=open(sys.argv[1],'rb').read(); "
    "sys.exit(0 if len(d)==200 and d==b'A'*200 else 1)\" {}"
)
# A weak check that ALWAYS passes — used to prove ESCAPED detection.
WEAK_CHECK = "python3 -c \"import sys; sys.exit(0)\" {}"

results = []


def check(name, cond):
    results.append((name, bool(cond)))
    print(f"  [{'PASS' if cond else 'FAIL'}] {name}")


def _scratch_target(content=b"A" * 200):
    d = tempfile.mkdtemp(prefix="gauntlet-test.")
    p = os.path.join(d, "data.bin")
    with open(p, "wb") as f:
        f.write(content)
    return d, p


def _run(*args):
    p = subprocess.run([sys.executable, GIFT, *args],
                       capture_output=True, text=True, timeout=120)
    return p.returncode, p.stdout, p.stderr


def t_held_when_strict_check_catches_truncation():
    """A strict check that rejects a truncated file => HELD (exit 0)."""
    d, p = _scratch_target()
    try:
        rc, out, _ = _run("--target", p, "--fault", "truncate", "--check", STRICT_CHECK)
        check("HELD when strict check catches truncation", rc == HELD and "HELD" in out)
    finally:
        shutil.rmtree(d, ignore_errors=True)


def t_escaped_when_weak_check_misses():
    """A check that always passes => ESCAPED (exit 3): the hole is detected."""
    d, p = _scratch_target()
    try:
        rc, out, _ = _run("--target", p, "--fault", "truncate", "--check", WEAK_CHECK)
        check("ESCAPED when weak check misses the fault", rc == ESCAPED and "ESCAPED" in out)
    finally:
        shutil.rmtree(d, ignore_errors=True)


def t_original_never_modified():
    """The original target file is byte-identical after a run (sandbox only)."""
    d, p = _scratch_target()
    try:
        before = open(p, "rb").read()
        _run("--target", p, "--fault", "bitflip", "--check", WEAK_CHECK)
        after = open(p, "rb").read()
        check("original file untouched after a run", before == after)
    finally:
        shutil.rmtree(d, ignore_errors=True)


def t_bitflip_held_by_strict_check():
    """A single-byte flip is caught by the strict check => HELD."""
    d, p = _scratch_target()
    try:
        rc, out, _ = _run("--target", p, "--fault", "bitflip", "--check", STRICT_CHECK)
        check("HELD on bitflip caught by strict check", rc == HELD and "byte flipped" in out)
    finally:
        shutil.rmtree(d, ignore_errors=True)


def t_replace_regression_held():
    """A find/replace regression the check rejects => HELD; note names the swap."""
    d = tempfile.mkdtemp(prefix="gauntlet-test.")
    p = os.path.join(d, "cfg.txt")
    open(p, "w").write("version: 3\n")
    # check passes only if it still says 'version: 3'
    chk = "python3 -c \"import sys; sys.exit(0 if 'version: 3' in open(sys.argv[1]).read() else 1)\" {}"
    try:
        rc, out, _ = _run("--target", p, "--fault", "replace",
                          "--from", "version: 3", "--to", "version: 2", "--check", chk)
        check("HELD on replace regression + note names swap",
              rc == HELD and "version: 3" in out and "version: 2" in out)
    finally:
        shutil.rmtree(d, ignore_errors=True)


def t_no_fault_when_replace_string_absent():
    """--replace with a --from that isn't present => NO-FAULT (exit 4), not a verdict."""
    d = tempfile.mkdtemp(prefix="gauntlet-test.")
    p = os.path.join(d, "cfg.txt")
    open(p, "w").write("hello\n")
    try:
        rc, out, _ = _run("--target", p, "--fault", "replace",
                          "--from", "not-here", "--to", "x", "--check", WEAK_CHECK)
        check("NO-FAULT when --from absent", rc == NO_FAULT and "NO-FAULT" in out)
    finally:
        shutil.rmtree(d, ignore_errors=True)


def t_usage_when_target_missing():
    """A missing target file is a usage error (exit 2)."""
    rc, out, err = _run("--target", "/no/such/file", "--fault", "truncate", "--check", WEAK_CHECK)
    check("USAGE when target file missing", rc == USAGE and "USAGE" in (out + err))


def t_usage_when_required_arg_missing():
    """Missing --check is a usage error, not a crash."""
    d, p = _scratch_target()
    try:
        rc, out, err = _run("--target", p, "--fault", "truncate")
        check("USAGE when --check missing", rc == USAGE and "USAGE" in (out + err))
    finally:
        shutil.rmtree(d, ignore_errors=True)


def t_verdict_inversion_is_correct():
    """The inversion holds: check-fails => HELD, check-passes => ESCAPED, same fault."""
    d, p = _scratch_target()
    try:
        rc_strict, _, _ = _run("--target", p, "--fault", "truncate", "--check", STRICT_CHECK)
        rc_weak, _, _ = _run("--target", p, "--fault", "truncate", "--check", WEAK_CHECK)
        check("inversion: strict->HELD(0), weak->ESCAPED(3)",
              rc_strict == HELD and rc_weak == ESCAPED)
    finally:
        shutil.rmtree(d, ignore_errors=True)


def t_deterministic_verdict():
    """Same target+fault+check => same verdict every run."""
    d, p = _scratch_target()
    try:
        verdicts = set()
        for _ in range(3):
            rc, _, _ = _run("--target", p, "--fault", "bitflip", "--check", STRICT_CHECK)
            verdicts.add(rc)
        check("deterministic: identical verdict across 3 runs", verdicts == {HELD})
    finally:
        shutil.rmtree(d, ignore_errors=True)


def t_json_mode_emits_status():
    """--json emits a parseable object with the status and check_exit."""
    import json as _json
    d, p = _scratch_target()
    try:
        rc, out, _ = _run("--target", p, "--fault", "truncate", "--check", STRICT_CHECK, "--json")
        obj = _json.loads(out)
        check("--json emits status + check_exit",
              obj.get("status") == "held" and obj.get("check_exit") is not None)
    finally:
        shutil.rmtree(d, ignore_errors=True)


def t_edge_present_and_printed():
    """The printed edge is in the artifact and prints on --edge."""
    src = open(GIFT).read()
    rc, out, _ = _run("--edge")
    check("edge present in source AND on --edge",
          "never modifies your original" in src and "never modifies your original" in out)


def main():
    print("test_gauntlet.py")
    for fn in [
        t_held_when_strict_check_catches_truncation,
        t_escaped_when_weak_check_misses,
        t_original_never_modified,
        t_bitflip_held_by_strict_check,
        t_replace_regression_held,
        t_no_fault_when_replace_string_absent,
        t_usage_when_target_missing,
        t_usage_when_required_arg_missing,
        t_verdict_inversion_is_correct,
        t_deterministic_verdict,
        t_json_mode_emits_status,
        t_edge_present_and_printed,
    ]:
        fn()
    passed = sum(1 for _, ok in results if ok)
    total = len(results)
    print(f"\n{passed}/{total} passing")
    return 0 if passed == total else 1


if __name__ == "__main__":
    sys.exit(main())