From 8001939b61c9a2b6bc60da6b657b1b09ad337c46 Mon Sep 17 00:00:00 2001 From: Edbert Chan Date: Sun, 11 Oct 2026 14:53:12 +0000 Subject: [PATCH 1/4] Add an opt-in hook-payload recorder and a replay-equivalence harness Review claim: - An opt-in payload recorder in the runner plus a replay script can feed recorded hook-event payloads through installed hook commands and produce a verdict-equivalence report (stdout/context/block/exit) per detector. Review lane: behavior Safety invariant: - Recording is off unless CATSTACK_HOOK_PAYLOAD_DIR is set; when on it never alters hook stdin/stdout/exit and caps each payload (default 256KB) and total dir (default 2000 files, oldest evicted). - Replay runs against a scratch metrics/state dir, never live settings or the live runs.jsonl. Goal: - Runner: when env var set, append the raw stdin payload to $CATSTACK_HOOK_PAYLOAD_DIR/-.json before spawning the hook child, where is sha256(stdin)[:12]. - scripts/test/replay_hook_payloads.py: for each recorded payload, run the hook command under a scratch env, capture stdout/stderr/exit/ findings; emit an equivalence report JSON + table comparing two command sets (fleet vs dispatcher args taken as parameters). Motivation: - The catastrophic regression risk of the dispatcher is a detector that silently stops speaking; the only credible gate is byte-identical verdicts on real payloads. Cherry-picked from origin/plan/hook-fleet-dispatcher-measure-the-herd-then-dispatch-it (e356e8c8), scoped to only this claim's files. Dropped this slice's dependency on the separate metrics-event-uid claim (17ef1517, not on this branch): the payload recorder computes its own local hash instead of reusing a published `event_uid` metrics-row field, and the README wording and test imports were adjusted to match. Co-Authored-By: Claude Sonnet 5 --- engine/hooks/_runner/README.md | 35 +++ engine/hooks/_runner/run.py | 43 ++++ engine/hooks/_runner/tests/test_run.py | 56 ++++- scripts/test/replay_hook_payloads.py | 246 ++++++++++++++++++++++ scripts/test/test_replay_hook_payloads.py | 100 +++++++++ 5 files changed, 477 insertions(+), 3 deletions(-) create mode 100644 scripts/test/replay_hook_payloads.py create mode 100644 scripts/test/test_replay_hook_payloads.py diff --git a/engine/hooks/_runner/README.md b/engine/hooks/_runner/README.md index 1b5768d45..f410ae929 100644 --- a/engine/hooks/_runner/README.md +++ b/engine/hooks/_runner/README.md @@ -125,6 +125,18 @@ hook bypasses the metrics runner: Rows are written to `~/.cache/catstack-hook-metrics/runs.jsonl` by default. Set `CATSTACK_HOOK_METRICS_DIR` to write `runs.jsonl` under a different directory. +Set `CATSTACK_HOOK_PAYLOAD_DIR` to record the raw hook-event payload bytes +before the hook subprocess starts. Recording is off by default. When enabled, +the runner writes one file named `-.json` under that +directory, where `` is the first 12 hex characters of the +stdin bytes' SHA-256 digest, leaves the hook's stdin/stdout and exit code +unchanged, skips payloads larger than 256KB, and keeps at most 2000 recorded +payload files by evicting the oldest files. Override those caps with +`CATSTACK_HOOK_PAYLOAD_MAX_BYTES` and `CATSTACK_HOOK_PAYLOAD_KEEP`. + +Two identical consecutive events for the same hook produce the same filename, +so the newer payload replaces the older identical payload. + Each row contains: - `ts`: UTC timestamp for the recorded run. @@ -236,3 +248,26 @@ unsupported --since value: With `--json`, the same report is printed as JSON with `registered`, `unregistered`, `malformed_rows`, `config_warnings`, and `window_rows`. + +## Replay Recorded Payloads + +Usage: + +```sh +python3 scripts/test/replay_hook_payloads.py \ + --payload-dir /tmp/hook-payloads \ + --fleet detector='python3 ~/.claude/hooks/example/check.py' \ + --dispatcher detector='python3 ~/.claude/hooks/_runner/dispatch.py --event Stop --timeout 10' \ + --report-json /tmp/hook-replay-report.json +``` + +Each `--fleet` and `--dispatcher` value is `label=command`; labels must match on +both sides. The command runs once per recorded payload with that payload on +stdin. Replay sets `CATSTACK_HOOK_METRICS_DIR` and known hook state env vars to a +temporary scratch tree and unsets `CATSTACK_HOOK_PAYLOAD_DIR`, so it does not +write the live `runs.jsonl` or recursively record replay payloads. + +The script prints a table with one `match` or `mismatch` row per +payload/label pair and writes the JSON report when `--report-json` is provided. +The equivalence check compares stdout bytes, extracted context, block verdict, +exit code, and reported findings. It exits 0 only when every row matches. diff --git a/engine/hooks/_runner/run.py b/engine/hooks/_runner/run.py index 081747e3c..414f75d80 100644 --- a/engine/hooks/_runner/run.py +++ b/engine/hooks/_runner/run.py @@ -2,6 +2,7 @@ import argparse import datetime +import hashlib import json import os import subprocess @@ -19,8 +20,13 @@ WELL_KNOWN_PYTHON_DIRS = ("/opt/homebrew/bin", "/usr/local/bin", os.path.expanduser("~/.local/bin")) METRICS_MAX_BYTES_ENV = "CATSTACK_HOOK_METRICS_MAX_BYTES" METRICS_KEEP_ENV = "CATSTACK_HOOK_METRICS_KEEP" +PAYLOAD_DIR_ENV = "CATSTACK_HOOK_PAYLOAD_DIR" +PAYLOAD_MAX_BYTES_ENV = "CATSTACK_HOOK_PAYLOAD_MAX_BYTES" +PAYLOAD_KEEP_ENV = "CATSTACK_HOOK_PAYLOAD_KEEP" DEFAULT_METRICS_MAX_BYTES = 20 * 1024 * 1024 DEFAULT_METRICS_KEEP = 3 +DEFAULT_PAYLOAD_MAX_BYTES = 256 * 1024 +DEFAULT_PAYLOAD_KEEP = 2000 REPLY_EVENTS = ("Stop", "SubagentStop") FENCE = "```" SKIP_MACHINE_DELIVERABLE = "machine-deliverable" @@ -152,6 +158,40 @@ def _write_metrics(row: dict[str, object], path: str) -> bytes: return message +def _payload_path(root: str, event_uid: str, hook: str) -> str: + safe_hook = "".join(char if char.isalnum() or char in "._-" else "_" for char in hook) or "unknown" + return os.path.join(root, f"{event_uid}-{safe_hook}.json") + + +def _evict_payloads(root: str, keep: int) -> None: + entries = [ + entry + for entry in os.scandir(root) + if entry.is_file() and entry.name.endswith(".json") + ] + for entry in sorted(entries, key=lambda item: (item.stat().st_mtime_ns, item.name))[: max(0, len(entries) - keep)]: + os.unlink(entry.path) + + +def _record_payload(stdin: bytes, hook: str) -> bytes: + root = os.environ.get(PAYLOAD_DIR_ENV) + if not root: + return b"" + max_bytes, _max_message = _positive_env(PAYLOAD_MAX_BYTES_ENV, DEFAULT_PAYLOAD_MAX_BYTES) + keep, _keep_message = _positive_env(PAYLOAD_KEEP_ENV, DEFAULT_PAYLOAD_KEEP) + if len(stdin) > max_bytes: + return b"" + try: + os.makedirs(root, exist_ok=True) + event_uid = hashlib.sha256(stdin).hexdigest()[:12] + with open(_payload_path(root, event_uid, hook), "wb") as handle: + handle.write(stdin) + _evict_payloads(root, keep) + except OSError as exc: + return f"catstack-hook-payloads: could not record payload: {exc}\n".encode() + return b"" + + def _make_findings_file() -> str: fd, path = tempfile.mkstemp(prefix="catstack-hook-findings-", suffix=".json") with os.fdopen(fd, "w", encoding="utf-8") as handle: @@ -258,6 +298,7 @@ def main(argv: list[str] | None = None) -> int: timed_out = False findings_path = _make_findings_file() findings_error = b"" + payload_error = b"" rule_ids: list[str] = [] hook_ran = False skipped = None if args.notify else _skip_reason(stdin) @@ -276,6 +317,7 @@ def main(argv: list[str] | None = None) -> int: f"Set {PYTHON_OVERRIDE_ENV} to a newer interpreter.\n" ).encode() else: + payload_error = _record_payload(meta, hook) env = os.environ.copy() env["CATSTACK_HOOK_FINDINGS_FILE"] = findings_path proc = subprocess.Popen( @@ -319,6 +361,7 @@ def main(argv: list[str] | None = None) -> int: sys.stdout.buffer.flush() sys.stderr.buffer.write(stderr) sys.stderr.buffer.write(findings_error) + sys.stderr.buffer.write(payload_error) sys.stderr.buffer.write(metrics_error) sys.stderr.buffer.flush() return exit_code diff --git a/engine/hooks/_runner/tests/test_run.py b/engine/hooks/_runner/tests/test_run.py index 563b95519..07fc1e565 100644 --- a/engine/hooks/_runner/tests/test_run.py +++ b/engine/hooks/_runner/tests/test_run.py @@ -1,5 +1,6 @@ from __future__ import annotations +import hashlib import json import os import shutil @@ -7,8 +8,11 @@ import sys import tempfile import unittest +from unittest import mock RUNNER_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) +sys.path.insert(0, RUNNER_DIR) +import run # noqa: E402 class RunnerCLI(unittest.TestCase): @@ -42,9 +46,13 @@ def _write_fixture(self, name: str, body: str) -> None: with open(os.path.join(self.fixture_dir, name), "w", encoding="utf-8") as handle: handle.write(body) - def _env(self, metrics_dir: str | None = None) -> dict[str, str]: + def _env(self, metrics_dir: str | None = None, payload_dir: str | None = None) -> dict[str, str]: env = os.environ.copy() env["CATSTACK_HOOK_METRICS_DIR"] = self.metrics_dir if metrics_dir is None else metrics_dir + if payload_dir is not None: + env["CATSTACK_HOOK_PAYLOAD_DIR"] = payload_dir + else: + env.pop("CATSTACK_HOOK_PAYLOAD_DIR", None) return env def _stdin(self) -> bytes: @@ -59,13 +67,19 @@ def _direct(self, script: str) -> subprocess.CompletedProcess[bytes]: env=self._env(), ) - def _runner(self, script: str, *args: str, metrics_dir: str | None = None) -> subprocess.CompletedProcess[bytes]: + def _runner( + self, + script: str, + *args: str, + metrics_dir: str | None = None, + payload_dir: str | None = None, + ) -> subprocess.CompletedProcess[bytes]: return subprocess.run( [sys.executable, os.path.join(self.runner_dir, "run.py"), *args, f"fixture/{script}"], input=self._stdin(), stdout=subprocess.PIPE, stderr=subprocess.PIPE, - env=self._env(metrics_dir), + env=self._env(metrics_dir, payload_dir), ) def _row(self) -> dict[str, object]: @@ -100,6 +114,42 @@ def _assert_run_matches_direct(self, script: str, outcome: str) -> None: self.assertEqual(row["rule_ids"], []) self.assertEqual(row["stdout_bytes"], len(direct.stdout)) + def test_payload_recorder_is_inert_without_env(self): + with mock.patch("run.os.makedirs") as makedirs, mock.patch("run.open", mock.mock_open()) as opened: + self.assertEqual(run._record_payload(b'{"x":1}', "fixture"), b"") + makedirs.assert_not_called() + opened.assert_not_called() + + def test_payload_recorder_writes_stdin_when_enabled(self): + payload_dir = os.path.join(self.tmp.name, "payloads") + stdin = self._stdin() + + wrapped = self._runner("silent.py", payload_dir=payload_dir) + + self.assertEqual((wrapped.returncode, wrapped.stdout, wrapped.stderr), (0, b"", b"")) + event_uid = hashlib.sha256(stdin).hexdigest()[:12] + with open(os.path.join(payload_dir, f"{event_uid}-fixture.json"), "rb") as handle: + self.assertEqual(handle.read(), stdin) + + def test_payload_recorder_evicts_old_payloads_past_keep_cap(self): + payload_dir = os.path.join(self.tmp.name, "payloads") + with mock.patch.dict( + os.environ, + { + "CATSTACK_HOOK_PAYLOAD_DIR": payload_dir, + "CATSTACK_HOOK_PAYLOAD_KEEP": "1", + }, + clear=False, + ): + self.assertEqual(run._record_payload(b"first", "fixture"), b"") + first_name = os.listdir(payload_dir)[0] + os.utime(os.path.join(payload_dir, first_name), (1, 1)) + self.assertEqual(run._record_payload(b"second", "fixture"), b"") + + names = os.listdir(payload_dir) + self.assertEqual(len(names), 1) + self.assertEqual(names[0], f"{hashlib.sha256(b'second').hexdigest()[:12]}-fixture.json") + def test_notify_mode_reads_the_payload_argument_not_stdin(self): argv_out = os.path.join(self.tmp.name, "argv.json") self._write_fixture("notify.py", f"import json, sys\njson.dump(sys.argv[1:], open({argv_out!r}, 'w'))\n") diff --git a/scripts/test/replay_hook_payloads.py b/scripts/test/replay_hook_payloads.py new file mode 100644 index 000000000..e52686ba8 --- /dev/null +++ b/scripts/test/replay_hook_payloads.py @@ -0,0 +1,246 @@ +#!/usr/bin/env python3 +from __future__ import annotations + +import argparse +import base64 +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import tempfile +from dataclasses import dataclass + +RUNNER_DIR = Path(__file__).resolve().parents[2] / "engine" / "hooks" / "_runner" +sys.path.insert(0, str(RUNNER_DIR)) + +import outcome # noqa: E402 + +STATE_ENVS = ( + "CATSTACK_HOOK_REMINDER_STATE_DIR", + "CATSTACK_NARROW_THE_SCOPE_STATE_DIR", + "CATSTACK_LLM_JUDGE_STATE_DIR", + "CATSTACK_UNVERIFIED_TAG_CHECK_STATE_DIR", + "CATSTACK_BUILD_THE_LEVER_STATE_DIR", + "CATSTACK_TAG_LEDGER_DIR", + "CATSTACK_SPLIT_SCOPE_STATE_DIR", +) + + +@dataclass(frozen=True) +class CommandSpec: + label: str + command: str + + +def _parse_command_spec(raw: str) -> CommandSpec: + label, sep, command = raw.partition("=") + if not sep or not label or not command: + raise argparse.ArgumentTypeError("expected LABEL=COMMAND") + return CommandSpec(label, command) + + +def _read_json_object(raw: bytes) -> dict[str, object] | None: + try: + payload = json.loads(raw.decode("utf-8")) + except (UnicodeDecodeError, json.JSONDecodeError): + return None + return payload if isinstance(payload, dict) else None + + +def _context(stdout: bytes) -> str: + payload = _read_json_object(stdout) + if payload is None: + return stdout.decode("utf-8", errors="replace").strip() + for key in ("additionalContext", "additional_context", "message", "reason", "user_message"): + value = payload.get(key) + if isinstance(value, str) and value: + return value + hook_output = payload.get("hookSpecificOutput") + if isinstance(hook_output, dict): + for key in ("additionalContext", "permissionDecisionReason", "reason"): + value = hook_output.get(key) + if isinstance(value, str) and value: + return value + return "" + + +def _rule_ids(metrics_dir: Path, findings_path: Path) -> list[str]: + seen: set[str] = set() + ordered: list[str] = [] + + def add(values: object) -> None: + if isinstance(values, list): + for value in values: + if isinstance(value, str) and value not in seen: + seen.add(value) + ordered.append(value) + + try: + add(json.loads(findings_path.read_text(encoding="utf-8"))) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + pass + runs = metrics_dir / "runs.jsonl" + try: + with runs.open(encoding="utf-8") as handle: + for line in handle: + row = json.loads(line) + if isinstance(row, dict): + add(row.get("rule_ids")) + except (OSError, UnicodeDecodeError, json.JSONDecodeError): + pass + return ordered + + +def _scratch_env(base: Path, side: str, label: str, payload_name: str) -> tuple[dict[str, str], Path, Path]: + root = base / side / label / payload_name + metrics_dir = root / "metrics" + state_dir = root / "state" + metrics_dir.mkdir(parents=True, exist_ok=True) + state_dir.mkdir(parents=True, exist_ok=True) + findings_path = root / "findings.json" + findings_path.write_text("[]", encoding="utf-8") + env = os.environ.copy() + env["CATSTACK_HOOK_METRICS_DIR"] = str(metrics_dir) + env["CATSTACK_HOOK_FINDINGS_FILE"] = str(findings_path) + env.pop("CATSTACK_HOOK_PAYLOAD_DIR", None) + for name in STATE_ENVS: + env[name] = str(state_dir / name.lower()) + return env, metrics_dir, findings_path + + +def _run_command(spec: CommandSpec, payload: bytes, payload_name: str, side: str, scratch: Path, timeout: float) -> dict: + env, metrics_dir, findings_path = _scratch_env(scratch, side, spec.label, payload_name) + timed_out = False + try: + proc = subprocess.run( + spec.command, + shell=True, + input=payload, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env, + timeout=timeout, + ) + stdout = proc.stdout + stderr = proc.stderr + exit_code = proc.returncode + except subprocess.TimeoutExpired as exc: + timed_out = True + stdout = exc.stdout or b"" + stderr = (exc.stderr or b"") + f"replay_hook_payloads: timed out after {timeout}s\n".encode() + exit_code = 1 + result = outcome.classify(exit_code, stdout, stderr, timed_out) + return { + "command": spec.command, + "exit_code": exit_code, + "outcome": result, + "blocked": result == "blocked", + "context": _context(stdout), + "findings": _rule_ids(metrics_dir, findings_path), + "stdout_b64": base64.b64encode(stdout).decode("ascii"), + "stdout_sha256": hashlib.sha256(stdout).hexdigest(), + "stdout_text": stdout.decode("utf-8", errors="replace"), + "stderr_b64": base64.b64encode(stderr).decode("ascii"), + "stderr_text": stderr.decode("utf-8", errors="replace"), + } + + +def _by_label(specs: list[CommandSpec], name: str) -> dict[str, CommandSpec]: + result: dict[str, CommandSpec] = {} + for spec in specs: + if spec.label in result: + raise SystemExit(f"duplicate {name} label: {spec.label}") + result[spec.label] = spec + return result + + +def _comparison_keys(verdict: dict) -> dict: + return { + "exit_code": verdict["exit_code"], + "blocked": verdict["blocked"], + "context": verdict["context"], + "findings": verdict["findings"], + "stdout_b64": verdict["stdout_b64"], + } + + +def _payload_files(payload_dir: Path) -> list[Path]: + return sorted(path for path in payload_dir.glob("*.json") if path.is_file()) + + +def _print_table(rows: list[dict]) -> None: + print("payload detector verdict fleet_exit dispatcher_exit fleet_outcome dispatcher_outcome") + for row in rows: + verdict = "match" if row["match"] else "mismatch" + print( + f"{row['payload']} {row['detector']} {verdict} " + f"{row['fleet']['exit_code']} {row['dispatcher']['exit_code']} " + f"{row['fleet']['outcome']} {row['dispatcher']['outcome']}" + ) + + +def _parse_args(argv: list[str]) -> argparse.Namespace: + parser = argparse.ArgumentParser() + parser.add_argument("--payload-dir", required=True) + parser.add_argument("--fleet", action="append", type=_parse_command_spec, required=True) + parser.add_argument("--dispatcher", action="append", type=_parse_command_spec, required=True) + parser.add_argument("--timeout", type=float, default=30.0) + parser.add_argument("--report-json") + parser.add_argument("--keep-scratch", action="store_true") + return parser.parse_args(argv) + + +def main(argv: list[str] | None = None) -> int: + args = _parse_args(sys.argv[1:] if argv is None else argv) + payload_dir = Path(args.payload_dir) + payloads = _payload_files(payload_dir) + if not payloads: + raise SystemExit(f"no recorded payloads in {payload_dir}") + fleet = _by_label(args.fleet, "fleet") + dispatcher = _by_label(args.dispatcher, "dispatcher") + if set(fleet) != set(dispatcher): + missing_dispatcher = sorted(set(fleet) - set(dispatcher)) + missing_fleet = sorted(set(dispatcher) - set(fleet)) + raise SystemExit( + f"fleet/dispatcher labels differ: missing dispatcher={missing_dispatcher} missing fleet={missing_fleet}" + ) + + scratch = Path(tempfile.mkdtemp(prefix="catstack-hook-replay-")) + rows: list[dict] = [] + try: + for payload_path in payloads: + payload = payload_path.read_bytes() + for label in sorted(fleet): + fleet_verdict = _run_command(fleet[label], payload, payload_path.name, "fleet", scratch, args.timeout) + dispatcher_verdict = _run_command( + dispatcher[label], payload, payload_path.name, "dispatcher", scratch, args.timeout + ) + rows.append( + { + "payload": payload_path.name, + "detector": label, + "match": _comparison_keys(fleet_verdict) == _comparison_keys(dispatcher_verdict), + "fleet": fleet_verdict, + "dispatcher": dispatcher_verdict, + } + ) + finally: + if args.keep_scratch: + print(f"scratch={scratch}", file=sys.stderr) + else: + shutil.rmtree(scratch) + + report = {"payload_dir": str(payload_dir), "match": all(row["match"] for row in rows), "rows": rows} + if args.report_json: + with open(args.report_json, "w", encoding="utf-8") as handle: + json.dump(report, handle, indent=2, sort_keys=True) + handle.write("\n") + _print_table(rows) + return 0 if report["match"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/test/test_replay_hook_payloads.py b/scripts/test/test_replay_hook_payloads.py new file mode 100644 index 000000000..263fce353 --- /dev/null +++ b/scripts/test/test_replay_hook_payloads.py @@ -0,0 +1,100 @@ +from __future__ import annotations + +import hashlib +import json +import os +from pathlib import Path +import shlex +import subprocess +import sys +import tempfile +import unittest + +SCRIPT = Path(__file__).resolve().parent / "replay_hook_payloads.py" + + +class ReplayHookPayloadsCLI(unittest.TestCase): + def setUp(self) -> None: + self.tmp = tempfile.TemporaryDirectory() + self.addCleanup(self.tmp.cleanup) + self.root = Path(self.tmp.name) + self.payload_dir = self.root / "payloads" + self.payload_dir.mkdir() + self.hook = self.root / "hook.py" + self.hook.write_text( + "import json\n" + "import os\n" + "import sys\n" + "payload = sys.stdin.buffer.read().decode('utf-8')\n" + "mode = sys.argv[1]\n" + "with open(os.environ['CATSTACK_HOOK_FINDINGS_FILE'], 'w', encoding='utf-8') as handle:\n" + " json.dump(['fixture.rule'], handle)\n" + "if mode == 'block':\n" + " print(json.dumps({'decision': 'block', 'reason': payload}))\n" + "elif mode == 'exit2':\n" + " sys.stderr.write(payload + '\\n')\n" + " sys.exit(2)\n" + "else:\n" + " print(json.dumps({'hookSpecificOutput': {'additionalContext': mode + ':' + payload}}))\n", + encoding="utf-8", + ) + + def _payload(self, body: bytes = b'{"hello":"world"}') -> str: + name = f"{hashlib.sha256(body).hexdigest()[:12]}-fixture.json" + (self.payload_dir / name).write_bytes(body) + return name + + def _command(self, mode: str) -> str: + return f"{shlex.quote(sys.executable)} {shlex.quote(str(self.hook))} {shlex.quote(mode)}" + + def _run(self, fleet_mode: str, dispatcher_mode: str) -> subprocess.CompletedProcess[str]: + report = self.root / "report.json" + return subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--payload-dir", + str(self.payload_dir), + "--fleet", + f"fixture={self._command(fleet_mode)}", + "--dispatcher", + f"fixture={self._command(dispatcher_mode)}", + "--report-json", + str(report), + ], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + ) + + def test_matching_commands_emit_match_table_and_json_report(self) -> None: + payload_name = self._payload() + + result = self._run("same", "same") + + self.assertEqual(result.returncode, 0, result.stderr) + self.assertIn(f"{payload_name} fixture match 0 0 spoke spoke", result.stdout) + with open(self.root / "report.json", encoding="utf-8") as handle: + report = json.load(handle) + self.assertTrue(report["match"]) + self.assertEqual(report["rows"][0]["detector"], "fixture") + self.assertEqual(report["rows"][0]["fleet"]["findings"], ["fixture.rule"]) + + def test_mismatching_context_returns_nonzero_and_marks_the_row(self) -> None: + self._payload() + + result = self._run("fleet", "dispatcher") + + self.assertEqual(result.returncode, 1, result.stdout + result.stderr) + self.assertIn("fixture mismatch 0 0 spoke spoke", result.stdout) + with open(self.root / "report.json", encoding="utf-8") as handle: + report = json.load(handle) + self.assertFalse(report["match"]) + self.assertNotEqual( + report["rows"][0]["fleet"]["context"], + report["rows"][0]["dispatcher"]["context"], + ) + + +if __name__ == "__main__": + unittest.main() From aa14b4f0ef863052e94dc2e3e65774cd0a31bc41 Mon Sep 17 00:00:00 2001 From: Invoker Bot Date: Sun, 11 Oct 2026 14:57:30 +0000 Subject: [PATCH 2/4] =?UTF-8?q?invoker:=20wf-1791728212248-12/publish-repl?= =?UTF-8?q?ay=20=E2=80=94=20Review=20claim:=20An=20opt-in=20payload=20reco?= =?UTF-8?q?rder=20in=20the=20runner=20plus=20a=20replay=20script=20can=20f?= =?UTF-8?q?eed=20recorded=20hook-event=20payloads=20through=20installed=20?= =?UTF-8?q?hook=20commands=20and=20produce=20a=20verdict-equivalence=20rep?= =?UTF-8?q?ort=20per=20detector.=20Review=20lane:=20behavior=20Safety=20in?= =?UTF-8?q?variant:=20Hook=20stdout=20and=20exit=20codes=20stay=20the=20sa?= =?UTF-8?q?me.=20This=20slice=20does=20not=20change=20the=20other=20four?= =?UTF-8?q?=20claims.=20Files:=20engine/hooks/=5Frunner/run.py,=20scripts/?= =?UTF-8?q?test/replay=5Fhook=5Fpayloads.py=20Change=20types:=20bugfix=20A?= =?UTF-8?q?cceptance=20criteria:=20The=20branch=20contains=20only=20this?= =?UTF-8?q?=20claim,=20and=20the=20review=20step=20publishes=20one=20pull?= =?UTF-8?q?=20request.?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0 Invoker-Finalize-Id: 61e213a0-a6cd-4950-ba23-0acc17742997 From d63b1fc922d925dc6f89685398b42e2b02139371 Mon Sep 17 00:00:00 2001 From: CI Bot Date: Sun, 11 Oct 2026 19:22:22 +0000 Subject: [PATCH 3/4] Fix replay hook payload lint --- scripts/test/test_replay_hook_payloads.py | 1 - 1 file changed, 1 deletion(-) diff --git a/scripts/test/test_replay_hook_payloads.py b/scripts/test/test_replay_hook_payloads.py index 263fce353..a8925a5ec 100644 --- a/scripts/test/test_replay_hook_payloads.py +++ b/scripts/test/test_replay_hook_payloads.py @@ -2,7 +2,6 @@ import hashlib import json -import os from pathlib import Path import shlex import subprocess From 9787af8f07826c77ba0072030a241cf253e6a08c Mon Sep 17 00:00:00 2001 From: CI Bot Date: Sun, 11 Oct 2026 19:22:27 +0000 Subject: [PATCH 4/4] =?UTF-8?q?invoker:=20wf-1791746491297-4/repair=20?= =?UTF-8?q?=E2=80=94=20Repair=20PR=20#1333=20(failed=20check=20lint)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Exit code: 0