From ad312ed48533d95581a2ff03efabbba786819ea3 Mon Sep 17 00:00:00 2001 From: pilotwaffle <75270828+pilotwaffle@users.noreply.github.com> Date: Fri, 25 Sep 2026 22:34:42 -0400 Subject: [PATCH] feat: print the next command after a dry-run demo After torq demo --run, write a short note to stderr so the stdout JSON stays byte-compatible. The note says no AI provider was contacted, no project files were changed, the run id, and the verify command. When evidence verify or fleet is pointed at a folder of runs, stdout is one JSON object with status run_folder, the run ids, and one verify command, and the same note goes to stderr. That case exits 2 and does not report evidence_missing or evidence_unstable. Single-run failure exit codes are unchanged. --- CHANGELOG.md | 7 + README.md | 5 +- docs/index.md | 5 +- docs/releases/torq-cli-v0.2.0-user-notes.md | 1 + src/torq_cli/application/demo.py | 21 ++ src/torq_cli/interfaces/cli.py | 77 ++++++- tests/test_demo_next_steps.py | 238 ++++++++++++++++++++ 7 files changed, 342 insertions(+), 12 deletions(-) create mode 100644 tests/test_demo_next_steps.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 7233834..ca25794 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,13 @@ this file is the canonical cumulative record. and attestation documents from the packaged registry and policy, optionally executes a real dry-run (`--run`), and leaves a verifiable run directory. No provider is contacted. +- After `torq demo --run`, a note on standard error names the dry run, the run + id, and the exact `torq evidence verify` command. The JSON on standard + output is unchanged. When `torq evidence verify` or `torq fleet` is pointed + at a folder of runs, standard output is one JSON object (`status` + `run_folder`, the run ids, and one verify command) and the same note is + repeated on standard error. That case does not report `evidence_missing` or + `evidence_unstable`. `torq --help` gives each command a one-line description. ### Changed - README rewritten around the visitor: pitch, comparison table, quickstart, diff --git a/README.md b/README.md index 4025af8..81d3db4 100644 --- a/README.md +++ b/README.md @@ -37,7 +37,7 @@ torq demo --goal "Add input validation to the login form" --run torq evidence verify --run-root ./torq-demo-runs/ ``` -`torq demo` scaffolds the identity and attestation documents, runs a real four-stage dry-run (G1D → G1R → Builder → G2A), and leaves a verifiable run directory behind. No provider is contacted and nothing outside the run root is written. The full install matrix is in [docs/install.md](docs/install.md). +`torq demo` scaffolds the identity and attestation documents, runs a real four-stage dry-run (G1D → G1R → Builder → G2A), and leaves a verifiable run directory behind. No provider is contacted and nothing outside the run root is written. After `--run`, a note on standard error gives the run id and the exact `torq evidence verify` command; the JSON lines on standard output stay the same. Pointing `torq evidence verify` or `torq fleet` at the folder that holds the runs prints one JSON object on standard output (`status` `run_folder`, the run ids, and one verify command) and the same note on standard error. The full install matrix is in [docs/install.md](docs/install.md). ## What it looks like @@ -50,6 +50,9 @@ $ torq demo --goal "Add input validation to the login form" --run {"status": "dry_run_complete", "report": {"mode": "dry_run", "verdict": "dry_run_complete", "planned_roles": ["g1d", "g1r", "builder", "g2a", "refine_bug", "refine_ui"], "attested": true, "receipts": "torq-demo-runs/run-0d04c989291241d58830a964ce578fc9", ...}} +No AI provider was contacted (dry run). No project files were changed. +Run id: run-0d04c989291241d58830a964ce578fc9 +torq evidence verify --run-root torq-demo-runs/run-0d04c989291241d58830a964ce578fc9 $ torq evidence verify --run-root torq-demo-runs/run-0d04c989291241d58830a964ce578fc9 {"finding": null, "status": "verified"} diff --git a/docs/index.md b/docs/index.md index 182e099..f68e7f9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -29,7 +29,7 @@ torq demo --goal "Add input validation to the login form" --run torq evidence verify --run-root ./torq-demo-runs/ ``` -`torq demo` scaffolds the identity and attestation documents, runs a real four-stage dry-run (G1D → G1R → Builder → G2A), and leaves a verifiable run directory behind. No provider is contacted and nothing outside the run root is written. The full install matrix is in [docs/install.md](docs/install.md). +`torq demo` scaffolds the identity and attestation documents, runs a real four-stage dry-run (G1D → G1R → Builder → G2A), and leaves a verifiable run directory behind. No provider is contacted and nothing outside the run root is written. After `--run`, a note on standard error gives the run id and the exact `torq evidence verify` command; the JSON lines on standard output stay the same. Pointing `torq evidence verify` or `torq fleet` at the folder that holds the runs prints one JSON object on standard output (`status` `run_folder`, the run ids, and one verify command) and the same note on standard error. The full install matrix is in [docs/install.md](docs/install.md). ## What it looks like @@ -42,6 +42,9 @@ $ torq demo --goal "Add input validation to the login form" --run {"status": "dry_run_complete", "report": {"mode": "dry_run", "verdict": "dry_run_complete", "planned_roles": ["g1d", "g1r", "builder", "g2a", "refine_bug", "refine_ui"], "attested": true, "receipts": "torq-demo-runs/run-0d04c989291241d58830a964ce578fc9", ...}} +No AI provider was contacted (dry run). No project files were changed. +Run id: run-0d04c989291241d58830a964ce578fc9 +torq evidence verify --run-root torq-demo-runs/run-0d04c989291241d58830a964ce578fc9 $ torq evidence verify --run-root torq-demo-runs/run-0d04c989291241d58830a964ce578fc9 {"finding": null, "status": "verified"} diff --git a/docs/releases/torq-cli-v0.2.0-user-notes.md b/docs/releases/torq-cli-v0.2.0-user-notes.md index 5b385cf..acf9cac 100644 --- a/docs/releases/torq-cli-v0.2.0-user-notes.md +++ b/docs/releases/torq-cli-v0.2.0-user-notes.md @@ -22,6 +22,7 @@ and backed by a verifiable receipt chain. - **One-command trial.** `torq demo --run` scaffolds a valid identity and attestation, executes a real dry-run, and leaves a run directory that `torq evidence verify` confirms — no provider calls, no hand-written JSON. + The dry run prints that verify command on standard error. - **Platform honesty.** Commands report exactly what is production-grade and what is not; `torq trust readiness` names the signing and anchoring gaps still open. diff --git a/src/torq_cli/application/demo.py b/src/torq_cli/application/demo.py index f8dbe8b..89c4651 100644 --- a/src/torq_cli/application/demo.py +++ b/src/torq_cli/application/demo.py @@ -10,6 +10,8 @@ from __future__ import annotations import json +import shlex +import sys from dataclasses import asdict from pathlib import Path from typing import Any @@ -69,6 +71,22 @@ def execute_demo(run_root: Path, goal: str) -> dict[str, Any]: ) +def _write_dry_run_note(report: dict[str, Any]) -> None: + """Tell a person what the dry run did, without touching stdout JSON.""" + run_id = report.get("run_id") + receipts = report.get("receipts") + if not isinstance(run_id, str) or not isinstance(receipts, str): + return + command = "torq evidence verify --run-root " + shlex.quote(receipts) + note = ( + "No AI provider was contacted (dry run). No project files were changed.\n" + f"Run id: {run_id}\n" + f"{command}\n" + ) + sys.stderr.write(note) + sys.stderr.flush() + + def demo_command(goal: str, run_root: Path, execute: bool) -> int: """CLI entry point. Prints JSON reports like the other commands.""" goal = goal or _DEFAULT_GOAL @@ -95,6 +113,9 @@ def demo_command(goal: str, run_root: Path, execute: bool) -> int: return 0 report = execute_demo(run_root, goal) print(json.dumps({"status": "dry_run_complete", "report": report}, sort_keys=True)) + # The note is on stderr so the two stdout JSON lines stay byte-compatible. + sys.stdout.flush() + _write_dry_run_note(report) return 0 except Exception as exc: print(json.dumps({"status": "blocked", "finding": str(exc)}, sort_keys=True)) diff --git a/src/torq_cli/interfaces/cli.py b/src/torq_cli/interfaces/cli.py index d6e7a68..b0ac32d 100644 --- a/src/torq_cli/interfaces/cli.py +++ b/src/torq_cli/interfaces/cli.py @@ -5,6 +5,7 @@ import argparse import getpass import json +import shlex import shutil import sys import tempfile @@ -101,22 +102,22 @@ def _parser() -> argparse.ArgumentParser: parser = argparse.ArgumentParser(prog="torq") parser.add_argument("--version", action="version", version=f"torq {__version__}") sub = parser.add_subparsers(dest="command", required=True) - profile = sub.add_parser("profile") + profile = sub.add_parser("profile", help="Validate a role profile") profile_sub = profile.add_subparsers(dest="profile_command", required=True) validate = profile_sub.add_parser("validate") validate.add_argument("--config", required=True) - status = sub.add_parser("status") + status = sub.add_parser("status", help="Attest machine and role status") status.add_argument("--offline", action="store_true") status.add_argument("--config", required=True) status.add_argument("--require-effective", action="store_true") status.add_argument("--runtime") - config = sub.add_parser("config") + config = sub.add_parser("config", help="Import Console V5 configuration") config_sub = config.add_subparsers(dest="config_command", required=True) import_v5 = config_sub.add_parser("import-v5-normalized") import_v5.add_argument("--config", required=True) import_v5_console_parser = config_sub.add_parser("import-v5-console") import_v5_console_parser.add_argument("--config", required=True) - auth = sub.add_parser("auth") + auth = sub.add_parser("auth", help="Store and check provider credentials") auth_sub = auth.add_subparsers(dest="auth_command", required=True) auth_status_parser = auth_sub.add_parser("status") auth_status_parser.add_argument("--credential-file") @@ -129,16 +130,16 @@ def _parser() -> argparse.ArgumentParser: default="platform_keychain", ) native.add_argument("--store-root") - harness = sub.add_parser("harness") + harness = sub.add_parser("harness", help="Inspect the live harness") harness_sub = harness.add_subparsers(dest="harness_command", required=True) inspect = harness_sub.add_parser("inspect") inspect.add_argument("--expected", required=True) inspect.add_argument("--actual", required=True) - setup = sub.add_parser("setup") + setup = sub.add_parser("setup", help="Write local config from answers") setup.add_argument("--config", required=True) setup.add_argument("--answers", required=True) setup.add_argument("--credential-file") - run = sub.add_parser("run") + run = sub.add_parser("run", help="Run a governed plan, dry-run by default") run.add_argument("--goal") run.add_argument("--resume") run.add_argument("--run-root", required=True) @@ -149,12 +150,12 @@ def _parser() -> argparse.ArgumentParser: run.add_argument("--live", action="store_true") run.add_argument("--allow-live", action="store_true") run.add_argument("--policy-allow-live", action="store_true") - evidence = sub.add_parser("evidence") + evidence = sub.add_parser("evidence", help="Verify a run's receipt chain") evidence_sub = evidence.add_subparsers(dest="evidence_command", required=True) verify = evidence_sub.add_parser("verify") verify.add_argument("--run-root", required=True) verify.add_argument("--trusted-public-key") - fleet = sub.add_parser("fleet") + fleet = sub.add_parser("fleet", help="Show the evidence-backed fleet view for one run") fleet.add_argument("--run-root") fleet.add_argument("--trusted-public-key") fleet.add_argument("--serve", action="store_true") @@ -180,7 +181,7 @@ def _parser() -> argparse.ArgumentParser: demo.add_argument("--goal", default="") demo.add_argument("--run-root", default="./torq-demo-runs") demo.add_argument("--run", action="store_true", help="execute the dry-run after scaffolding") - trust = sub.add_parser("trust") + trust = sub.add_parser("trust", help="Report production-trust gaps") trust_sub = trust.add_subparsers(dest="trust_command", required=True) trust_sub.add_parser("readiness") return parser @@ -332,6 +333,55 @@ def _handle_run(args: argparse.Namespace) -> int: return 5 +def _run_collection(root: Path) -> tuple[list[str], str] | None: + """Return run ids and one verify command, or None when this is not a folder of runs. + + A single run, including one whose evidence is missing or tampered, returns + None so the existing verifier and fleet exit codes stay in place. + """ + classified = classify_workspace_root(root) + if classified.get("root_kind") != "collection": + return None + runs = classified.get("runs") + if not isinstance(runs, list): + return None + run_ids = [run_id for run_id in runs if isinstance(run_id, str) and run_id] + if not run_ids: + return None + verify_command = "torq evidence verify --run-root " + shlex.quote(str(root / run_ids[0])) + return run_ids, verify_command + + +def _emit_run_collection_notice(root: Path) -> int | None: + found = _run_collection(root) + if found is None: + return None + run_ids, verify_command = found + listed = "\n".join(f" {run_id}" for run_id in run_ids) + notice = ( + "This is a folder of runs.\n" + "Run ids:\n" + f"{listed}\n" + "Verify one run:\n" + f"{verify_command}\n" + ) + # Scripts read one JSON object on stdout. The same facts go to stderr. + print( + json.dumps( + { + "status": "run_folder", + "run_ids": run_ids, + "verify_command": verify_command, + } + ), + flush=True, + ) + sys.stderr.write(notice) + sys.stderr.flush() + # Usage mistake: exit 2 (invalid). Single-run failures keep their own codes. + return 2 + + def main(argv: Sequence[str] | None = None) -> int: supplied = list(argv) if argv is not None else sys.argv[1:] import_boundary = ( @@ -595,6 +645,10 @@ def chat_snapshot_provider() -> Mapping[str, Any]: if task_runtime_root is not None: shutil.rmtree(task_runtime_root, ignore_errors=True) return 0 + if args.run_root is not None: + collection_code = _emit_run_collection_notice(selected_run_root) + if collection_code is not None: + return collection_code snapshot = projector.snapshot() print(json.dumps(snapshot, sort_keys=True)) return { @@ -606,6 +660,9 @@ def chat_snapshot_provider() -> Mapping[str, Any]: "unreadable": 4, }[str(snapshot["verification"]["state"])] if args.command == "evidence": + collection_code = _emit_run_collection_notice(Path(args.run_root)) + if collection_code is not None: + return collection_code result = verify_receipt_store(Path(args.run_root), trusted_public_key=trusted_public_key) print(json.dumps({"status": result.status, "finding": result.finding}, sort_keys=True)) return result.exit_code diff --git a/tests/test_demo_next_steps.py b/tests/test_demo_next_steps.py new file mode 100644 index 0000000..c775712 --- /dev/null +++ b/tests/test_demo_next_steps.py @@ -0,0 +1,238 @@ +from __future__ import annotations + +import json +import shlex +from pathlib import Path + +import pytest + +from torq_cli.interfaces.cli import main +from torq_cli.safety.receipts import FileRunKeyStore, ReceiptChain + + +def _json_lines(text: str) -> list[dict[str, object]]: + return [json.loads(line) for line in text.splitlines() if line] + + +def _verify_commands(text: str) -> list[str]: + return [ + line + for line in text.splitlines() + if line.startswith("torq evidence verify --run-root ") + ] + + +def _folder_stdout(run_ids: list[str], verify_command: str) -> str: + return json.dumps( + { + "status": "run_folder", + "run_ids": run_ids, + "verify_command": verify_command, + } + ) + "\n" + + +def _folder_stderr(run_ids: list[str], verify_command: str) -> str: + listed = "\n".join(f" {run_id}" for run_id in run_ids) + return ( + "This is a folder of runs.\n" + "Run ids:\n" + f"{listed}\n" + "Verify one run:\n" + f"{verify_command}\n" + ) + + +def test_demo_run_note_is_on_stderr_and_parent_folder_is_named( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + run_root = tmp_path / "demo runs" + code = main( + [ + "demo", + "--goal", + "Add input validation to the login form", + "--run", + "--run-root", + str(run_root), + ] + ) + assert code == 0 + captured = capsys.readouterr() + lines = _json_lines(captured.out) + assert [line["status"] for line in lines] == ["scaffolded", "dry_run_complete"] + report = lines[1]["report"] + assert isinstance(report, dict) + assert report["mode"] == "dry_run" + assert report["verdict"] == "dry_run_complete" + assert report["dispatched_roles"] == [] + assert report["proposal"] is None + run_id = report["run_id"] + receipts = report["receipts"] + assert isinstance(run_id, str) + assert isinstance(receipts, str) + assert "No AI provider was contacted (dry run)." in captured.err + assert "No project files were changed." in captured.err + assert f"Run id: {run_id}" in captured.err + assert "evidence_missing" not in captured.out + assert "evidence_missing" not in captured.err + commands = _verify_commands(captured.err) + assert commands == ["torq evidence verify --run-root " + shlex.quote(receipts)] + assert _verify_commands(captured.out) == [] + parsed = shlex.split(commands[0]) + assert parsed[4] == receipts + + verify_code = main(parsed[1:]) + verify_captured = capsys.readouterr() + assert verify_code == 0 + assert json.loads(verify_captured.out) == {"finding": None, "status": "verified"} + + parent_code = main(["evidence", "verify", "--run-root", str(run_root)]) + parent = capsys.readouterr() + verify_command = "torq evidence verify --run-root " + shlex.quote(receipts) + assert parent_code == 2 + assert parent.out == _folder_stdout([run_id], verify_command) + assert parent.err == _folder_stderr([run_id], verify_command) + assert json.loads(parent.out)["verify_command"] == verify_command + assert shlex.split(verify_command)[4] == receipts + assert "demo-inputs" not in parent.out + assert "demo-inputs" not in parent.err + assert "evidence_missing" not in parent.out + assert "evidence_unstable" not in parent.out + assert "evidence_missing" not in parent.err + assert "evidence_unstable" not in parent.err + + fleet_code = main(["fleet", "--run-root", str(run_root)]) + fleet = capsys.readouterr() + assert fleet_code == 2 + assert fleet.out == parent.out + assert fleet.err == parent.err + assert "torq-fleet-snapshot" not in fleet.out + + fleet_run_code = main(["fleet", "--run-root", receipts]) + fleet_run = capsys.readouterr() + assert fleet_run_code == 0 + snapshot = json.loads(fleet_run.out) + assert snapshot["verification"]["finding"] is None + assert snapshot["verification"]["state"] == "sealed_verified" + assert snapshot["run"]["run_id"] == run_id + + +def test_demo_without_run_does_not_print_the_dry_run_note( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + code = main(["demo", "--goal", "plan only", "--run-root", str(tmp_path / "runs")]) + assert code == 0 + captured = capsys.readouterr() + assert captured.err == "" + assert [json.loads(line)["status"] for line in captured.out.splitlines()] == [ + "scaffolded", + "hint", + ] + + +@pytest.mark.parametrize("command", (["evidence", "verify"], ["fleet"])) +def test_folder_of_runs_lists_every_run_id( + tmp_path: Path, capsys: pytest.CaptureFixture[str], command: list[str] +) -> None: + parent = tmp_path / "runs" + parent.mkdir() + for name in ("run-b", "run-a"): + chain = ReceiptChain( + parent, + name, + FileRunKeyStore(parent), + profile_version="1", + policy_version="3.1.3", + ) + chain.append("done", {}) + chain.seal() + + code = main([*command, "--run-root", str(parent)]) + captured = capsys.readouterr() + verify_command = "torq evidence verify --run-root " + shlex.quote(str(parent / "run-a")) + assert code == 2 + assert captured.out == _folder_stdout(["run-a", "run-b"], verify_command) + assert captured.err == _folder_stderr(["run-a", "run-b"], verify_command) + assert json.loads(captured.out) == { + "status": "run_folder", + "run_ids": ["run-a", "run-b"], + "verify_command": verify_command, + } + assert "evidence_missing" not in captured.out + assert "evidence_unstable" not in captured.out + assert "evidence_missing" not in captured.err + assert "evidence_unstable" not in captured.err + + single = main(["evidence", "verify", "--run-root", str(parent / "run-a")]) + single_out = capsys.readouterr() + assert single == 0 + assert json.loads(single_out.out) == {"finding": None, "status": "verified"} + + +def test_single_run_failures_keep_their_exit_codes( + tmp_path: Path, capsys: pytest.CaptureFixture[str] +) -> None: + missing = tmp_path / "missing-run" + missing.mkdir() + assert main(["evidence", "verify", "--run-root", str(missing)]) == 4 + assert json.loads(capsys.readouterr().out) == { + "finding": "evidence_missing", + "status": "incomplete", + } + + partial = tmp_path / "partial" + (partial / "run-partial").mkdir(parents=True) + assert main(["evidence", "verify", "--run-root", str(partial)]) == 4 + assert json.loads(capsys.readouterr().out)["finding"] == "evidence_missing" + + chain = ReceiptChain( + tmp_path, + "run-real", + FileRunKeyStore(tmp_path), + profile_version="1.0.0", + policy_version="3.1.3", + ) + chain.append("design", {"ok": True}) + chain.seal() + receipt_path = chain.root / "receipts.jsonl" + receipt_path.write_text( + receipt_path.read_text(encoding="utf-8").replace("design", "edited"), + encoding="utf-8", + ) + assert main(["evidence", "verify", "--run-root", str(chain.root)]) == 3 + tampered = json.loads(capsys.readouterr().out) + assert tampered == {"finding": "receipt_hash_mismatch", "status": "tampered"} + + assert main(["fleet", "--run-root", str(chain.root)]) == 3 + fleet = json.loads(capsys.readouterr().out) + assert fleet["verification"]["state"] == "tampered" + assert fleet["verification"]["finding"] == "receipt_hash_mismatch" + + empty = tmp_path / "empty-fleet" + empty.mkdir() + assert main(["fleet", "--run-root", str(empty)]) == 4 + fleet_empty = json.loads(capsys.readouterr().out) + assert fleet_empty["verification"]["finding"] == "evidence_unstable" + assert fleet_empty["verification"]["state"] == "incomplete" + + +def test_top_level_help_describes_every_command(capsys: pytest.CaptureFixture[str]) -> None: + with pytest.raises(SystemExit) as exc: + main(["--help"]) + assert exc.value.code == 0 + help_text = " ".join(capsys.readouterr().out.split()) + for phrase in ( + "Validate a role profile", + "Attest machine and role status", + "Import Console V5 configuration", + "Store and check provider credentials", + "Inspect the live harness", + "Write local config from answers", + "Run a governed plan, dry-run by default", + "Verify a run's receipt chain", + "Show the evidence-backed fleet view for one run", + "Scaffold a zero-config dry-run demo (no providers contacted)", + "Report production-trust gaps", + ): + assert phrase in help_text