diff --git a/.claude/templates/testing_prompt.template.md b/.claude/templates/testing_prompt.template.md index ee6a08fa2..bbcdc47fb 100644 --- a/.claude/templates/testing_prompt.template.md +++ b/.claude/templates/testing_prompt.template.md @@ -11,7 +11,10 @@ You are assigned to test the following features: {{TESTING_FEATURE_IDS}} 2. Read the feature's verification steps 3. Test the feature in the browser 4. Call `feature_mark_passing` or `feature_mark_failing` -5. Move to the next feature +5. Make sure the feature has an automated regression script (STEP 4) +6. Move to the next feature + +**Why scripts matter:** Before your session starts, the orchestrator runs each assigned feature's script from `tests/regression/manifest.json`. Features whose script passes are never sent to an agent again, which saves a full LLM session per feature. You are only seeing features that have **no script yet** or whose **script failed**. --- @@ -62,7 +65,7 @@ For the feature returned: #### If the feature PASSES: -The feature still works correctly. **DO NOT** call feature_mark_passing again -- it's already passing. End your session. +The feature still works correctly. **DO NOT** call feature_mark_passing again -- it's already passing. Continue to STEP 4. #### If the feature FAILS (regression found): @@ -102,6 +105,34 @@ A regression has been introduced. You MUST fix it: - Verified with browser automation" ``` +### STEP 4: AUTOMATE THE CHECK (REGRESSION SCRIPT) + +Once a feature is verified (passing, or fixed and passing again), make sure future regression runs can be done **without an agent** by giving it a deterministic test script. + +1. Write a script under `tests/regression/` that reproduces the feature's verification steps, e.g. `tests/regression/feature_.spec.ts` (Playwright Test), or a Node script / npm script for API-only features. Prefer the test tooling the project already has. If you need `@playwright/test`, install it as a dev dependency. +2. The script must: + - Exit with code **0** on success and **non-zero** on any failure + - Be non-interactive and headless + - Assume the app's dev server is already running if it needs it (or start/stop it itself) + - Create its own uniquely-named test data and clean it up + - Finish in well under 5 minutes +3. Register it in `tests/regression/manifest.json` (create the file if missing), keyed by feature ID: + ```json + { + "12": {"command": "npx playwright test tests/regression/feature_12.spec.ts"}, + "15": {"command": "node tests/regression/feature_15.mjs", "timeout": 120} + } + ``` + `timeout` is optional (seconds, default 300, max 900). The command runs from the project root and must only use allowed bash commands. +4. **Run the exact command from the manifest** and confirm it passes. If the feature arrived here because its script failed, fix whichever is wrong -- the app (regression) or the script (broken/flaky test) -- and re-run until it passes. +5. Commit the script and manifest: + ```bash + git add tests/regression + git commit -m "Add regression script for feature #" + ``` + +Keep scripts focused on the feature's verification steps; don't duplicate the whole suite in one script. + --- ## AVAILABLE TOOLS @@ -128,7 +159,7 @@ Use `playwright-cli` commands for browser interaction. Key commands: ## IMPORTANT REMINDERS -**Your Goal:** Test each assigned feature thoroughly. Verify it still works, and fix any regression found. Process ALL features in your list before ending your session. +**Your Goal:** Test each assigned feature thoroughly. Verify it still works, fix any regression found, and leave a passing regression script registered in `tests/regression/manifest.json` for it. Process ALL features in your list before ending your session. **Quality Bar:** - Zero console errors diff --git a/agent.py b/agent.py index 668391f7b..5ad4acec0 100644 --- a/agent.py +++ b/agent.py @@ -45,6 +45,7 @@ is_rate_limit_error, parse_retry_after, ) +from regression_runner import format_failures_for_prompt, run_scripted_regression # Configuration AUTO_CONTINUE_DELAY_SECONDS = 3 @@ -218,6 +219,7 @@ async def run_autonomous_agent( agent_type = "coding" is_initializer = agent_type == "initializer" + scripted_failures_context = "" if is_initializer: print("Running as INITIALIZER agent") @@ -233,6 +235,21 @@ async def run_autonomous_agent( elif agent_type == "testing": print("Running as TESTING agent (regression testing)") print_progress_summary(project_dir) + + # Run automated regression scripts first; only features without a + # script (or whose script failed) need an LLM session. + assigned_ids = testing_feature_ids or ([testing_feature_id] if testing_feature_id is not None else []) + if assigned_ids: + report = run_scripted_regression(project_dir, assigned_ids) + if report.passed: + print(f"Scripted regression passed for: {', '.join(f'#{fid}' for fid in report.passed)}") + remaining_ids = report.needs_agent + if not remaining_ids: + print("All assigned features passed their regression scripts - no agent session needed.") + return + testing_feature_id = None + testing_feature_ids = remaining_ids + scripted_failures_context = format_failures_for_prompt(report.failed) else: print("Running as CODING agent") print_progress_summary(project_dir) @@ -278,6 +295,8 @@ async def run_autonomous_agent( prompt = get_initializer_prompt(project_dir) elif agent_type == "testing": prompt = get_testing_prompt(project_dir, testing_feature_id, testing_feature_ids) + if scripted_failures_context: + prompt = f"{scripted_failures_context}\n\n---\n\n{prompt}" elif feature_ids and len(feature_ids) > 1: # Batch mode (used by orchestrator for multi-feature coding agents) prompt = get_batch_feature_prompt(feature_ids, project_dir, yolo_mode) diff --git a/regression_runner.py b/regression_runner.py new file mode 100644 index 000000000..875811c69 --- /dev/null +++ b/regression_runner.py @@ -0,0 +1,208 @@ +""" +Scripted Regression Tests +========================= + +Runs per-feature automated regression test scripts so that testing agents +don't need an LLM session to re-verify features that already have a script. + +Testing agents author these scripts the first time they verify a feature and +register them in a manifest committed with the generated app: + + tests/regression/manifest.json + { + "12": {"command": "npx playwright test tests/regression/feature_12.spec.ts"}, + "15": {"command": "node tests/regression/feature_15.mjs", "timeout": 120} + } + +Before starting an LLM session, the testing agent runs the scripts for its +assigned features. Features whose scripts pass are done without spending any +tokens; features without a script (or whose script failed) are handed to the +LLM, which verifies them, fixes regressions and writes/repairs the script. + +Commands are validated against the same bash allowlist the agent itself is +subject to (security.validate_bash_command) and run from the project root. +""" + +import json +import subprocess +import sys +import time +from dataclasses import dataclass, field +from pathlib import Path + +from security import validate_bash_command +from server.utils.process_utils import kill_process_tree + +# Manifest location, relative to the project directory +REGRESSION_MANIFEST = Path("tests") / "regression" / "manifest.json" + +DEFAULT_TIMEOUT_SECONDS = 300 +MAX_TIMEOUT_SECONDS = 900 + +# How much of a failing script's output to hand to the LLM +OUTPUT_TAIL_CHARS = 4000 + + +@dataclass +class ScriptResult: + """Outcome of running one feature's regression script.""" + + feature_id: int + command: str + passed: bool + duration: float + output: str + + +@dataclass +class ScriptedRegressionReport: + """Aggregated outcome of a scripted regression run for a batch of features.""" + + passed: list[int] = field(default_factory=list) + failed: list[ScriptResult] = field(default_factory=list) + unscripted: list[int] = field(default_factory=list) + + @property + def needs_agent(self) -> list[int]: + """Feature IDs that still need an LLM testing session (failed first).""" + return [r.feature_id for r in self.failed] + self.unscripted + + +def load_regression_manifest(project_dir: Path) -> dict[int, dict]: + """Load the regression manifest, returning {feature_id: entry}. + + Invalid or missing manifests yield an empty mapping (every feature is then + treated as unscripted and falls back to agent-driven testing). + """ + path = project_dir / REGRESSION_MANIFEST + if not path.is_file(): + return {} + try: + raw = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError) as e: + print(f"Warning: could not read {REGRESSION_MANIFEST}: {e}") + return {} + if not isinstance(raw, dict): + print(f"Warning: {REGRESSION_MANIFEST} must be a JSON object keyed by feature ID") + return {} + + manifest: dict[int, dict] = {} + for key, entry in raw.items(): + try: + feature_id = int(key) + except (TypeError, ValueError): + continue + # Allow the shorthand {"12": "npm run test:feature-12"} + if isinstance(entry, str): + entry = {"command": entry} + if not isinstance(entry, dict): + continue + command = entry.get("command") + if isinstance(command, str) and command.strip(): + manifest[feature_id] = entry + return manifest + + +def _entry_timeout(entry: dict) -> float: + timeout = entry.get("timeout", DEFAULT_TIMEOUT_SECONDS) + if not isinstance(timeout, (int, float)) or timeout <= 0: + return DEFAULT_TIMEOUT_SECONDS + return min(float(timeout), MAX_TIMEOUT_SECONDS) + + +def _tail(text: str, limit: int = OUTPUT_TAIL_CHARS) -> str: + return text if len(text) <= limit else "...\n" + text[-limit:] + + +def run_feature_script(project_dir: Path, feature_id: int, entry: dict) -> ScriptResult: + """Run a single feature's regression command and report the result.""" + command = entry["command"].strip() + + allowed, reason = validate_bash_command(command, project_dir) + if not allowed: + return ScriptResult(feature_id, command, False, 0.0, + f"Regression command rejected by security policy: {reason}") + + popen_kwargs: dict = { + "cwd": str(project_dir), + "shell": True, + "stdin": subprocess.DEVNULL, + "stdout": subprocess.PIPE, + "stderr": subprocess.STDOUT, + "text": True, + "encoding": "utf-8", + "errors": "replace", + } + if sys.platform == "win32": + popen_kwargs["creationflags"] = subprocess.CREATE_NO_WINDOW + + start = time.monotonic() + try: + proc = subprocess.Popen(command, **popen_kwargs) + except OSError as e: + return ScriptResult(feature_id, command, False, 0.0, f"Failed to start regression command: {e}") + + timeout = _entry_timeout(entry) + try: + output, _ = proc.communicate(timeout=timeout) + except subprocess.TimeoutExpired: + # Kill the whole tree so browsers/dev servers spawned by the test don't linger + kill_process_tree(proc, timeout=2.0) + output, _ = proc.communicate() + return ScriptResult(feature_id, command, False, time.monotonic() - start, + _tail((output or "") + f"\n[timed out after {timeout:.0f}s]")) + + return ScriptResult(feature_id, command, proc.returncode == 0, + time.monotonic() - start, _tail(output or "")) + + +def run_scripted_regression(project_dir: Path, feature_ids: list[int]) -> ScriptedRegressionReport: + """Run regression scripts for the given features. + + Features with no manifest entry are reported as unscripted. + """ + report = ScriptedRegressionReport() + manifest = load_regression_manifest(project_dir) + + for feature_id in feature_ids: + entry = manifest.get(feature_id) + if entry is None: + report.unscripted.append(feature_id) + continue + + print(f"[Feature #{feature_id}] Running scripted regression test: {entry['command']}", flush=True) + result = run_feature_script(project_dir, feature_id, entry) + status = "PASSED" if result.passed else "FAILED" + print(f"[Feature #{feature_id}] Scripted regression test {status} ({result.duration:.1f}s)", flush=True) + if result.passed: + report.passed.append(feature_id) + else: + report.failed.append(result) + + return report + + +def format_failures_for_prompt(failed: list[ScriptResult]) -> str: + """Describe failing scripts so the testing agent can start from the evidence.""" + if not failed: + return "" + lines = [ + "## SCRIPTED REGRESSION FAILURES", + "", + "The automated regression scripts below FAILED before this session started. " + "For each one, determine whether the feature has regressed (fix the app) or " + "the script itself is broken/flaky (fix the script). Re-run the command and make " + "sure it passes before ending your session.", + "", + ] + for r in failed: + lines += [ + f"### Feature #{r.feature_id}", + f"Command: `{r.command}`", + "Output (tail):", + "```", + r.output.strip() or "(no output)", + "```", + "", + ] + return "\n".join(lines) diff --git a/security.py b/security.py index 9d928b5a9..f0cf5c94c 100644 --- a/security.py +++ b/security.py @@ -893,43 +893,27 @@ def is_command_allowed(command: str, allowed_commands: set[str]) -> bool: return False -async def bash_security_hook(input_data, tool_use_id=None, context=None): +def validate_bash_command(command: str, project_dir: Optional[Path] = None) -> tuple[bool, str]: """ - Pre-tool-use hook that validates bash commands using an allowlist. + Validate a bash command string against the effective allow/block lists. - Only commands in ALLOWED_COMMANDS and project-specific commands are permitted. + This is the synchronous core of bash_security_hook, reusable by callers + that execute commands outside the agent's tool loop (e.g. scripted + regression tests run by regression_runner.py). Args: - input_data: Dict containing tool_name and tool_input - tool_use_id: Optional tool use ID - context: Optional context dict with 'project_dir' key + command: The full command string to validate + project_dir: Project directory used to resolve project/org config Returns: - Empty dict to allow, or {"decision": "block", "reason": "..."} to block + Tuple of (allowed, reason). reason is empty when allowed. """ - if input_data.get("tool_name") != "Bash": - return {} - - command = input_data.get("tool_input", {}).get("command", "") - if not command: - return {} - # Extract all commands from the command string commands = extract_commands(command) if not commands: # Could not parse - fail safe by blocking - return { - "decision": "block", - "reason": f"Could not parse command for security validation: {command}", - } - - # Get project directory from context - project_dir = None - if context and isinstance(context, dict): - project_dir_str = context.get("project_dir") - if project_dir_str: - project_dir = Path(project_dir_str) + return False, f"Could not parse command for security validation: {command}" # Get effective commands using hierarchy resolution allowed_commands, blocked_commands = get_effective_commands(project_dir) @@ -944,10 +928,7 @@ async def bash_security_hook(input_data, tool_use_id=None, context=None): for cmd in commands: # Check blocklist first (highest priority) if cmd in blocked_commands: - return { - "decision": "block", - "reason": f"Command '{cmd}' is blocked at organization level and cannot be approved.", - } + return False, f"Command '{cmd}' is blocked at organization level and cannot be approved." # Check allowlist (with pattern matching) if not is_command_allowed(cmd, allowed_commands): @@ -957,10 +938,7 @@ async def bash_security_hook(input_data, tool_use_id=None, context=None): error_msg += " 1. Add to .autoforge/allowed_commands.yaml for this project, OR\n" error_msg += " 2. Request mid-session approval (the agent can ask)\n" error_msg += "Note: Some commands are blocked at org-level and cannot be overridden." - return { - "decision": "block", - "reason": error_msg, - } + return False, error_msg # Additional validation for sensitive commands if cmd in COMMANDS_NEEDING_EXTRA_VALIDATION: @@ -979,18 +957,52 @@ async def bash_security_hook(input_data, tool_use_id=None, context=None): extra_procs = pkill_processes - DEFAULT_PKILL_PROCESSES allowed, reason = validate_pkill_command(cmd_segment, extra_procs if extra_procs else None) if not allowed: - return {"decision": "block", "reason": reason} + return False, reason elif cmd == "chmod": allowed, reason = validate_chmod_command(cmd_segment) if not allowed: - return {"decision": "block", "reason": reason} + return False, reason elif cmd == "init.sh": allowed, reason = validate_init_script(cmd_segment) if not allowed: - return {"decision": "block", "reason": reason} + return False, reason elif cmd == "playwright-cli": allowed, reason = validate_playwright_command(cmd_segment) if not allowed: - return {"decision": "block", "reason": reason} + return False, reason + + return True, "" + + +async def bash_security_hook(input_data, tool_use_id=None, context=None): + """ + Pre-tool-use hook that validates bash commands using an allowlist. + + Only commands in ALLOWED_COMMANDS and project-specific commands are permitted. + + Args: + input_data: Dict containing tool_name and tool_input + tool_use_id: Optional tool use ID + context: Optional context dict with 'project_dir' key + + Returns: + Empty dict to allow, or {"decision": "block", "reason": "..."} to block + """ + if input_data.get("tool_name") != "Bash": + return {} + + command = input_data.get("tool_input", {}).get("command", "") + if not command: + return {} + + # Get project directory from context + project_dir = None + if context and isinstance(context, dict): + project_dir_str = context.get("project_dir") + if project_dir_str: + project_dir = Path(project_dir_str) + allowed, reason = validate_bash_command(command, project_dir) + if not allowed: + return {"decision": "block", "reason": reason} return {}