From a8b07cc6d681d8222c8fdc01355510dba56085fe Mon Sep 17 00:00:00 2001 From: mohamed zwanski Date: Thu, 22 Oct 2026 21:39:51 +0100 Subject: [PATCH] Keep AutoForge's spec and progress notes out of the project root (#132) Prompt templates now read the spec from .autoforge/prompts/app_spec.txt and write progress notes to .autoforge/claude-progress.txt, so the project root only holds the application being built (plus CLAUDE.md/.claude, which Claude Code requires at the root). Migration v2 moves claude-progress.txt into .autoforge/, rewrites bare root references in existing project prompts, and removes the root app_spec.txt copy when it is identical to the canonical spec. copy_spec_to_project now only creates the root copy for customised prompts that still read it from the root. Co-Authored-By: Claude Opus 5.5 --- .../templates/auto_improve_prompt.template.md | 6 +- .claude/templates/coding_prompt.template.md | 12 +- .../templates/initializer_prompt.template.md | 12 +- CLAUDE.md | 5 +- README.md | 35 +++-- agent.py | 2 +- prompts.py | 130 +++++++++++++++-- test_project_layout.py | 133 ++++++++++++++++++ 8 files changed, 298 insertions(+), 37 deletions(-) create mode 100644 test_project_layout.py diff --git a/.claude/templates/auto_improve_prompt.template.md b/.claude/templates/auto_improve_prompt.template.md index d9ac67b93..7a7995df8 100644 --- a/.claude/templates/auto_improve_prompt.template.md +++ b/.claude/templates/auto_improve_prompt.template.md @@ -12,13 +12,13 @@ Start by orienting yourself: # Understand the project pwd ls -la -cat app_spec.txt 2>/dev/null || cat .autoforge/prompts/app_spec.txt 2>/dev/null +cat .autoforge/prompts/app_spec.txt 2>/dev/null || cat app_spec.txt 2>/dev/null # See what's been done recently (previous auto-improvements, other commits) git log --oneline -20 # See recent progress notes if they exist -tail -200 claude-progress.txt 2>/dev/null || true +tail -200 .autoforge/claude-progress.txt 2>/dev/null || tail -200 claude-progress.txt 2>/dev/null || true ``` Then use MCP tools to check feature status: @@ -73,7 +73,7 @@ Implement the change fully. Keep scope tight: - Don't add speculative abstractions or "while I'm here" refactors. - Don't add comments/docstrings to code you didn't touch. - Don't rename things that don't need renaming. -- If you discover a bug that is NOT your chosen improvement, leave it alone (or note it in `claude-progress.txt` for a future session). +- If you discover a bug that is NOT your chosen improvement, leave it alone (or note it in `.autoforge/claude-progress.txt` for a future session). If your improvement is a UI change, actually look at the result — take a screenshot with `playwright-cli` if the dev server is running, or at minimum open the relevant component and verify your edit makes sense. diff --git a/.claude/templates/coding_prompt.template.md b/.claude/templates/coding_prompt.template.md index 832eb5996..47360fcde 100644 --- a/.claude/templates/coding_prompt.template.md +++ b/.claude/templates/coding_prompt.template.md @@ -15,10 +15,10 @@ pwd ls -la # 3. Read the project specification to understand what you're building -cat app_spec.txt +cat .autoforge/prompts/app_spec.txt # 4. Read progress notes from previous sessions (last 500 lines to avoid context overflow) -tail -500 claude-progress.txt +tail -500 .autoforge/claude-progress.txt # 5. Check recent git history git log --oneline -20 @@ -31,7 +31,7 @@ Then use MCP tools to check feature status: Use the feature_get_stats tool ``` -Understanding the `app_spec.txt` is critical - it contains the full requirements +Understanding the `.autoforge/prompts/app_spec.txt` is critical - it contains the full requirements for the application you're building. ### STEP 2: START SERVERS (IF NOT RUNNING) @@ -71,7 +71,7 @@ If you must skip (truly external blocker only): Use the feature_skip tool with feature_id={id} ``` -Document the SPECIFIC external blocker in `claude-progress.txt`. "Functionality not built" is NEVER a valid reason. +Document the SPECIFIC external blocker in `.autoforge/claude-progress.txt`. "Functionality not built" is NEVER a valid reason. ### STEP 4: IMPLEMENT THE FEATURE @@ -179,7 +179,7 @@ git commit -m "feat: implement [feature name] with browser verification" ### STEP 8: UPDATE PROGRESS NOTES -Update `claude-progress.txt` with: +Update `.autoforge/claude-progress.txt` with: - What you accomplished this session - Which test(s) you completed @@ -192,7 +192,7 @@ Update `claude-progress.txt` with: Before context fills up: 1. Commit all working code -2. Update claude-progress.txt +2. Update .autoforge/claude-progress.txt 3. Mark features as passing if tests verified 4. Ensure no uncommitted changes 5. Leave app in working state (no broken features) diff --git a/.claude/templates/initializer_prompt.template.md b/.claude/templates/initializer_prompt.template.md index bb914e2d4..7bcda730e 100644 --- a/.claude/templates/initializer_prompt.template.md +++ b/.claude/templates/initializer_prompt.template.md @@ -5,7 +5,7 @@ Your job is to set up the foundation for all future coding agents. ### FIRST: Read the Project Specification -Start by reading `app_spec.txt` in your working directory. This file contains +Start by reading `.autoforge/prompts/app_spec.txt` (relative to your working directory). This file contains the complete specification for what you need to build. Read it carefully before proceeding. @@ -21,7 +21,7 @@ This number was determined during spec creation and must be followed precisely. ### CRITICAL FIRST TASK: Create Features -Based on `app_spec.txt`, create features using the feature_create_bulk tool. The features are stored in a SQLite database, +Based on `.autoforge/prompts/app_spec.txt`, create features using the feature_create_bulk tool. The features are stored in a SQLite database, which is the single source of truth for what needs to be built. **Creating Features:** @@ -34,7 +34,7 @@ Use the feature_create_bulk tool to add all features at once. You can create fea **Requirements for features:** -- Feature count must match the `feature_count` specified in app_spec.txt +- Feature count must match the `feature_count` specified in `.autoforge/prompts/app_spec.txt` - Reference tiers for other projects: - **Simple apps**: ~165 tests (includes 5 infrastructure) - **Medium apps**: ~265 tests (includes 5 infrastructure) @@ -146,7 +146,7 @@ Steps: Steps: 1. Connect to database directly (sqlite3, psql, etc.) 2. List all tables in the database -3. Verify tables match what's defined in app_spec.txt +3. Verify tables match what's defined in `.autoforge/prompts/app_spec.txt` 4. Verify key columns exist on each table ``` @@ -319,7 +319,7 @@ set up and run the development environment. The script should: 2. Start any necessary servers or services 3. Print helpful information about how to access the running application -Base the script on the technology stack specified in `app_spec.txt`. +Base the script on the technology stack specified in `.autoforge/prompts/app_spec.txt`. ### THIRD TASK: Initialize Git @@ -335,7 +335,7 @@ Commit message: "Initial setup: init.sh, project structure, and features created ### FOURTH TASK: Create Project Structure -Set up the basic project structure based on what's specified in `app_spec.txt`. +Set up the basic project structure based on what's specified in `.autoforge/prompts/app_spec.txt`. This typically includes directories for frontend, backend, and any other components mentioned in the spec. diff --git a/CLAUDE.md b/CLAUDE.md index 6f59910be..f35016202 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -120,6 +120,7 @@ python test_security_integration.py # Integration tests (9 tests) python -m pytest test_client.py # Client tests (20 tests) python -m pytest test_dependency_resolver.py # Dependency resolver tests (12 tests) python -m pytest test_rate_limit_utils.py # Rate limit tests (22 tests) +python -m pytest test_project_layout.py # Project layout / migration tests ``` ### React UI @@ -292,9 +293,9 @@ Projects can be stored in any directory (registered in `~/.autoforge/registry.db - `.playwright/cli.config.json` - Browser configuration (headless, viewport, etc.) - `.playwright-cli/` - Playwright CLI daemon artifacts (screenshots, snapshots) - gitignored - `CLAUDE.md` - Stays at project root (SDK convention) -- `app_spec.txt` - Root copy for agent template compatibility +- `.autoforge/claude-progress.txt` - Agent session progress notes -Legacy projects with files at root level (e.g., `features.db`, `prompts/`) are auto-migrated to `.autoforge/` on next agent start. Dual-path resolution ensures old and new layouts work transparently. +Legacy projects with files at root level (e.g., `features.db`, `prompts/`, `app_spec.txt`, `claude-progress.txt`) are auto-migrated to `.autoforge/` on next agent start (`migrate_project_layout` in `autoforge_paths.py`, then `migrate_project_to_current` in `prompts.py`). Prompt templates reference `.autoforge/prompts/app_spec.txt` directly; a root `app_spec.txt` copy is only made for customised prompts that still read it from the root. Dual-path resolution ensures old and new layouts work transparently. ### Security Model diff --git a/README.md b/README.md index 0ad71c941..cd639fd4e 100644 --- a/README.md +++ b/README.md @@ -209,20 +209,33 @@ autoforge/ ## Generated Project Structure -After the agent runs, your project directory will contain: +After the agent runs, your project directory will contain your application code at the root, +with everything AutoForge uses to drive the agents kept in a separate `.autoforge/` folder: ``` -generations/my_project/ -├── features.db # SQLite database (feature test cases) -├── prompts/ -│ ├── app_spec.txt # Your app specification -│ ├── initializer_prompt.md # First session prompt -│ └── coding_prompt.md # Continuation session prompt -├── init.sh # Environment setup script -├── claude-progress.txt # Session progress notes -└── [application files] # Generated application code +my_project/ +├── .autoforge/ # AutoForge's own files (not part of your app) +│ ├── prompts/ +│ │ ├── app_spec.txt # Your app specification +│ │ ├── initializer_prompt.md # First session prompt +│ │ ├── coding_prompt.md # Continuation session prompt +│ │ └── testing_prompt.md # Regression testing prompt +│ ├── features.db # SQLite database (feature test cases) +│ ├── claude-progress.txt # Agent session progress notes +│ ├── allowed_commands.yaml # Optional per-project bash allowlist +│ └── .gitignore # Ignores runtime files (locks, databases) +├── .claude/skills/ # Claude Code skills used by the agent (e.g. playwright-cli) +├── .playwright/ # Browser automation config (gitignored) +├── CLAUDE.md # Project notes for the agent (Claude Code convention) +├── init.sh # Environment setup script for your app +└── [application files] # Generated application code ``` +`CLAUDE.md` and `.claude/` stay at the project root because Claude Code only looks for them there. +Projects created by older versions (with `prompts/`, `features.db`, `app_spec.txt` or +`claude-progress.txt` in the root) are migrated into `.autoforge/` automatically the next time +an agent starts. + --- ## Running the Generated Application @@ -361,7 +374,7 @@ Requires `gcloud auth application-default login` first. Note the `@` separator ( ### Changing the Application -Use the `/create-spec` command when creating a new project, or manually edit the files in your project's `prompts/` directory: +Use the `/create-spec` command when creating a new project, or manually edit the files in your project's `.autoforge/prompts/` directory: - `app_spec.txt` - Your application specification - `initializer_prompt.md` - Controls feature generation diff --git a/agent.py b/agent.py index 668391f7b..0dd0618e5 100644 --- a/agent.py +++ b/agent.py @@ -228,7 +228,7 @@ async def run_autonomous_agent( print(" This may appear to hang - it's working. Watch for [Tool: ...] output.") print("=" * 70) print() - # Copy the app spec into the project directory for the agent to read + # Legacy prompts only: copy the app spec into the project root for the agent to read copy_spec_to_project(project_dir) elif agent_type == "testing": print("Running as TESTING agent (regression testing)") diff --git a/prompts.py b/prompts.py index 6031080d2..ae523da1c 100644 --- a/prompts.py +++ b/prompts.py @@ -17,7 +17,15 @@ TEMPLATES_DIR = Path(__file__).parent / ".claude" / "templates" # Migration version — bump when adding new migration steps -CURRENT_MIGRATION_VERSION = 1 +CURRENT_MIGRATION_VERSION = 2 + +# Agent-facing files that used to live in the project root and now live under +# .autoforge/ so the project root only contains the application being built. +PROGRESS_NOTES_FILENAME = "claude-progress.txt" + +# Matches a bare reference to a root-level file (e.g. ``cat app_spec.txt``) +# but not one that is already part of a path (``.autoforge/prompts/app_spec.txt``). +_BARE_REF_PREFIX = r"(? Path: @@ -482,18 +490,47 @@ def has_project_prompts(project_dir: Path) -> bool: return False +def _references_root_file(content: str, filename: str) -> bool: + """Return True if prompt text refers to ``filename`` as a bare, root-level path.""" + return re.search(_BARE_REF_PREFIX + re.escape(filename), content) is not None + + +def _rewrite_root_file_refs(content: str, filename: str, new_path: str) -> str: + """Rewrite bare root-level references to ``filename`` so they point at ``new_path``.""" + return re.sub(_BARE_REF_PREFIX + re.escape(filename), new_path, content) + + +def _project_prompts_reference_root_spec(project_dir: Path) -> bool: + """Check whether any project-specific prompt still reads app_spec.txt from the root.""" + prompts_dir = get_project_prompts_dir(project_dir) + for name in ("initializer_prompt.md", "coding_prompt.md", "testing_prompt.md"): + prompt_path = prompts_dir / name + if not prompt_path.exists(): + continue + try: + if _references_root_file(prompt_path.read_text(encoding="utf-8"), "app_spec.txt"): + return True + except (OSError, PermissionError): + continue + return False + + def copy_spec_to_project(project_dir: Path) -> None: """ - Copy the app spec file into the project root directory for the agent to read. - - This maintains backwards compatibility - the agent expects app_spec.txt - in the project root directory. + Copy the app spec into the project root, but only for legacy prompts. - The spec is sourced from: {project_dir}/prompts/app_spec.txt + Current prompt templates read the spec directly from + ``.autoforge/prompts/app_spec.txt`` so the project root stays free of + AutoForge files. A root-level copy is only made when a project-specific + prompt still tells the agent to read ``app_spec.txt`` from the root + (e.g. a customised prompt that the migration could not rewrite). Args: project_dir: The project directory """ + if not _project_prompts_reference_root_spec(project_dir): + return + spec_dest = project_dir / "app_spec.txt" # Don't overwrite if already exists @@ -506,7 +543,7 @@ def copy_spec_to_project(project_dir: Path) -> None: if project_spec.exists(): try: shutil.copy(project_spec, spec_dest) - print("Copied app_spec.txt to project directory") + print("Copied app_spec.txt to project directory (legacy prompt layout)") return except (OSError, PermissionError) as e: print(f"Warning: Could not copy app_spec.txt: {e}") @@ -821,6 +858,82 @@ def _migrate_v0_to_v1(project_dir: Path) -> list[str]: return migrated +def _migrate_v1_to_v2(project_dir: Path) -> list[str]: + """Migrate from v1 to v2: keep AutoForge's own files out of the project root. + + Idempotent sub-steps: + A. Make sure the canonical spec exists in the prompts directory + B. Move root-level claude-progress.txt into .autoforge/ + C. Point project prompts at the .autoforge/ locations + D. Remove the root-level app_spec.txt copy if it is identical to the canonical one + """ + from autoforge_paths import get_autoforge_dir + + migrated: list[str] = [] + prompts_dir = get_project_prompts_dir(project_dir) + autoforge_dir = get_autoforge_dir(project_dir) + canonical_spec = prompts_dir / "app_spec.txt" + root_spec = project_dir / "app_spec.txt" + + # A. Legacy projects may only have the spec at the root + if root_spec.exists() and not canonical_spec.exists(): + try: + prompts_dir.mkdir(parents=True, exist_ok=True) + shutil.copy2(root_spec, canonical_spec) + migrated.append(f"Copied app_spec.txt into {prompts_dir.relative_to(project_dir).as_posix()}/") + except (OSError, PermissionError, ValueError) as e: + print(f" Warning: Could not copy app_spec.txt into prompts directory: {e}") + + # B. Move progress notes + root_progress = project_dir / PROGRESS_NOTES_FILENAME + new_progress = autoforge_dir / PROGRESS_NOTES_FILENAME + if root_progress.exists() and not new_progress.exists(): + try: + autoforge_dir.mkdir(parents=True, exist_ok=True) + shutil.move(str(root_progress), str(new_progress)) + migrated.append(f"Moved {PROGRESS_NOTES_FILENAME} into .autoforge/") + except (OSError, PermissionError) as e: + print(f" Warning: Could not move {PROGRESS_NOTES_FILENAME}: {e}") + + # C. Rewrite prompt references + try: + spec_ref = canonical_spec.relative_to(project_dir).as_posix() + progress_ref = new_progress.relative_to(project_dir).as_posix() + except ValueError: + spec_ref, progress_ref = ".autoforge/prompts/app_spec.txt", f".autoforge/{PROGRESS_NOTES_FILENAME}" + + for name in ("initializer_prompt.md", "coding_prompt.md", "testing_prompt.md"): + prompt_path = prompts_dir / name + if not prompt_path.exists(): + continue + try: + content = prompt_path.read_text(encoding="utf-8") + updated = _rewrite_root_file_refs(content, "app_spec.txt", spec_ref) + if new_progress.exists() or not root_progress.exists(): + updated = _rewrite_root_file_refs(updated, PROGRESS_NOTES_FILENAME, progress_ref) + if updated != content: + prompt_path.write_text(updated, encoding="utf-8") + migrated.append(f"Updated {name} to read AutoForge files from .autoforge/") + except (OSError, PermissionError) as e: + print(f" Warning: Could not update {name}: {e}") + + # D. Drop the redundant root copy of the spec (only if nothing was edited in it) + if root_spec.exists() and canonical_spec.exists() and not _project_prompts_reference_root_spec(project_dir): + try: + if root_spec.read_bytes() == canonical_spec.read_bytes(): + root_spec.unlink() + migrated.append("Removed duplicate app_spec.txt from project root") + else: + print( + f" Note: {root_spec} differs from {canonical_spec}; leaving it in place. " + f"The agent now reads {spec_ref}." + ) + except (OSError, PermissionError) as e: + print(f" Warning: Could not remove root app_spec.txt: {e}") + + return migrated + + def migrate_project_to_current(project_dir: Path) -> list[str]: """Migrate an existing project to the current AutoForge version. @@ -836,7 +949,8 @@ def migrate_project_to_current(project_dir: Path) -> list[str]: if current < 1: migrated.extend(_migrate_v0_to_v1(project_dir)) - # Future: if current < 2: migrated.extend(_migrate_v1_to_v2(project_dir)) + if current < 2: + migrated.extend(_migrate_v1_to_v2(project_dir)) _set_migration_version(project_dir, CURRENT_MIGRATION_VERSION) return migrated diff --git a/test_project_layout.py b/test_project_layout.py new file mode 100644 index 000000000..6f4f1a9e6 --- /dev/null +++ b/test_project_layout.py @@ -0,0 +1,133 @@ +""" +Project Layout Tests +==================== + +Verify that AutoForge keeps its own files (spec, progress notes) under +``.autoforge/`` instead of the project root, and that older projects are +migrated without losing data. + +Run with: python -m pytest test_project_layout.py +""" + +from pathlib import Path + +import prompts +from prompts import ( + CURRENT_MIGRATION_VERSION, + _migrate_v1_to_v2, + _set_migration_version, + copy_spec_to_project, + migrate_project_to_current, + scaffold_project_prompts, +) + +SPEC = "demo\n" + +LEGACY_CODING_PROMPT = """\ +cat app_spec.txt +tail -500 claude-progress.txt +Understanding the `app_spec.txt` is critical. +Update `claude-progress.txt` with notes. +""" + + +def _make_v1_project(tmp_path: Path, coding_prompt: str = LEGACY_CODING_PROMPT) -> Path: + prompts_dir = tmp_path / ".autoforge" / "prompts" + prompts_dir.mkdir(parents=True) + (prompts_dir / "app_spec.txt").write_text(SPEC, encoding="utf-8") + (prompts_dir / "coding_prompt.md").write_text(coding_prompt, encoding="utf-8") + _set_migration_version(tmp_path, 1) + return tmp_path + + +def test_templates_do_not_reference_root_files(): + for name in ("coding_prompt", "initializer_prompt", "testing_prompt"): + content = (prompts.TEMPLATES_DIR / f"{name}.template.md").read_text(encoding="utf-8") + assert not prompts._references_root_file(content, "app_spec.txt"), name + assert not prompts._references_root_file(content, "claude-progress.txt"), name + + +def test_new_project_does_not_get_root_spec_copy(tmp_path): + scaffold_project_prompts(tmp_path) + (tmp_path / ".autoforge" / "prompts" / "app_spec.txt").write_text(SPEC, encoding="utf-8") + + copy_spec_to_project(tmp_path) + + assert not (tmp_path / "app_spec.txt").exists() + + +def test_legacy_prompt_still_gets_root_spec_copy(tmp_path): + _make_v1_project(tmp_path) + + copy_spec_to_project(tmp_path) + + assert (tmp_path / "app_spec.txt").read_text(encoding="utf-8") == SPEC + + +def test_migration_moves_files_and_rewrites_prompts(tmp_path): + project = _make_v1_project(tmp_path) + (project / "app_spec.txt").write_text(SPEC, encoding="utf-8") + (project / "claude-progress.txt").write_text("session 1 notes\n", encoding="utf-8") + + migrated = migrate_project_to_current(project) + + assert migrated + assert not (project / "app_spec.txt").exists() + assert not (project / "claude-progress.txt").exists() + assert (project / ".autoforge" / "claude-progress.txt").read_text(encoding="utf-8") == "session 1 notes\n" + + coding = (project / ".autoforge" / "prompts" / "coding_prompt.md").read_text(encoding="utf-8") + assert "cat .autoforge/prompts/app_spec.txt" in coding + assert "tail -500 .autoforge/claude-progress.txt" in coding + assert "`.autoforge/prompts/app_spec.txt`" in coding + assert not prompts._references_root_file(coding, "app_spec.txt") + + assert prompts._get_migration_version(project) == CURRENT_MIGRATION_VERSION + # Idempotent + assert migrate_project_to_current(project) == [] + + +def test_migration_is_idempotent_on_rewritten_prompts(tmp_path): + project = _make_v1_project(tmp_path) + _migrate_v1_to_v2(project) + first = (project / ".autoforge" / "prompts" / "coding_prompt.md").read_text(encoding="utf-8") + + _migrate_v1_to_v2(project) + second = (project / ".autoforge" / "prompts" / "coding_prompt.md").read_text(encoding="utf-8") + + assert first == second + assert ".autoforge/.autoforge" not in second + assert "prompts/.autoforge" not in second + + +def test_migration_keeps_edited_root_spec(tmp_path): + project = _make_v1_project(tmp_path) + (project / "app_spec.txt").write_text("edited by hand\n", encoding="utf-8") + + _migrate_v1_to_v2(project) + + assert (project / "app_spec.txt").read_text(encoding="utf-8") == "edited by hand\n" + + +def test_migration_copies_root_only_spec_into_prompts(tmp_path): + project = _make_v1_project(tmp_path) + canonical = project / ".autoforge" / "prompts" / "app_spec.txt" + canonical.unlink() + (project / "app_spec.txt").write_text(SPEC, encoding="utf-8") + + _migrate_v1_to_v2(project) + + assert canonical.read_text(encoding="utf-8") == SPEC + # Identical copy at the root is now redundant + assert not (project / "app_spec.txt").exists() + + +def test_migration_does_not_overwrite_existing_progress_notes(tmp_path): + project = _make_v1_project(tmp_path) + (project / "claude-progress.txt").write_text("root notes\n", encoding="utf-8") + (project / ".autoforge" / "claude-progress.txt").write_text("new notes\n", encoding="utf-8") + + _migrate_v1_to_v2(project) + + assert (project / "claude-progress.txt").read_text(encoding="utf-8") == "root notes\n" + assert (project / ".autoforge" / "claude-progress.txt").read_text(encoding="utf-8") == "new notes\n"