feat(docker): bake the Pi CLI into the runtime kit

docker/Dockerfile

@@ -34,13 +34,16 @@ ARG CLAUDE_CODE_VERSION=2.1.281
 RUN curl -fsSL https://deb.nodesource.com/setup_22.x | bash - \
     && apt-get install -y --no-install-recommends nodejs \
     && rm -rf /var/lib/apt/lists/* \
-    && npm install -g @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}
+    && npm install -g --engine-strict @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}
 
 # Pi Node CLI, pinned — same rationale as the Claude Code pin above (the agent
 # binary is a dominant non-model driver of results; @latest would freeze
 # nondeterministically under layer caching). Node 22 + npm are already present.
+# --engine-strict (here and in docker/Dockerfile.runtime) turns a CLI pin whose
+# `engines.node` floor is above the installed Node into a build failure instead
+# of an EBADENGINE warning that installs anyway.
 ARG PI_VERSION=0.87.1
-RUN npm install -g @earendil-works/pi-coding-agent@${PI_VERSION}
+RUN npm install -g --engine-strict @earendil-works/pi-coding-agent@${PI_VERSION}
 
 # uv: matches host sandbox.py's `uv venv` + `uv pip install` fast path.
 # Pinned for the same reason as the CLIs above: agents run `uv` themselves
@@ -87,7 +90,10 @@ COPY experiments/default.yaml ./experiments/default.yaml
 # so an OpenCode env-auth run would authenticate once the CLI is baked; the CLI is
 # the remaining blocker.) Adding it means a pinned version that travels with the release
 # tag (as CLAUDE_CODE_VERSION / PI_VERSION do) plus an env_passthrough block; see
-# docs/agents/OPENCODE.md "Running in Docker".
+# docs/agents/OPENCODE.md "Running in Docker". A Node CLI baked here must also be
+# baked, at the same pin, into docker/Dockerfile.runtime, with a pin-parity test
+# in tests/test_image_from_dockerfiles.py — otherwise inject-mode tasks find no
+# CLI, as happened with Pi before it was added to the kit.
 #
 # CODER_EVAL_UV_EXTRAS carries ADDITIONAL opt-in extras on top of those; it
 # defaults to none. `make docker-image-full` passes `--extra uipath`, which

docker/Dockerfile.runtime

@@ -10,7 +10,7 @@
 #
 # Contract (must match convert_skillsbench_task.py's inject block + the host):
 #   /opt/coder-eval/venv/bin/coder-eval          the CLI (its own Python)
-#   /opt/coder-eval/node/bin/{node,claude}       Node + the Claude Code CLI
+#   /opt/coder-eval/node/bin/{node,claude,pi}    Node + the Claude Code and Pi CLIs
 #   /usr/local/bin/coder_eval_entrypoint.sh      entrypoint (host pins --entrypoint here)
 #   LABEL org.coder-eval.version=<ver>           for `docker inspect` / parity
 #
@@ -65,7 +65,14 @@ RUN export SAFE_CHAIN_MINIMUM_PACKAGE_AGE_EXCLUSIONS="${SAFE_CHAIN_MINIMUM_PACKA
     uv pip install --python /opt/coder-eval/venv/bin/python --no-deps /src
 
 # --- 3. Node LTS + the Claude Code CLI, under the kit dir. -------------------
-ARG NODE_VERSION=22.14.0
+# NODE_VERSION tracks the newest Node 22 LTS, which is what the framework image
+# gets from nodesource's setup_22.x — so an injected task runs the CLIs on the
+# same Node as a rebased one. Every `npm install -g` below passes
+# --engine-strict: a CLI whose `engines.node` floor is above NODE_VERSION then
+# fails the build, instead of npm printing EBADENGINE and installing anyway
+# (Pi 0.87.1 needs >=22.19.0; on the previous 22.14.0 pin it installed, ran
+# `pi --version` fine and would only have misbehaved mid-task).
+ARG NODE_VERSION=22.23.4
 ARG CLAUDE_CODE_VERSION=2.1.281
 # Download to a file first so a bad URL fails the build loudly (a piped
 # `curl | tar` would mask a 404 and only surface later as a missing npm), and
@@ -80,9 +87,20 @@ RUN mkdir -p /opt/coder-eval/node \
     && grep " ${NODE_TARBALL}\$" /tmp/node.shasums | sed 's#  .*#  /tmp/node.tar.xz#' | sha256sum -c - \
     && tar -xJf /tmp/node.tar.xz -C /opt/coder-eval/node --strip-components=1 \
     && rm /tmp/node.tar.xz /tmp/node.shasums \
-    && PATH="/opt/coder-eval/node/bin:${PATH}" npm install -g --prefix /opt/coder-eval/node \
+    && PATH="/opt/coder-eval/node/bin:${PATH}" npm install -g --engine-strict --prefix /opt/coder-eval/node \
        @anthropic-ai/claude-code@${CLAUDE_CODE_VERSION}
 
+# --- 4. The Pi CLI, under the kit dir. --------------------------------------
+# Same package and pin as docker/Dockerfile, so `--driver docker --type pi` runs
+# in an injected task image as it does in a rebased one; a parity test
+# (tests/test_image_from_dockerfiles.py) keeps the two PI_VERSION pins equal.
+# `pi --version` fails the build here if the package's bin did not land on the
+# kit's PATH, instead of every pi task failing later with "pi CLI not found".
+ARG PI_VERSION=0.87.1
+RUN PATH="/opt/coder-eval/node/bin:${PATH}" npm install -g --engine-strict --prefix /opt/coder-eval/node \
+       @earendil-works/pi-coding-agent@${PI_VERSION} \
+    && PATH="/opt/coder-eval/node/bin:${PATH}" pi --version
+
 # Sanity: the CLI runs from the kit (no reliance on a system Python). Run it in
 # the builder, where the kit lives; the final stage is a bare carrier.
 RUN /opt/coder-eval/venv/bin/coder-eval _run-task-internal --help > /dev/null
@@ -99,7 +117,7 @@ COPY --from=builder /opt/coder-eval /opt/coder-eval
 # Entrypoint: puts the kit's bins on PATH, then runs the orchestrator. Same
 # filename + location as the framework image, so the host's fixed
 # `--entrypoint /usr/local/bin/coder_eval_entrypoint.sh` works for injected
-# images too — coder-eval/claude/node all resolve from the kit regardless of what
+# images too — coder-eval/claude/pi/node all resolve from the kit regardless of what
 # the task's base image provides.
 COPY docker/coder_eval_runtime_entrypoint.sh /usr/local/bin/coder_eval_entrypoint.sh
 RUN chmod +x /usr/local/bin/coder_eval_entrypoint.sh

docker/coder_eval_runtime_entrypoint.sh

@@ -5,8 +5,8 @@
 # injected image keeps the task's own base and only has the self-contained kit
 # under /opt/coder-eval. So this entrypoint prepends the kit's Python venv and
 # Node bin dirs to PATH before launching the orchestrator — making `coder-eval`,
-# `claude`, and `node` resolve from the kit regardless of what the task's base
-# image provides (or doesn't).
+# `claude`, `pi`, and `node` resolve from the kit regardless of what the task's
+# base image provides (or doesn't).
 #
 # Same filename/location as the framework entrypoint, so the host's fixed
 # `docker run --entrypoint /usr/local/bin/coder_eval_entrypoint.sh` works for

docs/DOCKER_ISOLATION.md

@@ -145,7 +145,7 @@ LABEL org.coder-eval.version="<ver>"
 ```
 
 `make coder-eval-runtime` builds the kit (`docker/Dockerfile.runtime`): a
-standalone CPython + Node + the `coder-eval` CLI + Claude Code, all under
+standalone CPython + Node + the `coder-eval` CLI + Claude Code + Pi, all under
 `/opt/coder-eval`, plus the entrypoint at the **same** `/usr/local/bin/...` path
 the host pins. The kit is **glibc-only** — it runs on debian/ubuntu/fedora/rhel/…
 but not musl/Alpine.
@@ -158,6 +158,9 @@ individual targets when you only need one.
 > The kit installs the **no-credential** set only (core + codex), like `make docker-image` —
 > it never installs the `[uipath]` extra, so there is no `make docker-images-full`. An
 > inject-mode task that needs the LLMGW/`uipath` judge isn't supported by the kit as built.
+> Agents under inject mode: `claude-code`, `codex` and `pi` run from the kit; `antigravity`
+> and the `litellm` judge route are in the framework image only, so a task that needs them
+> must rebase onto `coder-eval-agent`.
 
 ```dockerfile
 # environment/Dockerfile

docs/agents/PI.md

@@ -216,10 +216,12 @@ success — the error names the unrecognized event types it saw. Intentional cut
 ## Running in Docker
 
 Pi **is supported under `--driver docker`.** The pinned Pi CLI is baked into
-`docker/Dockerfile` (`ARG PI_VERSION`, alongside the claude-code CLI), and
-`OPENROUTER_API_KEY` is forwarded into the container by the docker driver's
-default `env_passthrough` allowlist — so `--driver docker --type pi` runs
-end-to-end with no per-task `env_passthrough_extra`:
+`docker/Dockerfile` (`ARG PI_VERSION`, alongside the claude-code CLI) and, at the
+same pin, into the runtime kit (`docker/Dockerfile.runtime`), so a task that
+brings its own base image runs Pi too. `OPENROUTER_API_KEY` is forwarded into
+the container by the docker driver's default `env_passthrough` allowlist — so
+`--driver docker --type pi` runs end-to-end with no per-task
+`env_passthrough_extra`:
 
 ```bash
 export OPENROUTER_API_KEY="sk-or-..."

tests/test_image_from_dockerfiles.py

@@ -418,6 +418,19 @@ def _runtime_dockerfile() -> Path | None:
     return df if df.is_file() else None
 
 
+def _framework_dockerfile_text() -> str:
+    return (Path(__file__).resolve().parents[1] / "docker" / "Dockerfile").read_text(encoding="utf-8")
+
+
+def _arg_pin(df_text: str, name: str) -> str | None:
+    """The value of the first `ARG <name>=<value>` line in a Dockerfile's text, or None."""
+    for ln in df_text.splitlines():
+        m = re.match(rf"\s*ARG {re.escape(name)}=(\S+)", ln)
+        if m:
+            return m.group(1)
+    return None
+
+
 def test_runtime_kit_stamps_version_label() -> None:
     """The kit Dockerfile must stamp `org.coder-eval.version`.
 
@@ -460,46 +473,58 @@ def test_runtime_kit_path_matches_install_dirs() -> None:
         )
 
 
-def test_claude_code_version_pin_matches_framework() -> None:
-    """The Claude Code pin must be identical in the framework and runtime-kit
-    Dockerfiles — else inject-mode tasks run a different agent version than
-    rebase-mode tasks (a reproducibility/fairness drift, since the agent binary
-    is a dominant non-model driver of results)."""
-    root = Path(__file__).resolve().parents[1]
+@pytest.mark.parametrize("arg", ["CLAUDE_CODE_VERSION", "PI_VERSION"])
+def test_cli_version_pin_matches_framework(arg: str) -> None:
+    """Each baked agent CLI's pin must be identical in the framework and
+    runtime-kit Dockerfiles — else inject-mode tasks run a different agent
+    version than rebase-mode tasks (a reproducibility/fairness drift, since the
+    agent binary is a dominant non-model driver of results), or no agent at all
+    when the kit lacks the CLI."""
     runtime_df = _runtime_dockerfile()
     if runtime_df is None:
         pytest.skip("docker/Dockerfile.runtime not present")
 
-    def _pin(df: Path) -> str | None:
-        for ln in df.read_text(encoding="utf-8").splitlines():
-            m = re.match(r"\s*ARG CLAUDE_CODE_VERSION=(\S+)", ln)
-            if m:
-                return m.group(1)
-        return None
-
-    fw, kit = _pin(root / "docker" / "Dockerfile"), _pin(runtime_df)
-    assert fw and kit, f"CLAUDE_CODE_VERSION ARG missing (framework={fw!r}, kit={kit!r})"
+    fw = _arg_pin(_framework_dockerfile_text(), arg)
+    kit = _arg_pin(runtime_df.read_text(encoding="utf-8"), arg)
+    assert fw and kit, f"{arg} ARG missing (framework={fw!r}, kit={kit!r})"
     assert fw == kit, (
-        f"CLAUDE_CODE_VERSION drift: docker/Dockerfile pins {fw!r} but docker/Dockerfile.runtime pins {kit!r} — "
-        "inject and rebase tasks would run different Claude Code versions."
+        f"{arg} drift: docker/Dockerfile pins {fw!r} but docker/Dockerfile.runtime pins {kit!r} — "
+        "inject and rebase tasks would run different CLI versions."
     )
 
 
+def test_npm_global_installs_are_engine_strict() -> None:
+    """Every `npm install -g` in both Dockerfiles must pass --engine-strict.
+    Without it npm only warns (EBADENGINE) when a CLI's `engines.node` floor is
+    above the image's Node and installs anyway, so the build — and the kit's
+    `pi --version` gate — stays green while the agent runs on a Node it does not
+    support (Pi 0.87.1 needs >=22.19.0; the kit pinned 22.14.0)."""
+    sources = {"docker/Dockerfile": _framework_dockerfile_text()}
+    runtime_df = _runtime_dockerfile()
+    if runtime_df is not None:
+        sources["docker/Dockerfile.runtime"] = runtime_df.read_text(encoding="utf-8")
+    for label, text in sources.items():
+        installs = [
+            ln.strip() for ln in text.splitlines() if "npm install -g" in ln and not ln.lstrip().startswith("#")
+        ]
+        assert installs, f"{label}: no `npm install -g` found — did the CLI installs move?"
+        for install in installs:
+            assert "--engine-strict" in install, (
+                f"{label}: `{install}` lacks --engine-strict, so an unsupported Node would install the CLI "
+                "with only a warning instead of failing the build."
+            )
+
+
 def test_uv_pinned_in_agent_image() -> None:
     """docker/Dockerfile must install a PINNED uv through the versioned
     installer URL. Agents run `uv` themselves, so its install speed is part of
     task wall clock: the unpinned `install.sh` moved the image from uv 0.12.9 to
     0.12.12 on a rebuild, and 0.12.11+ copies cross-filesystem (cache in the
     image, venv on the bind-mounted workspace) ~40x slower. Fails on an
     unversioned installer URL or a missing/`latest` pin."""
-    df_text = (Path(__file__).resolve().parents[1] / "docker" / "Dockerfile").read_text(encoding="utf-8")
+    df_text = _framework_dockerfile_text()
 
-    pin: str | None = None
-    for ln in df_text.splitlines():
-        m = re.match(r"\s*ARG UV_VERSION=(\S+)", ln)
-        if m:
-            pin = m.group(1)
-            break
+    pin = _arg_pin(df_text, "UV_VERSION")
     assert pin, "ARG UV_VERSION=<version> missing from docker/Dockerfile — uv is not pinned."
     assert re.fullmatch(r"\d+\.\d+\.\d+", pin), f"UV_VERSION must be an exact x.y.z version, got {pin!r}."
 
@@ -513,31 +538,24 @@ def test_uv_pinned_in_agent_image() -> None:
 
 
 def test_pi_cli_baked_and_pinned() -> None:
-    """docker/Dockerfile must ship a PINNED Pi CLI, and its install line must
-    reference the ARG (not a hardcoded literal) so a bump is one edit and the
-    pin can't drift from the install. Mirrors the claude-code pinned-CLI guard
-    above — Pi's presence is a positive, mechanically-checkable invariant
-    (`--driver docker --type pi` depends on it). Fails on @latest or removal.
-    Scope: main docker/Dockerfile only — Dockerfile.runtime deliberately does
-    NOT bake Pi (inject-mode is out of scope), so it is not asserted here."""
-    df_text = (Path(__file__).resolve().parents[1] / "docker" / "Dockerfile").read_text(encoding="utf-8")
-
-    pin: str | None = None
-    for ln in df_text.splitlines():
-        m = re.match(r"\s*ARG PI_VERSION=(\S+)", ln)
-        if m:
-            pin = m.group(1)
-            break
-    assert pin, "ARG PI_VERSION=<version> missing from docker/Dockerfile — the Pi CLI is not pinned."
-    assert pin.lower() != "latest", (
-        f"PI_VERSION is pinned to {pin!r} — the Pi CLI must be an exact version, not `latest`, so layer "
-        "caching cannot freeze it nondeterministically across rebuilds."
-    )
-
-    assert re.search(
-        r"npm install -g @earendil-works/pi-coding-agent@\$\{PI_VERSION\}",
-        df_text,
-    ), (
-        "docker/Dockerfile must `npm install -g @earendil-works/pi-coding-agent@${PI_VERSION}` — the install "
-        "must reference the PI_VERSION ARG so the pin and the install cannot drift."
-    )
+    """Both images must ship a PINNED Pi CLI, and the install must reference
+    the ARG (not a hardcoded literal) so a bump is one edit and the pin can't
+    drift from the install. Pi's presence is a positive, mechanically-checkable
+    invariant (`--driver docker --type pi` depends on it, in a rebased and in an
+    injected task image alike). Fails on @latest, a literal, or removal; the
+    pin-parity test above keeps the two ARGs equal."""
+    sources = {"docker/Dockerfile": _framework_dockerfile_text()}
+    runtime_df = _runtime_dockerfile()
+    if runtime_df is not None:
+        sources["docker/Dockerfile.runtime"] = runtime_df.read_text(encoding="utf-8")
+    for label, df_text in sources.items():
+        pin = _arg_pin(df_text, "PI_VERSION")
+        assert pin, f"ARG PI_VERSION=<version> missing from {label} — the Pi CLI is not pinned."
+        assert pin.lower() != "latest", (
+            f"{label}: PI_VERSION is pinned to {pin!r} — the Pi CLI must be an exact version, not `latest`, so "
+            "layer caching cannot freeze it nondeterministically across rebuilds."
+        )
+        assert re.findall(r"@earendil-works/pi-coding-agent@(\S+)", df_text) == ["${PI_VERSION}"], (
+            f"{label} must install @earendil-works/pi-coding-agent@${{PI_VERSION}} exactly once — the install "
+            "must reference the PI_VERSION ARG so the pin and the install cannot drift."
+        )