diff --git a/.github/workflows/ci-image.yml b/.github/workflows/ci-image.yml index 9598c81cbd..92b7baecdb 100644 --- a/.github/workflows/ci-image.yml +++ b/.github/workflows/ci-image.yml @@ -465,3 +465,203 @@ jobs: - name: Skip notice if: steps.check.outputs.should_build == 'false' run: echo "::notice::App image build skipped - no changes to app/Dockerfile, the nginx configuration or the build inputs" + + # The renderer image (agents/renderer/Dockerfile) is what every Cloud Run + # sandbox runs the model-written code with, and what the local Docker backend + # runs in development. Without this job its first build ever would be the + # bootstrap deploy, after the merge. `sandbox do` exists only on Cloud Run, so + # the job checks the image the way the local backend uses it: the harness + # renders a seaborn plot under the deployed rlimits, with no network and a + # read-only root filesystem, and the service boots and reports itself. The + # event-shape handling is the API job's in short form. + renderer-image: + name: Build renderer image and render through the harness + runs-on: ubuntu-latest + permissions: + contents: read + # A cold build installs pandas, numpy, matplotlib and seaborn; 20 fails + # loudly instead of hanging on a slow package index. + timeout-minutes: 20 + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + + - name: Check for renderer-image changes + id: check + env: + EVENT_NAME: ${{ github.event_name }} + PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} + PR_HEAD_SHA: ${{ github.event.pull_request.head.sha }} + MG_BASE_SHA: ${{ github.event.merge_group.base_sha }} + MG_HEAD_SHA: ${{ github.event.merge_group.head_sha }} + PUSH_BEFORE: ${{ github.event.before }} + PUSH_AFTER: ${{ github.event.after }} + FORCE_RUN: ${{ inputs.force_run }} + run: | + set -uo pipefail + CHANGED_FILES="" + case "$EVENT_NAME" in + pull_request) + if ! CHANGED_FILES=$(git diff --name-only "$PR_BASE_SHA" "$PR_HEAD_SHA"); then + echo "::error::could not diff $PR_BASE_SHA..$PR_HEAD_SHA — refusing to decide whether the renderer image needs building" + exit 1 + fi ;; + merge_group) + if ! CHANGED_FILES=$(git diff --name-only "$MG_BASE_SHA" "$MG_HEAD_SHA"); then + echo "::error::could not diff $MG_BASE_SHA..$MG_HEAD_SHA — refusing to decide whether the renderer image needs building" + exit 1 + fi ;; + push) + if [[ "$PUSH_BEFORE" =~ ^0+$ ]] || ! CHANGED_FILES=$(git diff --name-only "$PUSH_BEFORE" "$PUSH_AFTER"); then + echo "::warning::the pushed range is not diffable — building the renderer image rather than skipping the gate" + echo "should_build=true" >> "$GITHUB_OUTPUT" + exit 0 + fi ;; + *) + if ! CHANGED_FILES=$(git diff --name-only HEAD~1 HEAD); then + echo "::warning::no parent commit to diff against — building the renderer image rather than skipping the gate" + echo "should_build=true" >> "$GITHUB_OUTPUT" + exit 0 + fi ;; + esac + + echo "Changed files:" + echo "$CHANGED_FILES" + + # What the image is built from: its Dockerfile, the service package, + # the harness it copies, the dependency lock, README.md (copied next + # to pyproject.toml) and the root .dockerignore (the build context). + RENDERER_CHANGES=$(echo "$CHANGED_FILES" | grep -E '^(agents/renderer/|agents/__init__\.py$|agents/anyplot/render/harness\.py$|pyproject\.toml$|uv\.lock$|README\.md$|\.dockerignore$|\.github/workflows/ci-image\.yml$)' || true) + + if [[ "$EVENT_NAME" == "workflow_dispatch" && "$FORCE_RUN" == "true" ]]; then + echo "Manual trigger with force_run=true, will build the renderer image" + echo "should_build=true" >> "$GITHUB_OUTPUT" + elif [[ -n "$RENDERER_CHANGES" ]]; then + echo "Found renderer-image changes, will build the renderer image" + echo "should_build=true" >> "$GITHUB_OUTPUT" + else + echo "No renderer-image changes, skipping the build" + echo "should_build=false" >> "$GITHUB_OUTPUT" + fi + + - name: Set up Buildx + if: steps.check.outputs.should_build == 'true' + uses: docker/setup-buildx-action@f87e5991a6d7451dcb8d9637bfbc97413f497069 # v4.4.1 + + # Its own cache scope, like the app image: no layer is shared with the API. + - name: Build the renderer image + if: steps.check.outputs.should_build == 'true' + uses: docker/build-push-action@c3c9e263c25d99ce0380d002d59b67737d91b0dc # v7.4.0 + with: + context: . + file: agents/renderer/Dockerfile + push: false + load: true + tags: anyplot-renderer:ci + cache-from: type=gha,scope=renderer-image + cache-to: type=gha,mode=max,scope=renderer-image + + # THE assert this job exists for: the image renders. The flags are the + # local backend's (agents/anyplot/render/backends/local.py) and the + # environment is the sandbox's (agents/renderer/executor.py), except + # RLIMIT_NPROC: outside a sandbox it counts every process of the runner's + # user, so it would test the runner, not the image. + - name: Render a seaborn plot through the harness + if: steps.check.outputs.should_build == 'true' + run: | + set -euo pipefail + mkdir -p work + printf 'x,y\na,1\nb,3\nc,2\n' > work/data.csv + cat > work/plot.py <<'EOF' + import os + + import matplotlib.pyplot as plt + import pandas as pd + import seaborn as sns + + THEME = os.environ["ANYPLOT_THEME"] + df = pd.read_csv("data.csv") + fig, ax = plt.subplots(figsize=(16, 9), dpi=200) + sns.barplot(data=df, x="x", y="y", ax=ax) + fig.savefig(f"plot-{THEME}.png") + EOF + docker run --rm --network none --read-only --tmpfs /tmp \ + --cap-drop ALL --security-opt no-new-privileges --user "$(id -u):$(id -g)" \ + -v "$PWD/work:/work" -w /work \ + -e ANYPLOT_THEME=light -e MPLBACKEND=Agg -e MPLCONFIGDIR=/tmp/mpl -e ANYPLOT_MPL_SEED=/opt/mplconfig \ + -e HOME=/tmp -e PYTHONDONTWRITEBYTECODE=1 -e OPENBLAS_NUM_THREADS=1 -e OMP_NUM_THREADS=1 \ + -e ANYPLOT_RLIMIT_CPU_S=60 -e ANYPLOT_RLIMIT_FSIZE_MB=50 -e ANYPLOT_RLIMIT_AS_MB=1024 \ + anyplot-renderer:ci /app/.venv/bin/python -I /opt/anyplot/harness.py plot.py > harness.out + grep -qxF 'HARNESS {"event": "start"}' harness.out || { echo "::error::no HARNESS start line"; cat harness.out; exit 1; } + grep -q '^HARNESS {"event": "end"' harness.out || { echo "::error::no HARNESS end line"; cat harness.out; exit 1; } + python3 - <<'EOF' + import json, struct + png = open("work/plot-light.png", "rb").read() + assert png[:8] == b"\x89PNG\r\n\x1a\n", "not a PNG" + width, height = struct.unpack(">II", png[16:24]) + assert (width, height) == (3200, 1800), (width, height) + probe = json.load(open("work/probe-light.json")) + assert probe["canvas"] == [3200, 1800], probe.get("canvas") + print(f"OK: plot-light.png {width}x{height}, {len(png)} bytes; probe written") + EOF + tail -n 1 harness.out + + - name: The image carries no ADK and the version it reports + if: steps.check.outputs.should_build == 'true' + run: | + set -euo pipefail + docker run --rm anyplot-renderer:ci /app/.venv/bin/python -c " + import importlib.util, sys + try: + spec = importlib.util.find_spec('google.adk') + except ModuleNotFoundError: + spec = None + sys.exit('ADK is installed in the renderer image' if spec else 0) + " + echo "OK: no ADK" + want=$(python3 -c "import tomllib;print(tomllib.load(open('pyproject.toml','rb'))['project']['version'])") + + # The service boots (development: no caller check, no launcher on a + # runner) and reports itself; a render is refused, not attempted. + docker run -d --name renderer -p 8003:8080 -e ENVIRONMENT=development anyplot-renderer:ci + ready=0 + for _ in $(seq 30); do + if curl -fsS localhost:8003/status -o status.json 2>/dev/null; then ready=1; break; fi + sleep 1 + done + if [ "$ready" -ne 1 ]; then + echo "::error::the renderer never answered /status within 30 s" + exit 1 + fi + python3 -c " + import json, sys + s = json.load(open('status.json')) + assert s['name'] == 'anyplot-renderer' and s['sandbox'] is False, s + assert s['version'] == sys.argv[1], (s['version'], sys.argv[1]) + print('OK: /status', s['version']) + " "$want" + code=$(curl -s -o render.json -w '%{http_code}' -H 'Content-Type: application/json' \ + --data '{"job_id": "ci", "language": "python", "library": "matplotlib", "source": "x = 1", "data_csv": "", "themes": ["light"], "timeout_s": 10}' \ + localhost:8003/render) + test "$code" = "503" && grep -q '"sandbox_unavailable"' render.json \ + || { echo "::error::expected 503 sandbox_unavailable, got $code: $(cat render.json)"; exit 1; } + echo "OK: a render without a launcher is refused" + + - name: Container logs on failure + if: failure() && steps.check.outputs.should_build == 'true' + run: docker logs renderer 2>&1 || true + + # Same threshold as the two Dockerfiles above; checked locally against + # hadolint 2.15.1 with no finding. + - name: Hadolint (agents/renderer/Dockerfile) + if: steps.check.outputs.should_build == 'true' + uses: hadolint/hadolint-action@06be81baf89a55ffd0e24b8f04a4185738dd3387 # v3.5.0 + with: + dockerfile: agents/renderer/Dockerfile + failure-threshold: warning + + - name: Skip notice + if: steps.check.outputs.should_build == 'false' + run: echo "::notice::Renderer image build skipped - no changes to agents/renderer/, the harness or the dependency lock" diff --git a/agents/README.md b/agents/README.md index 3d5eb2a553..8965e4cb81 100644 --- a/agents/README.md +++ b/agents/README.md @@ -4,7 +4,7 @@ This directory holds the anyplot agent network: the service that lets an admin p ## What is built -The runtime core runs locally: the agents, the plot pipeline, the guardrail plugins, the render layer, the private `/v1` service with its run queue, and the theme toggle. Not built yet: the Cloud Run sandbox render backend (it waits for spike S), the container image, the deploy, the regression harness and the evals. +The runtime core runs locally: the agents, the plot pipeline, the guardrail plugins, the render layer, the private `/v1` service with its run queue, and the theme toggle. The renderer service `anyplot-renderer` (`renderer/`), which runs the adapted code in Cloud Run sandboxes, and the `remote` backend that calls it are built, with the renderer's image and Cloud Build config, but not deployed yet (status of 2026-10-10). Not built yet: the agents image, the deploy of the agents service, the regression harness and the evals. The model is **Claude Haiku 5.5 on Vertex AI** (`claude-haiku-5-5`) by default. **Gemini 3.8 Flash** is the second arm: set `AGENT_PROVIDER=gemini` together with Gemini model ids, so the two can be compared on price and quality later. Every agent and the scope judge run on the configured provider. @@ -24,7 +24,8 @@ The model is **Claude Haiku 5.5 on Vertex AI** (`claude-haiku-5-5`) by default. | `anyplot/sub_agents/` | One single-turn adapter per enabled library and the tool-less reviewer | | `anyplot/tools/session.py` | The root's tools: `get_dataset_profile`, `get_spec_brief`, `get_current_code`, `set_bindings` and the `plot_pipeline` workflow | | `anyplot/plugins/` | `ScopeGuardPlugin`, `BudgetPlugin`, `ToolSafetyPlugin` and the request ledger they share | -| `anyplot/render/` | The render contract, the probe harness, the host gates (R1-R3, advisory G3/G5/G7/G8), PNG hardening, the render store, the `fake`, `local` and `sandbox` backends, and `serial.py`, the one render semaphore in front of every backend | +| `anyplot/render/` | The render contract, the probe harness, the host gates (R1-R3, advisory G3/G5/G7/G8), PNG hardening, the render store, the `remote` (the renderer service, phase 1), `local`, `fake` and `sandbox` (an unused in-process stub) backends, and `serial.py`, the one render semaphore in front of every backend | +| `renderer/` | The `anyplot-renderer` service: `main.py` (`POST /render`, `GET /status`, one render slot), `executor.py` (one `sandbox do` per theme with the kill path, the byte watchdog and the launcher retry), `auth.py` (the caller check), `settings.py` (`RENDERER_*`), `wire.py` (the JSON contract the `remote` backend shares), `Dockerfile` and `cloudbuild.yaml`. No ADK import and no `__init__.py`, so `adk web agents` does not list it as an agent | | `anyplot/opening.py`, `session_state.py`, `services.py`, `briefs.py` | Opening a session and taking in a dataset, the server-set session state, the process-wide stores, and the spec and dataset briefs | | `anyplot/dev_fixture.py` | The development-only session seed from an eval fixture case | | `anyplot/settings.py` | `AgentSettings`, read from `AGENT_*` environment variables | @@ -36,7 +37,7 @@ The unit tests are in `tests/unit/agents/`; `tests/unit/agents/runtime/` drives Three rules hold for everything here: -- **Relative imports inside `anyplot/`.** ADK's loader imports the package as the top-level module `anyplot`, while the service imports it as `agents.anyplot`. Absolute imports of `core` are fine. +- **Relative imports inside `anyplot/`.** ADK's loader imports the package as the top-level module `anyplot`, while the service imports it as `agents.anyplot`. Absolute imports of `core` and of `agents.renderer.wire` (no ADK, no `anyplot` import) are fine. The other direction is closed: nothing in `renderer/` imports `agents.anyplot`, whose package import loads ADK. - **Models only through `models.py`.** Model ids, the project and the location come from `AgentSettings`; the settings refuse `-latest` aliases, regional locations such as `europe-west4`, and model ids that do not match the provider. - **No content in logs.** The attribution log carries ids, counts and verdicts, never message text, code or data; the service switches span content capture off. @@ -50,8 +51,10 @@ Three rules hold for everything here: | `AGENT_LOCATION` | `eu` | Vertex AI location: `eu`, `us` or `global` | | `AGENT_PROJECT`, else `GOOGLE_CLOUD_PROJECT` | `anyplot` | Project that serves and bills the model calls | | `AGENT_LIBRARIES` | `matplotlib,seaborn` | Enabled libraries; each needs a phase-1 runtime (`matplotlib`, `seaborn`), others are refused at startup | -| `AGENT_RENDERER` | `sandbox` | `sandbox`, `local` (Docker, development only), `fake` (fixture PNGs, development and test only) or `remote` | -| `AGENT_RENDER_IMAGE` | `anyplot-agents:dev` | Image the `local` renderer runs | +| `AGENT_RENDERER` | `remote` | `remote` (the renderer service), `local` (Docker, development only), `fake` (fixture PNGs, development and test only) or `sandbox` (an in-process stub, not used in phase 1) | +| `AGENT_RENDER_URL` | unset | The renderer service URL; the `remote` backend refuses to start without it; `http://` only in development and test | +| `AGENT_RENDER_TOKEN` | unset | Development only: an ID token for the renderer; without it the backend mints one (see [Render through the deployed renderer](#render-through-the-deployed-renderer)) | +| `AGENT_RENDER_IMAGE` | `anyplot-renderer:dev` | Image the `local` renderer runs, built from `renderer/Dockerfile` | | `AGENT_RENDER_CONCURRENCY` | `1` | Theme renders at the same time, for every backend (`render/serial.py`); serial, because one 4 GiB instance holds one sandbox safely (spikes S and S2) | | `AGENT_RUN_CONCURRENCY` | `1` | Pipeline runs (whole `/messages` turns) in flight; the run queue holds the rest | | `AGENT_RUNS_PER_MINUTE` | `1` | Runs that may start within any 60 seconds (a sliding window) | @@ -99,7 +102,11 @@ Three rules hold for everything here: For the Gemini arm, set `AGENT_PROVIDER=gemini AGENT_MODEL=gemini-3.8-flash AGENT_JUDGE_MODEL=gemini-3.5-flash-lite` instead. -4. Choose a renderer: `AGENT_RENDERER=local` renders in Docker with the image in `AGENT_RENDER_IMAGE` (the agents image is not built yet, so this needs an image with Python, matplotlib and seaborn at `/app/.venv/bin/python`); `AGENT_RENDERER=fake` returns fixture PNGs and runs no code. +4. Choose a renderer: + + - `AGENT_RENDERER=remote` renders through the deployed renderer service; see [Render through the deployed renderer](#render-through-the-deployed-renderer). + - `AGENT_RENDERER=local` renders in Docker with the image in `AGENT_RENDER_IMAGE`. Build it once from the repository root with `docker build -f agents/renderer/Dockerfile -t anyplot-renderer:dev .`; the local backend mounts the harness from your checkout. + - `AGENT_RENDERER=fake` returns fixture PNGs and runs no code. ### Try the agents in ADK's development UI @@ -131,8 +138,59 @@ The service needs the header `X-Anyplot-User` on every `/v1` route; outside `ENV At the defaults only one run may start a minute, so a second "Create plot" within a minute waits in the run queue and the stream shows `status` events with `step: "queued"`. To iterate faster on your own machine, export `AGENT_RUNS_PER_MINUTE=60`. The theme toggle (`POST /v1/sessions/{sid}/versions/{version}/render {"theme": "dark"}`) never waits in the queue. +### Render through the deployed renderer + +`adk web` and the local `/v1` service can render through the deployed `anyplot-renderer`, the same sandboxes the deployed agents service uses. The renderer accepts a token only when Cloud Run IAM lets it through (`roles/run.invoker` on the service) and its claims pass the renderer's own check: the token's `aud` in `RENDERER_AUDIENCES` and its `email` in `RENDERER_ALLOWED_CALLERS`. + +1. Make sure your account has `roles/run.invoker` on `anyplot-renderer` and is listed in its `RENDERER_ALLOWED_CALLERS`, and that the OAuth client id of your token (step 3) is in its `RENDERER_AUDIENCES` (owner task 14 in the design doc). + +2. Point the backend at the renderer's service URL. Cloud Run accepts a service account's token only for that URL, never for a tag URL such as the deploy's `candidate`: + + ```bash + export AGENT_RENDERER=remote \ + AGENT_RENDER_URL=$(gcloud run services describe anyplot-renderer --region=europe-west4 --format='value(status.url)') + ``` + +3. Give the backend a token. Either export one from the gcloud CLI, which spike S proved against Cloud Run IAM; it is valid for one hour, and the backend refuses an expired one with a hint: + + ```bash + export AGENT_RENDER_TOKEN=$(gcloud auth print-identity-token) + ``` + + Or leave `AGENT_RENDER_TOKEN` unset: in development the backend then refreshes your Application Default Credentials (`gcloud auth application-default login`) and sends their ID token, renewing it before it expires. With a service-account key or an impersonated service account in `GOOGLE_APPLICATION_CREDENTIALS`, it mints a token for the renderer URL instead. + + A user's ID token cannot name an audience, so its `aud` is the OAuth client id of the tool that minted it: `32555940559.apps.googleusercontent.com` for `gcloud auth print-identity-token`, and `764086051850-6qr4p6gpi6hn506pt8ejuq83di341hur.apps.googleusercontent.com` for the Application Default Credentials of `gcloud auth application-default login`. The bootstrap lists both in the deploy's `_AUDIENCES`. Whether Cloud Run IAM accepts the Application Default Credentials token was not verified before the first deploy; the gcloud token is the proven path. + +4. Start `adk web` or the service as above. A render that the renderer refuses ends the pipeline run with `failed`, reason `error`, and the backend logs the cause, such as `remote renderer: the renderer refused this caller (HTTP 403, forbidden)`. + +The backend rides out a cold start: it retries a connection error or a Cloud Run front-end 429 or 5xx after 1, 3, 8 and 15 seconds, within 30 seconds, because spike S saw a renderer scaled to zero answer only after about 17 seconds. It never retries the renderer's own refusals, such as `503 busy` or `500 internal`. When you stop a run, or an answer does not come in time, the backend also posts `/render/{job_id}/cancel`, so the renderer stops the sandbox and frees its only render slot. + +Never set `AGENT_RENDER_TOKEN` on a deployed service: the settings refuse it outside `ENVIRONMENT=development`, because a deployed service mints its token from the metadata server. + +### Run the renderer service + +The renderer runs only on Cloud Run, because `sandbox do` exists only there. On your own machine the service starts, answers `GET /status` with `"sandbox": false`, and refuses every render with `503 sandbox_unavailable`: + +```bash +ENVIRONMENT=development uv run uvicorn agents.renderer.main:app --port 8003 +``` + +Its settings are `RENDERER_*` variables, documented in `renderer/settings.py`. The defaults are the deployed limits: a 64 MiB byte budget and 10,000 files per run, a 30-second wait for the one render slot, a 1,024 MiB `MemAvailable` floor before a render and a 512 MiB floor during one (below it the watchdog kills the sandbox), and the rlimits 60 s CPU, 50 MiB per file, 1 GiB address space and 64 processes. + +On Cloud Run the service also checks itself: it refuses every render with `503 volume_missing` when `/tmp/runs` is not a mount of its own of at most 1 GiB (the deploy's in-memory volume), and it ends its own process when a sandbox launcher outlives its kill, so Cloud Run starts a fresh instance instead of answering `503 stuck` until the instance goes idle. `GET /status` reports both. A caller that goes away, or a `POST /render/{job_id}/cancel`, stops the render in progress. `job_id` and the requested themes are an idempotency key: a replayed `POST /render` with the same payload joins the render in flight or gets the stored answer of a finished one, and another payload under the same key is refused with `409 job_conflict`, so the `remote` backend's retries never run a job's code twice. + +The image builds from the repository root: + +```bash +docker build -f agents/renderer/Dockerfile -t anyplot-renderer:dev . +``` + +`renderer/cloudbuild.yaml` builds, deploys a `candidate` revision without traffic, smokes it (IAM refuses an anonymous caller; with `_SMOKE_RENDER=true`, `/status` reports the launcher and the volume and a real matplotlib render goes through the candidate, with a token minted for the service URL), and promotes it. The first build also creates the service; the deploy adds the service URL to `RENDERER_AUDIENCES` itself, and `_MIN_INSTANCES=1` keeps an instance warm. CI builds the same image before merge (`.github/workflows/ci-image.yml`). The deploy flags and the owner tasks are in [Agent network design](../docs/concepts/agent-network.md#serving-and-infrastructure). + ### Test ```bash uv run pytest tests/unit/agents -q ``` + +`tests/unit/agents/renderer/` runs the executor against a fake `sandbox` launcher that runs the real harness with this interpreter, the service in process, and the `remote` backend against the service through httpx's ASGI transport. diff --git a/agents/anyplot/render/__init__.py b/agents/anyplot/render/__init__.py index 6a83cafaa3..50f6dbc9b2 100644 --- a/agents/anyplot/render/__init__.py +++ b/agents/anyplot/render/__init__.py @@ -6,18 +6,22 @@ from ..settings import FAKE_RENDERER_ENVIRONMENTS, AgentSettings from .backends.fake import FakeBackend from .backends.local import LocalDockerBackend +from .backends.remote import IdTokenSource, RemoteBackend from .backends.sandbox import SandboxBackend from .contract import RenderBackend, RendererUnavailable from .runtimes.python import PythonRuntime def make_backend(settings: AgentSettings) -> RenderBackend: - """The render backend `AGENT_RENDERER` names; `remote` arrives with the phase-2 renderer split. + """The render backend `AGENT_RENDERER` names. - The settings already refuse `fake` and `local` outside their environments; the - checks here are the second line for a settings object built without validation. - The backend holds no semaphore: `Services.backend` wraps it in `SerialRenderer` - (`serial.py`), the one place `AGENT_RENDER_CONCURRENCY` is enforced. + Phase 1 renders through `remote`, the anyplot-renderer service. The `sandbox` + backend (sandboxes inside the agents service) stays as a stub for that in-process + variant and is not used in phase 1. The settings already refuse `fake` and + `local` outside their environments; the checks here are the second line for a + settings object built without validation. The backend holds no semaphore: + `Services.backend` wraps it in `SerialRenderer` (`serial.py`), the one place + `AGENT_RENDER_CONCURRENCY` is enforced. """ runtime = PythonRuntime(cpu_seconds=settings.render_timeout_s) if settings.renderer == "fake": @@ -30,4 +34,8 @@ def make_backend(settings: AgentSettings) -> RenderBackend: return LocalDockerBackend(image=settings.render_image, runtime=runtime, environment=settings.environment) if settings.renderer == "sandbox": return SandboxBackend(runtime=runtime) - raise RendererUnavailable("AGENT_RENDERER=remote is the phase-2 renderer split and is not built yet") + if not settings.render_url: + raise RendererUnavailable("AGENT_RENDERER=remote needs AGENT_RENDER_URL, the anyplot-renderer service URL") + static_token = settings.render_token.get_secret_value() if settings.render_token is not None else None + tokens = IdTokenSource(settings.render_url, development=settings.is_development, static_token=static_token) + return RemoteBackend(url=settings.render_url, tokens=tokens) diff --git a/agents/anyplot/render/backends/__init__.py b/agents/anyplot/render/backends/__init__.py index 8d8cbe154a..8897aa1e1a 100644 --- a/agents/anyplot/render/backends/__init__.py +++ b/agents/anyplot/render/backends/__init__.py @@ -1 +1 @@ -"""Render backends: `sandbox` (Cloud Run, phase 1), `local` (Docker, development only) and `fake` (tests).""" +"""Render backends: `remote` (the anyplot-renderer service, phase 1), `sandbox` (in-process, unused stub), `local` (Docker, development only) and `fake` (tests).""" diff --git a/agents/anyplot/render/backends/local.py b/agents/anyplot/render/backends/local.py index eb4c68fc6c..8d96281284 100644 --- a/agents/anyplot/render/backends/local.py +++ b/agents/anyplot/render/backends/local.py @@ -8,7 +8,8 @@ -e ANYPLOT_THEME= -e MPLBACKEND=Agg ... /app/.venv/bin/python -I /opt/anyplot/harness.py plot.py started with `asyncio.create_subprocess_exec` (never a shell) in its own process -group. The image is `AGENT_RENDER_IMAGE`, the production agents image. The harness +group. The image is `AGENT_RENDER_IMAGE`, the production renderer image built from +`agents/renderer/Dockerfile` (it has the plotting venv at `/app/.venv`). The harness is mounted read-only from this checkout, so a render uses the harness of the code under development. On timeout the container is killed by name. The run directory is a fresh temporary directory that is removed afterwards. stdout is discarded and stderr diff --git a/agents/anyplot/render/backends/remote.py b/agents/anyplot/render/backends/remote.py new file mode 100644 index 0000000000..b59f716717 --- /dev/null +++ b/agents/anyplot/render/backends/remote.py @@ -0,0 +1,355 @@ +"""The remote render backend: the anyplot-renderer Cloud Run service (phase 1). + +`AGENT_RENDERER=remote` posts every job to `AGENT_RENDER_URL/render` and turns the +answer back into a `RenderResult`; the host gates (`gates.py`) judge it exactly as +they judge any other backend's output. The JSON contract is `agents/renderer/wire.py`, +which both sides read (it has no ADK import, so the renderer image needs no ADK). +Because the renderer is its own service, a developer's `adk web` and the deployed +agents service render through the same sandboxes. + +**ID token.** The renderer runs with `--no-allow-unauthenticated`, so every request +carries a Google-signed ID token, minted by `IdTokenSource`: + +* Deployed (any environment but `development`): + `google.oauth2.id_token.fetch_id_token_credentials(AGENT_RENDER_URL)`, which on + Cloud Run asks the metadata server for a token whose `aud` is the renderer URL and + whose `email` is the agents service account (a service-account or + impersonated-service-account file in `GOOGLE_APPLICATION_CREDENTIALS` works the same). + `AGENT_RENDER_URL` must be the service URL (`status.url`): Cloud Run accepts no + other audience, not even a tag URL. +* Development: `AGENT_RENDER_TOKEN` when set (a token from + `gcloud auth print-identity-token`, valid for one hour); otherwise Application + Default Credentials. A user's ADC (`gcloud auth application-default login`) cannot + mint a token for an audience, so the backend refreshes them and sends their + `id_token`, whose `aud` is the ADC OAuth client id; the renderer accepts it once + that id is in `RENDERER_AUDIENCES` and the user's email in + `RENDERER_ALLOWED_CALLERS`. Other ADC (a service-account key in + `GOOGLE_APPLICATION_CREDENTIALS`) go through `fetch_id_token_credentials`. + +Tokens are cached and refreshed five minutes before they expire; minting runs in a +worker thread, because google-auth blocks. + +**Failures.** A connection error, or a Cloud Run front-end answer of 429 or 5xx +without the renderer's own JSON `detail`, is retried after 1, 3, 8 and 15 seconds, +within `RETRY_BUDGET_S`: spike S saw a 500, then a 429 ten seconds later, and a 200 +only about 17 s after the first request to a service scaled to zero. The renderer's +own refusals (`503 busy`, `500 internal`, ...) carry a JSON `detail` and are never +retried. Everything else maps to `RendererUnavailable` with a message that names a +status, a fixed error code or an exception class, never a response body; the +message is also logged, so a refused caller or a wrong audience shows in the log. +A theme the renderer did not answer is left out of the result, so R1 reports it; a +theme whose sandbox launcher failed twice (`launcher`) is `RendererUnavailable`, +because it is the renderer's failure and not the code's; the other reasons +(`disk_budget`, `file_budget`, `memory`, `output_rejected`) go to the gates with the +measured value and the limit. + +**Cancellation.** When this render is cancelled (an abort, the request deadline) or +its answer does not come in time, the backend also posts +`/render/{job_id}/cancel`, because a front end that does not pass the closed +connection on would leave the sandbox running and the renderer's only slot taken. + +No semaphore here: `SerialRenderer` (`render/serial.py`) already hands this backend +one theme at a time, and the renderer keeps its own single render slot for the +callers it serves together. +""" + +import asyncio +import base64 +import binascii +import json +import logging +import os +import re +import time +from collections.abc import Awaitable, Callable, Sequence +from typing import Protocol + +import google.auth +import google.auth.exceptions +import google.oauth2.credentials +import httpx +from google.auth.transport.requests import Request as GoogleAuthRequest +from google.oauth2 import id_token as google_id_token +from pydantic import ValidationError + +from agents.renderer.wire import MAX_PNG_BYTES, RenderRequest, RenderResponse + +from ..contract import RendererUnavailable, RenderJob, RenderResult, ThemeOutput + + +logger = logging.getLogger(__name__) + +TOKEN_REFRESH_MARGIN_S = 300.0 +TOKEN_FALLBACK_LIFETIME_S = 3000.0 +"""Used when a token carries no readable `exp` claim; Google ID tokens live an hour.""" +CONNECT_TIMEOUT_S = 10.0 +RESPONSE_MARGIN_S = 45.0 +"""On top of the job's own time per theme: the renderer's slot wait (`RENDERER_SLOT_WAIT_S`, +30 s), the sandbox start and delete, and the transfer of up to 10 MiB of PNG.""" +RETRY_DELAYS_S = (1.0, 3.0, 8.0, 15.0) +"""Pauses before the retries of a connection error or a front-end 429/5xx (spike S's cold start).""" +RETRY_BUDGET_S = 30.0 +"""No retry starts later than this after the first request.""" +CANCEL_TIMEOUT_S = 5.0 +FRONT_END_RETRY = frozenset({429, 500, 502, 503, 504}) +_DETAIL = re.compile(r"^[a-z_]{1,32}$") +_MAX_PNG_BASE64 = 4 * ((MAX_PNG_BYTES + 2) // 3) + + +class TokenSource(Protocol): + """Where the backend gets the bearer token of the next request.""" + + async def token(self) -> str: ... + + +def _unavailable(message: str) -> RendererUnavailable: + """The error to raise, logged first: every message here is built from statuses, codes and class names.""" + logger.warning("remote renderer: %s", message) + return RendererUnavailable(message) + + +def token_expiry(token: str) -> float | None: + """The `exp` claim of a JWT, read without verifying it (the renderer side relies on Cloud Run IAM).""" + parts = token.split(".") + if len(parts) != 3: + return None + try: + claims = json.loads(base64.urlsafe_b64decode(parts[1] + "=" * (-len(parts[1]) % 4))) + except (ValueError, binascii.Error): + return None + expiry = claims.get("exp") if isinstance(claims, dict) else None + return float(expiry) if isinstance(expiry, int | float) else None + + +class IdTokenSource: + """Mints, caches and refreshes the ID token for `audience` (see the module docstring for the order).""" + + def __init__( + self, + audience: str, + *, + development: bool, + static_token: str | None = None, + clock: Callable[[], float] = time.time, + ) -> None: + self.audience = audience + self.development = development + self.static_token = static_token + self.clock = clock + self._token: str | None = None + self._expires = 0.0 + self._lock = asyncio.Lock() + + async def token(self) -> str: + if self.static_token is not None: + expiry = token_expiry(self.static_token) + if expiry is not None and expiry <= self.clock(): + raise _unavailable( + "AGENT_RENDER_TOKEN has expired; export a fresh one from `gcloud auth print-identity-token`" + ) + return self.static_token + async with self._lock: + if self._token is None or self._expires - self.clock() < TOKEN_REFRESH_MARGIN_S: + try: + token = await asyncio.to_thread(self._mint) + except google.auth.exceptions.GoogleAuthError as exc: + raise _unavailable( + f"no ID token for the renderer ({type(exc).__name__}); on a developer machine run " + "`gcloud auth application-default login` or set AGENT_RENDER_TOKEN" + ) from None + self._token = token + self._expires = token_expiry(token) or self.clock() + TOKEN_FALLBACK_LIFETIME_S + return self._token + + def _mint(self) -> str: + """Blocking: a fresh ID token.""" + request = GoogleAuthRequest() + if self.development and not os.environ.get("GOOGLE_APPLICATION_CREDENTIALS"): + credentials, _ = google.auth.default() + if isinstance(credentials, google.oauth2.credentials.Credentials): + credentials.refresh(request) + user_token: str | None = credentials.id_token + if not user_token: + raise _unavailable( + "the Application Default Credentials carry no ID token; run " + "`gcloud auth application-default login` again or set AGENT_RENDER_TOKEN" + ) + return user_token + id_credentials = google_id_token.fetch_id_token_credentials(self.audience, request=request) + id_credentials.refresh(request) + token: str = id_credentials.token + return token + + +def _detail(response: httpx.Response) -> str | None: + """The renderer's own error code (`{"detail": "busy"}`); None for any other body.""" + if not response.headers.get("content-type", "").startswith("application/json"): + return None + try: + body = response.json() + except ValueError: + return None + detail = body.get("detail") if isinstance(body, dict) else None + return detail if isinstance(detail, str) and _DETAIL.fullmatch(detail) else None + + +def _png(value: str | None) -> bytes | None: + if value is None or len(value) > _MAX_PNG_BASE64: + return None + try: + return base64.b64decode(value, validate=True) + except (ValueError, binascii.Error): + return None + + +class RemoteBackend: + """Renders through the anyplot-renderer service.""" + + name = "remote" + + def __init__( + self, + *, + url: str, + tokens: TokenSource, + transport: httpx.AsyncBaseTransport | None = None, + retry_delays_s: Sequence[float] = RETRY_DELAYS_S, + retry_budget_s: float = RETRY_BUDGET_S, + clock: Callable[[], float] = time.monotonic, + sleep: Callable[[float], Awaitable[None]] = asyncio.sleep, + ) -> None: + self.url = url.rstrip("/") + self.tokens = tokens + self.retry_delays_s = tuple(retry_delays_s) + self.retry_budget_s = retry_budget_s + self.clock = clock + self.sleep = sleep + self._client = httpx.AsyncClient(transport=transport, follow_redirects=False) + self._cancels: set[asyncio.Task[None]] = set() + + async def aclose(self) -> None: + if self._cancels: + await asyncio.wait(self._cancels, timeout=CANCEL_TIMEOUT_S) + await self._client.aclose() + + async def render(self, job: RenderJob) -> RenderResult: + try: + body = RenderRequest.model_validate( + { + "job_id": job.job_id, + "language": job.language, + "library": job.library, + "source": job.source, + "data_csv": job.data_csv, + "themes": list(job.themes), + "timeout_s": job.timeout_s, + } + ).model_dump(mode="json") + except ValidationError as exc: + fields = ",".join(sorted({str(error["loc"][0]) for error in exc.errors() if error["loc"]})) + raise _unavailable(f"the job does not fit the renderer contract ({fields})") from None + timeout = httpx.Timeout(job.timeout_s * len(job.themes) + RESPONSE_MARGIN_S, connect=CONNECT_TIMEOUT_S) + try: + response = await self._post(body, timeout) + except asyncio.CancelledError: + await self._cancel_remote(job.job_id) + raise + except httpx.TimeoutException as exc: # the answer did not come in time; ConnectTimeout is retried in _post + await self._cancel_remote(job.job_id) + raise _unavailable(f"the renderer did not answer in time ({type(exc).__name__})") from None + return self._result(job, response) + + async def _post(self, body: dict[str, object], timeout: httpx.Timeout) -> httpx.Response: + """POST the job, retrying connection errors and front-end 429/5xx with backoff inside the budget.""" + delays = iter(self.retry_delays_s) + started = self.clock() + while True: + headers = {"Authorization": f"Bearer {await self.tokens.token()}"} + response: httpx.Response | None = None + try: + response = await self._client.post(f"{self.url}/render", json=body, headers=headers, timeout=timeout) + except (httpx.ConnectError, httpx.ConnectTimeout, httpx.RemoteProtocolError) as exc: + failure = f"could not be reached ({type(exc).__name__})" + except httpx.TimeoutException: + raise + except httpx.HTTPError as exc: + raise _unavailable(f"the render request failed ({type(exc).__name__})") from None + else: + if response.status_code not in FRONT_END_RETRY or _detail(response) is not None: + return response + failure = f"front end answered HTTP {response.status_code}" + delay = next(delays, None) + if delay is None or self.clock() - started + delay > self.retry_budget_s: + if response is not None: + return response # `_result` names the status + raise _unavailable(f"the renderer {failure}") + logger.warning("remote renderer %s; retrying in %.0f s", failure, delay) + await self.sleep(delay) + + async def _cancel_remote(self, job_id: str) -> None: + """Best effort, shielded: ask the renderer to stop the job's sandbox and free its slot.""" + task = asyncio.create_task(self._send_cancel(job_id)) + self._cancels.add(task) + task.add_done_callback(self._cancels.discard) + await asyncio.shield(task) + + async def _send_cancel(self, job_id: str) -> None: + try: + async with asyncio.timeout(CANCEL_TIMEOUT_S): + headers = {"Authorization": f"Bearer {await self.tokens.token()}"} + response = await self._client.post( + f"{self.url}/render/{job_id}/cancel", headers=headers, timeout=CANCEL_TIMEOUT_S + ) + logger.info("remote render %s: cancel sent (HTTP %s)", job_id, response.status_code) + except (TimeoutError, httpx.HTTPError, RendererUnavailable) as exc: + logger.warning("remote render %s: the cancel did not reach the renderer (%s)", job_id, type(exc).__name__) + + def _result(self, job: RenderJob, response: httpx.Response) -> RenderResult: + status = response.status_code + if status != 200: + detail = _detail(response) + suffix = f", {detail}" if detail else "" + if status in (401, 403): + raise _unavailable( + f"the renderer refused this caller (HTTP {status}{suffix}): check roles/run.invoker, " + "AGENT_RENDER_URL (the service URL), RENDERER_AUDIENCES and RENDERER_ALLOWED_CALLERS" + ) + raise _unavailable(f"the renderer answered HTTP {status}{suffix}") + try: + answer = RenderResponse.model_validate_json(response.content) + except ValidationError: + raise _unavailable("the renderer's answer does not match the contract") from None + if answer.job_id != job.job_id: + raise _unavailable("the renderer answered for another job") + result = RenderResult(job_id=job.job_id) + for theme in job.themes: + run = answer.outputs.get(theme) + if run is None: + continue # R1 reports a theme without an output + logger.info( + "remote render %s (%s): exit %s, reason %s, attempts %s, %.2f s", + job.job_id, + theme, + run.exit_code, + run.reason, + run.attempts, + run.wall_s, + ) + if run.reason == "launcher": + # The renderer retries a launcher failure once when the job's time + # allows it, so this is one failure or two: `attempts` says which. + times = "twice" if run.attempts > 1 else "once, with no time left for a retry" + raise _unavailable(f"the renderer's sandbox launcher failed {times} ({theme})") + result.outputs[theme] = ThemeOutput( + theme=theme, + exit_code=run.exit_code, + timed_out=run.timed_out, + png=_png(run.png_base64), + probe=run.probe, + stderr_tail=run.stderr_tail, + wall_s=run.wall_s, + reason=run.reason, + measured=run.measured, + limit=run.limit, + ) + return result diff --git a/agents/anyplot/render/backends/sandbox.py b/agents/anyplot/render/backends/sandbox.py index f63424bba0..7646e864ec 100644 --- a/agents/anyplot/render/backends/sandbox.py +++ b/agents/anyplot/render/backends/sandbox.py @@ -1,34 +1,28 @@ -"""The Cloud Run sandbox render backend (phase 1 production): a typed stub until spike S reports. - -The design runs each theme as - - /usr/local/gcp/bin/sandbox do --sandbox-name r-- --write - --mount type=bind,source=/tmp/runs/,destination=/work -w /work - --env ANYPLOT_THEME= --env MPLBACKEND=Agg --env MPLCONFIGDIR=/opt/mplconfig --env HOME=/tmp - -- /app/.venv/bin/python -I /opt/anyplot/harness.py plot.py - -(docs/concepts/agent-network.md, "Render"). Spike S measures the real command shape, -the timing, the memory per sandbox and the probe suite in europe-west4; until then -this backend refuses to run, so `AGENT_RENDERER=sandbox` fails loudly instead of -guessing a command line. Use `local` (Docker) or `fake` in development. - -Like the local backend, the implementation must stop its sandboxes when the render -is cancelled (an abort or the request deadline): catch `asyncio.CancelledError`, -run `sandbox delete` for each theme under `asyncio.shield`, then re-raise. It needs -no semaphore of its own: `SerialRenderer` (`render/serial.py`), which -`Services.backend` puts in front of every backend, already hands it one theme at a -time under `AGENT_RENDER_CONCURRENCY` slots. +"""The in-process sandbox render backend: a typed stub that phase 1 does not use. + +Phase 1 renders through the anyplot-renderer service (`AGENT_RENDERER=remote`, +`backends/remote.py`): the owner decided on 2026-10-10 that the sandbox launcher +runs in a service of its own, with a zero-role service account, so a local +`adk web` and the deployed agents service share one renderer. The command shape, +the kill path, the byte watchdog and the retry that spikes S and S2 settled live in +`agents/renderer/executor.py`. + +This backend stays for the variant that launches sandboxes inside the agents +service itself. It refuses to run, so `AGENT_RENDERER=sandbox` fails loudly instead +of guessing; an implementation would reuse the renderer's executor rather than copy +it. It needs no semaphore of its own: `SerialRenderer` (`render/serial.py`), which +`Services.backend` puts in front of every backend, hands it one theme at a time. """ from ..contract import RenderJob, RenderResult, RuntimeAdapter SANDBOX_BINARY = "/usr/local/gcp/bin/sandbox" -FORBIDDEN_FLAGS = ("--allow-egress",) +FORBIDDEN_FLAGS = ("--allow-egress", "--write") class SandboxBackend: - """Cloud Run `sandbox do` per theme; not implemented before spike S.""" + """Cloud Run `sandbox do` inside the agents service; not used in phase 1.""" name = "sandbox" @@ -37,6 +31,6 @@ def __init__(self, *, runtime: RuntimeAdapter) -> None: async def render(self, job: RenderJob) -> RenderResult: raise NotImplementedError( - "the sandbox renderer waits for spike S (Cloud Run sandbox command shape, see " - "docs/concepts/agent-network.md, 'Phase 0'); use AGENT_RENDERER=local or fake" + "the in-process sandbox backend is not used in phase 1: renders go through the anyplot-renderer " + "service (AGENT_RENDERER=remote, AGENT_RENDER_URL); use local or fake in development" ) diff --git a/agents/anyplot/render/contract.py b/agents/anyplot/render/contract.py index 1ea8b4b99a..ad0fa62797 100644 --- a/agents/anyplot/render/contract.py +++ b/agents/anyplot/render/contract.py @@ -66,6 +66,14 @@ class ThemeOutput: probe: dict[str, Any] | None = None stderr_tail: str = "" wall_s: float = 0.0 + reason: str | None = None + """Why a renderer stopped or refused the run when it knows more than the exit code (the + `remote` backend's `RunReason`: `disk_budget`, `file_budget`, `memory`, `output_rejected`, + `timeout`); R1 turns it into a line that names the limit.""" + measured: int | None = None + """The value that broke the limit (bytes, entries or MiB of `MemAvailable`, by `reason`).""" + limit: int | None = None + """The limit `measured` broke, in the same unit.""" def __post_init__(self) -> None: self.stderr_tail = self.stderr_tail[-MAX_STDERR_CHARS:] diff --git a/agents/anyplot/render/gates.py b/agents/anyplot/render/gates.py index 6bd7cadc22..64ef642e5f 100644 --- a/agents/anyplot/render/gates.py +++ b/agents/anyplot/render/gates.py @@ -2,7 +2,7 @@ | Gate | Checks | Effect | |---|---|---| -| R1 | an output for each of the job's themes, exit code 0, no timeout, a PNG per theme | blocking: the render is discarded, the error becomes repair feedback | +| R1 | an output for each of the job's themes, exit code 0, no timeout, no renderer limit hit (`limit_line`), a PNG per theme | blocking: the render is discarded, the error becomes repair feedback | | R2 | PNG hardening (`png.harden`): signature, decode, size and pixel caps, not blank, re-encoded | blocking, like R1 | | R3 | canvas within 16 px of 3200x1800 or 2400x2400 (`core.canvas.check_canvas`) | repair-triggering: the VQ-05 defect line goes to the repair; a padded copy of each missed theme is kept as the fallback | | G3 | probe: text boxes beyond the canvas edge | advisory: an AR-09 line | @@ -154,6 +154,49 @@ def error_summary(stderr_tail: str) -> str: return f"{names[-1]} at line {int(frames[-1])}" if frames else names[-1] +def _mib(value: int | None) -> str: + return "more than allowed" if value is None else f"{value / (1024 * 1024):.1f} MiB" + + +def limit_line(theme: Theme, reason: str | None, measured: int | None, limit: int | None) -> str | None: + """The R1 line for a run the renderer stopped or refused at one of its limits, naming the value and the limit. + + None for any other reason: a timeout has its own line, and `launcher` never gets + here (the `remote` backend raises `RendererUnavailable` for it). The numbers come + from the renderer, never from the code under test. + """ + png = f"plot-{theme}.png" + if reason == "disk_budget": + return ( + f"render ({theme}): the code wrote {_mib(measured)} to its working directory, over the {_mib(limit)} " + f"limit, and was stopped → save only {png} and write no other files. Likely cause: data, caches or " + "extra images saved to disk." + ) + if reason == "file_budget": + made = "a directory tree too deep to measure" if measured is None else f"{measured} files and directories" + return ( + f"render ({theme}): the code created {made}, over the limit of {limit}, and was stopped → write no " + f"files besides {png}. Likely cause: a file written per row or per group, or nested output folders." + ) + if reason == "memory": + return ( + f"render ({theme}): the renderer's free memory fell to {measured} MiB, below its {limit} MiB floor, and " + "the code was stopped → use less memory. Likely cause: very many marks, a huge figure or large " + "temporary files." + ) + if reason == "output_rejected": + if measured is not None and limit is not None: + return ( + f"render ({theme}): {png} is {_mib(measured)}, over the {_mib(limit)} limit → save it once at the " + "catalogue canvas size. Likely cause: a high dpi or a very large figure size." + ) + return ( + f'render ({theme}): {png} is not a regular file → save the figure with fig.savefig("{png}"), never a ' + "link or a special file." + ) + return None + + def data_rows(data_csv: str) -> int: """Rows of the canonical data.csv (header excluded; the parser writes one row per line).""" return max(0, len([line for line in data_csv.split("\n") if line]) - 1) @@ -177,6 +220,11 @@ def evaluate(result: RenderResult, *, themes: Sequence[Theme], library: str, row report.blocking.append(_line(f"render ({theme}): the code did not finish within the time limit")) report.passed_host_gates = False continue + stopped = limit_line(theme, output.reason, output.measured, output.limit) + if stopped is not None: + report.blocking.append(_line(stopped)) + report.passed_host_gates = False + continue if output.exit_code != 0 or output.png is None: if output.exit_code == 0: reason = "the code ran but saved no plot-" + theme + ".png" diff --git a/agents/anyplot/render/harness.py b/agents/anyplot/render/harness.py index 26b4cbd860..f8fcdae9e8 100644 --- a/agents/anyplot/render/harness.py +++ b/agents/anyplot/render/harness.py @@ -6,32 +6,51 @@ It is a standalone script with no anyplot import, because it runs inside the sandbox with the isolated interpreter (`-I`: no user site, no current directory on -`sys.path`, no `PYTHON*` environment). In order it: - -1. sets resource limits: CPU seconds (`ANYPLOT_RLIMIT_CPU_S`, default 60), the - largest file it may write (`ANYPLOT_RLIMIT_FSIZE_MB`, default 32) and, when - `ANYPLOT_RLIMIT_AS_MB` is set, the address space (tuned in the sandbox spike); -2. wraps `matplotlib.figure.Figure.savefig` so that every save also writes +`sys.path`, no `PYTHON*` environment). The renderer service +(`agents/renderer/`) and the local Docker backend both run it; the renderer's image +copies it to `/opt/anyplot/harness.py`. In order it: + +1. prints `HARNESS {"event": "start"}` on stdout before anything else, so the host + can tell a harness that ran (and failed in the plot code) from a launcher that + never started it; the renderer retries only the second kind; +2. sets resource limits, soft and hard alike so the code cannot raise them again: + CPU seconds (`ANYPLOT_RLIMIT_CPU_S`, default 60), the largest file it may write + (`ANYPLOT_RLIMIT_FSIZE_MB`, default 32) and, only when set, the address space + (`ANYPLOT_RLIMIT_AS_MB`) and the number of processes (`ANYPLOT_RLIMIT_NPROC`). + The last two are opt-in because outside a sandbox they bind the whole user: + `RLIMIT_NPROC` counts every process of the real user id on a developer machine; +3. when `ANYPLOT_MPL_SEED` names a directory, copies it into `MPLCONFIGDIR`, so + matplotlib starts from the font cache baked into the image instead of + rebuilding it in the sandbox's empty private `/tmp` (spike S measured the copy + as the faster variant); +4. wraps `matplotlib.figure.Figure.savefig` so that every save also writes `probe-.json` next to the plot: the figure size and dpi, the bounding boxes of all visible text in display pixels, tick-label overlaps per axis, annotations outside their axes, and the number of drawn point marks; -3. runs the plot file with `runpy.run_path` as `__main__`. +5. runs the plot file with `runpy.run_path` as `__main__`; +6. prints `HARNESS {"event": "end", "max_rss_mb": ..., "cpu_s": ...}` when the plot + file returns or raises; an exception still propagates, so its traceback reaches + stderr and the exit code stays non-zero. The probe feeds the advisory gates G3 (clipped text), G5 (annotation outside its axes), G7 (tick-label overlap) and G8 (more point marks than data rows). It is advisory because the code under test could tamper with it; the host never fails a -render on it. +render on it. The same holds for the `HARNESS` lines: the code under test shares +stdout, so the host reads them as hints (a retry decision, a memory figure), never +as a verdict. """ import json import os import resource import runpy +import shutil import sys PROBE_LIMIT = 400 # text boxes recorded per figure THEME_ENV = "ANYPLOT_THEME" +MARKER = "HARNESS" def _limit(kind: int, value: int) -> None: @@ -39,7 +58,7 @@ def _limit(kind: int, value: int) -> None: soft, hard = resource.getrlimit(kind) if hard != resource.RLIM_INFINITY: value = min(value, hard) - resource.setrlimit(kind, (value, hard)) + resource.setrlimit(kind, (value, value)) except (ValueError, OSError): pass # a limit the platform refuses is left to the sandbox and the host timeout @@ -52,6 +71,30 @@ def set_limits() -> None: address_space = os.environ.get("ANYPLOT_RLIMIT_AS_MB") if address_space: _limit(resource.RLIMIT_AS, int(address_space) * 1024 * 1024) + processes = os.environ.get("ANYPLOT_RLIMIT_NPROC") + if processes: + _limit(resource.RLIMIT_NPROC, int(processes)) + + +def seed_mplconfig() -> None: + """Copy the baked matplotlib cache into the writable `MPLCONFIGDIR`, before matplotlib is imported.""" + seed = os.environ.get("ANYPLOT_MPL_SEED") + target = os.environ.get("MPLCONFIGDIR") + if not seed or not target or not os.path.isdir(seed): + return + try: + shutil.copytree(seed, target, dirs_exist_ok=True) + except OSError: + pass # matplotlib rebuilds a missing cache itself, only slower + + +def _report(event, **fields): + print(f"{MARKER} {json.dumps({'event': event, **fields})}", flush=True) + + +def _usage(): + usage = resource.getrusage(resource.RUSAGE_SELF) + return {"max_rss_mb": round(usage.ru_maxrss / 1024, 1), "cpu_s": round(usage.ru_utime + usage.ru_stime, 3)} def _box(extent): @@ -150,6 +193,7 @@ def savefig(self, *args, **kwargs): def main(argv): + _report("start") if len(argv) != 2: print("usage: harness.py ", file=sys.stderr) return 2 @@ -158,8 +202,15 @@ def main(argv): print(f"{THEME_ENV} must be light or dark", file=sys.stderr) return 2 set_limits() + seed_mplconfig() install_probe(theme) - runpy.run_path(argv[1], run_name="__main__") + try: + runpy.run_path(argv[1], run_name="__main__") + finally: + try: + _report("end", **_usage()) + except Exception: + pass # a closed stdout must not hide the plot's own exception return 0 diff --git a/agents/anyplot/render/runtimes/python.py b/agents/anyplot/render/runtimes/python.py index 6dc26f9de3..1c67c98d24 100644 --- a/agents/anyplot/render/runtimes/python.py +++ b/agents/anyplot/render/runtimes/python.py @@ -23,6 +23,7 @@ IMAGE_PYTHON = "/app/.venv/bin/python" IMAGE_HARNESS = "/opt/anyplot/harness.py" +MPL_SEED = "/opt/mplconfig" HARNESS_SOURCE = Path(__file__).resolve().parents[1] / "harness.py" @@ -53,6 +54,9 @@ def env(self, theme: Theme) -> dict[str, str]: "ANYPLOT_THEME": theme, "MPLBACKEND": "Agg", "MPLCONFIGDIR": self.mplconfig, + # The renderer image's baked font cache; the harness copies it into + # MPLCONFIGDIR and skips the copy where the directory does not exist. + "ANYPLOT_MPL_SEED": MPL_SEED, "HOME": "/tmp", "PYTHONDONTWRITEBYTECODE": "1", "ANYPLOT_RLIMIT_CPU_S": str(self.cpu_seconds), diff --git a/agents/anyplot/settings.py b/agents/anyplot/settings.py index abb395fc90..2f0503fc89 100644 --- a/agents/anyplot/settings.py +++ b/agents/anyplot/settings.py @@ -14,8 +14,10 @@ | `AGENT_LOCATION` | `eu` | Vertex AI location of every model call: `eu`, `us` or `global` | | `AGENT_PROJECT`, else `GOOGLE_CLOUD_PROJECT` | `anyplot` | Google Cloud project that serves (and bills) the model calls | | `AGENT_LIBRARIES` | `matplotlib,seaborn` | Enabled libraries; each needs a phase-1 runtime (`matplotlib`, `seaborn`) | -| `AGENT_RENDERER` | `sandbox` | Render backend: `sandbox`, `local` (development only), `fake` (development and test only) or `remote` | -| `AGENT_RENDER_IMAGE` | `anyplot-agents:dev` | Image the `local` renderer runs with Docker | +| `AGENT_RENDERER` | `remote` | Render backend: `remote` (the anyplot-renderer service, phase 1), `sandbox` (in-process sandboxes, not used in phase 1), `local` (development only) or `fake` (development and test only) | +| `AGENT_RENDER_URL` | unset | The anyplot-renderer service URL the `remote` backend calls; `https://` outside development and test | +| `AGENT_RENDER_TOKEN` | unset | Development only: an ID token for the renderer (`gcloud auth print-identity-token`); without it the backend mints one from Application Default Credentials | +| `AGENT_RENDER_IMAGE` | `anyplot-renderer:dev` | Image the `local` renderer runs with Docker (build it from `agents/renderer/Dockerfile`) | | `AGENT_RENDER_CONCURRENCY` | `1` | Renders (one theme each) that may run at the same time; serial, because one 4 GiB instance holds one sandbox safely (spikes S and S2) | | `AGENT_RUN_CONCURRENCY` | `1` | Pipeline runs (whole `/messages` turns) in flight per instance; the run queue holds the rest | | `AGENT_RUNS_PER_MINUTE` | `1` | Runs that may start within any 60 seconds (a sliding window) | @@ -31,7 +33,7 @@ | `AGENT_JUDGE_TIMEOUT_S` | `4` | Wall-clock budget of one judge verdict, its single retry included | | `AGENT_SESSION_IDLE_S` | `3600` | Idle seconds after which the sweeper drops a session and its stores | | `AGENT_ALLOWED_CALLERS` | empty | Service-account emails allowed to call `/v1` (comma-separated) | -| `AGENT_SERVICE_URLS` | empty | ID-token audiences accepted on `/v1`: the service URL and the candidate-tag URL (comma-separated) | +| `AGENT_SERVICE_URLS` | empty | ID-token audiences accepted on `/v1` (comma-separated): the service URL, which Cloud Run requires as the audience even for a request to a tag URL | | `AGENT_DEV_FIXTURE` | unset | Development only: an eval fixture case id that seeds every new session | | `ENVIRONMENT` | `production` | Deployment environment, shared with the API; `development` (the caller check off) is refused on Cloud Run | @@ -46,7 +48,7 @@ from functools import lru_cache from typing import Annotated, Any, Literal, Self -from pydantic import AliasChoices, Field, PositiveFloat, PositiveInt, field_validator, model_validator +from pydantic import AliasChoices, Field, PositiveFloat, PositiveInt, SecretStr, field_validator, model_validator from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict from core.constants import SUPPORTED_LIBRARIES @@ -66,8 +68,12 @@ """Model families: Claude through Vertex AI's Anthropic endpoint, or Gemini through Vertex AI.""" Renderer = Literal["sandbox", "local", "fake", "remote"] -"""Render backends: `sandbox` (Cloud Run sandboxes, phase 1), `local` (Docker, -development only), `fake` (unit tests) and `remote` (the phase-2 renderer split).""" +"""Render backends: `remote` (the anyplot-renderer service, phase 1), `sandbox` (Cloud +Run sandboxes inside this service, a stub that phase 1 does not use), `local` (Docker, +development only) and `fake` (unit tests).""" + +RENDER_URL_PATTERN = r"^https?://[A-Za-z0-9.-]{1,253}(:[0-9]{1,5})?/?$" +"""A service origin: scheme, host and optional port, no path, query or credentials.""" LOCATIONS: tuple[str, ...] = ("eu", "us", "global") MODEL_PREFIXES: dict[str, str] = {"anthropic-vertex": "claude-", "gemini": "gemini-"} @@ -118,11 +124,19 @@ class AgentSettings(BaseSettings): libraries: Annotated[list[str], NoDecode] = Field(default=["matplotlib", "seaborn"], min_length=1) """Libraries with an enabled runtime (`AGENT_LIBRARIES`, comma-separated).""" - renderer: Renderer = "sandbox" + renderer: Renderer = "remote" """Render backend (`AGENT_RENDERER`); `local` only in development, `fake` only in development and test.""" - render_image: str = Field(default="anyplot-agents:dev", pattern=r"^[A-Za-z0-9][A-Za-z0-9._/:@-]{0,199}$") - """Image the local renderer runs (`AGENT_RENDER_IMAGE`).""" + render_url: str | None = Field(default=None, pattern=RENDER_URL_PATTERN) + """The anyplot-renderer service URL (`AGENT_RENDER_URL`), also the audience of its ID token. + The `remote` backend refuses to start without it; plain `http://` only in development and test.""" + + render_token: SecretStr | None = None + """Development only (`AGENT_RENDER_TOKEN`): an ID token for the renderer, typically from + `gcloud auth print-identity-token`. Without it the `remote` backend mints one itself.""" + + render_image: str = Field(default="anyplot-renderer:dev", pattern=r"^[A-Za-z0-9][A-Za-z0-9._/:@-]{0,199}$") + """Image the local renderer runs (`AGENT_RENDER_IMAGE`), built from `agents/renderer/Dockerfile`.""" render_concurrency: PositiveInt = 1 """Theme renders that may run at the same time (`AGENT_RENDER_CONCURRENCY`), enforced for @@ -177,7 +191,8 @@ class AgentSettings(BaseSettings): service_urls: Annotated[list[str], NoDecode] = [] """ID-token audiences accepted on `/v1` (`AGENT_SERVICE_URLS`, comma-separated): the - service URL and the candidate-tag URL. Empty means no caller passes outside development.""" + service URL, which Cloud Run requires as `aud` even for a request to a traffic tag's + URL. Empty means no caller passes outside development.""" dev_fixture: str | None = Field(default=None, pattern=FIXTURE_PATTERN) """Eval fixture case that seeds every new session (`AGENT_DEV_FIXTURE`); development only.""" @@ -230,13 +245,18 @@ def _project_id(cls, value: str) -> str: def _parse_list(cls, value: Any) -> Any: return _split_list(value) - @field_validator("dev_fixture", mode="before") + @field_validator("dev_fixture", "render_url", "render_token", mode="before") @classmethod - def _blank_fixture_is_none(cls, value: Any) -> Any: + def _blank_is_none(cls, value: Any) -> Any: if isinstance(value, str) and not value.strip(): return None return value.strip() if isinstance(value, str) else value + @field_validator("render_url") + @classmethod + def _origin_without_slash(cls, value: str | None) -> str | None: + return value.rstrip("/") if value else value + @field_validator("libraries") @classmethod def _known_libraries(cls, value: list[str]) -> list[str]: @@ -295,6 +315,19 @@ def _consistent(self) -> Self: raise ValueError( f"AGENT_DEV_FIXTURE is allowed only when ENVIRONMENT=development, not {self.environment!r}" ) + if ( + self.render_url + and self.render_url.startswith("http://") + and self.environment not in FAKE_RENDERER_ENVIRONMENTS + ): + raise ValueError( + "AGENT_RENDER_URL must be https:// outside development and test: the request carries an ID token" + ) + if self.render_token is not None and not self.is_development: + raise ValueError( + f"AGENT_RENDER_TOKEN is allowed only when ENVIRONMENT=development, not {self.environment!r}: " + "a deployed service mints its own token from the metadata server" + ) if self.soft_deadline_s >= self.request_deadline_s: raise ValueError( f"AGENT_SOFT_DEADLINE_S ({self.soft_deadline_s}) must be below " diff --git a/agents/main.py b/agents/main.py index b1f6195da8..f4275d7b77 100644 --- a/agents/main.py +++ b/agents/main.py @@ -25,7 +25,9 @@ owned by that id, so another user's session id is `404 session_expired`. Outside development the caller check decodes the IAM-forwarded ID token (Cloud Run has already verified it and replaced the signature) and requires its `aud` in -`AGENT_SERVICE_URLS` and its `email` in `AGENT_ALLOWED_CALLERS`. +`AGENT_SERVICE_URLS` and its `email` in `AGENT_ALLOWED_CALLERS`. It reads +`X-Serverless-Authorization` whenever that header is present, because Cloud Run then +checks only that one and passes `Authorization` through unverified. Every `/messages` turn goes through the run queue (`anyplot/run_queue.py`): the route answers `503 capacity` when the queue is full, otherwise the stream sends `ready`, @@ -326,12 +328,22 @@ def decode_claims(authorization: str | None) -> dict[str, Any] | None: return claims if isinstance(claims, dict) else None +def checked_token(request: Request) -> str | None: + """The header Cloud Run IAM verified: `X-Serverless-Authorization` when present, else `Authorization`. + + With both headers present Cloud Run checks only `X-Serverless-Authorization`, so an + invoker could otherwise add an unsigned `Authorization` that names an allowed email. + """ + serverless = request.headers.get("x-serverless-authorization") + return serverless if serverless is not None else request.headers.get("authorization") + + def check_caller(request: Request) -> None: """Outside development: an IAM-forwarded ID token whose `aud` and `email` are allowed.""" settings = get_settings() if settings.is_development: return - claims = decode_claims(request.headers.get("authorization")) + claims = decode_claims(checked_token(request)) if claims is None: raise AgentsError(401, "unauthenticated") audience = claims.get("aud") diff --git a/agents/renderer/Dockerfile b/agents/renderer/Dockerfile new file mode 100644 index 0000000000..fa102f6a36 --- /dev/null +++ b/agents/renderer/Dockerfile @@ -0,0 +1,77 @@ +# anyplot-renderer: the Cloud Run service that runs adapted plot code in sandboxes +# (agents/renderer/, docs/concepts/agent-network.md "Render" and "Serving and +# infrastructure"). +# +# Two jobs in one image. The service process (uvicorn agents.renderer.main:app) +# launches sandboxes through Cloud Run's `sandbox do`, and every sandbox runs +# `/app/.venv/bin/python -I /opt/anyplot/harness.py plot.py` from this same image: +# a sandbox sees the image read-only (spike S), so the venv and the harness here +# are what model-written code runs with. The local Docker backend +# (AGENT_RENDERER=local, development only) runs this image too: +# +# docker build -f agents/renderer/Dockerfile -t anyplot-renderer:dev . +# +# Same two-stage shape as api/Dockerfile (read its comments for the uv pin, the +# bytecode trade and why both stages share one base image). The build CONTEXT is +# the repo root, and the root `.dockerignore` is an allowlist: `agents`, +# `pyproject.toml`, `uv.lock` and `README.md` are in it, so the COPYs below see them. + +FROM python:3.13-slim AS builder + +WORKDIR /app + +RUN pip install --no-cache-dir uv==0.10.9 + +ENV UV_COMPILE_BYTECODE=1 +ENV UV_PYTHON=/usr/local/bin/python3.13 +ENV UV_PYTHON_DOWNLOADS=never + +COPY pyproject.toml uv.lock README.md ./ + +# The base dependencies carry the service (fastapi, uvicorn, pydantic-settings) and +# the two extras carry the phase-1 plotting stack. No ADK: nothing in the renderer +# imports it. `--no-install-project` keeps the project itself out of the venv, so a +# sandbox's interpreter finds no anyplot package on its path; the service imports +# `agents.renderer` from the working directory, which uvicorn puts on `sys.path`. +RUN uv venv && uv sync --frozen --no-install-project --extra lib-matplotlib --extra lib-seaborn + +# The image does not install the project, so its version is written to a file +# that `GET /status` reports. +RUN .venv/bin/python -c "import tomllib; print(tomllib.load(open('pyproject.toml', 'rb'))['project']['version'])" > VERSION + + +FROM python:3.13-slim + +WORKDIR /app + +COPY --from=builder /app/.venv ./.venv + +# Only the renderer package and the package marker of `agents/`: the rest of +# agents/ (the ADK runtime, the prompts) has no business in an image whose files +# every sandbox can read. +COPY agents/__init__.py ./agents/__init__.py +COPY agents/renderer/ ./agents/renderer/ +COPY --from=builder /app/VERSION ./agents/renderer/VERSION +COPY agents/anyplot/render/harness.py /opt/anyplot/harness.py + +# Bake the matplotlib font cache. A sandbox starts with an empty private /tmp, and +# the harness copies this directory into MPLCONFIGDIR=/tmp/mpl before matplotlib +# loads (spike S: faster than rebuilding the cache in every render). +RUN mkdir -p /opt/mplconfig \ + && MPLCONFIGDIR=/opt/mplconfig .venv/bin/python -c "import matplotlib.font_manager as fm; fm.findfont('DejaVu Sans')" \ + && chmod -R a+rX /opt/mplconfig /opt/anyplot + +RUN .venv/bin/python -m compileall -q agents + +ENV PYTHONUNBUFFERED=1 +ENV PORT=8080 +EXPOSE 8080 + +# No `USER`: the service runs as the image default, root. `sandbox do` creates +# gVisor sandboxes (namespaces, mounts, the gofer that serves /work), which the +# launcher only does for root; spikes S and S2 ran the supervisor as root for that +# reason. What runs as root is the supervisor alone, which holds no secret and no +# role (its service account `anyplot-renderer@` has none). The model-written code +# runs inside the sandbox: uid 0 there too, but behind gVisor, with no network, +# no metadata server, a read-only root filesystem and the harness's rlimits. +CMD [".venv/bin/uvicorn", "agents.renderer.main:app", "--host", "0.0.0.0", "--port", "8080"] diff --git a/agents/renderer/auth.py b/agents/renderer/auth.py new file mode 100644 index 0000000000..113f9e77c3 --- /dev/null +++ b/agents/renderer/auth.py @@ -0,0 +1,77 @@ +"""The caller check of the renderer, copied from `agents/main.py` (which cannot be imported: it loads ADK). + +Cloud Run IAM (`--no-allow-unauthenticated`, `roles/run.invoker`) verifies the ID +token before a request reaches the container and forwards it with the signature +replaced (spike S), so the claims are decoded here without verifying a signature. +Outside `ENVIRONMENT=development` the token's `aud` must be in `RENDERER_AUDIENCES` +and its `email` in `RENDERER_ALLOWED_CALLERS`. + +**Which header.** A caller may send its token in `X-Serverless-Authorization` +instead of `Authorization`, and when both are present Cloud Run checks only +`X-Serverless-Authorization` and passes `Authorization` through unchecked. So the +check reads `X-Serverless-Authorization` whenever it is present, and +`Authorization` only when it is not; otherwise any invoker could add a forged, +unsigned `Authorization` that names an allowed email. + +**Audiences.** Cloud Run accepts a token only when its `aud` is the service URL, +even for a request to a traffic tag's URL, so `RENDERER_AUDIENCES` holds the +service URL (the deploy step adds it) and never a tag URL. A user's ID token +cannot name an audience: `gcloud auth print-identity-token` carries gcloud's +OAuth client id as `aud`, and Application Default Credentials carry theirs. For +the owner's `adk web` against the deployed renderer, that client id joins the +list and the owner's email joins `RENDERER_ALLOWED_CALLERS` (agents/README.md). +""" + +import base64 +import binascii +import json +from typing import Annotated, Any + +from fastapi import Depends, Request + +from .settings import RendererSettings, get_settings + + +class CallerRefused(Exception): + """The request carries no acceptable ID token; answered as `{"detail": code}`.""" + + def __init__(self, status: int, code: str) -> None: + super().__init__(code) + self.status = status + self.code = code + + +def decode_claims(authorization: str | None) -> dict[str, Any] | None: + """The claims of a bearer JWT, decoded without verifying the signature (Cloud Run IAM did).""" + if not authorization or not authorization.lower().startswith("bearer "): + return None + parts = authorization[7:].strip().split(".") + if len(parts) != 3: + return None + try: + payload = parts[1] + "=" * (-len(parts[1]) % 4) + claims = json.loads(base64.urlsafe_b64decode(payload.encode())) + except (ValueError, binascii.Error): + return None + return claims if isinstance(claims, dict) else None + + +def checked_token(request: Request) -> str | None: + """The header Cloud Run IAM verified: `X-Serverless-Authorization` when present, else `Authorization`.""" + serverless = request.headers.get("x-serverless-authorization") + return serverless if serverless is not None else request.headers.get("authorization") + + +def check_caller(request: Request, settings: Annotated[RendererSettings, Depends(get_settings)]) -> None: + """Outside development: an IAM-forwarded ID token whose `aud` and `email` are allowed.""" + if settings.is_development: + return + claims = decode_claims(checked_token(request)) + if claims is None: + raise CallerRefused(401, "unauthenticated") + audience = claims.get("aud") + audiences = audience if isinstance(audience, list) else [audience] + if not any(isinstance(item, str) and item in settings.audiences for item in audiences): + raise CallerRefused(403, "forbidden") + if claims.get("email") not in settings.allowed_callers: + raise CallerRefused(403, "forbidden") diff --git a/agents/renderer/cloudbuild.yaml b/agents/renderer/cloudbuild.yaml new file mode 100644 index 0000000000..ff58da954c --- /dev/null +++ b/agents/renderer/cloudbuild.yaml @@ -0,0 +1,243 @@ +# Cloud Build configuration for anyplot-renderer, the sandbox renderer of the +# agent network (agents/renderer/, docs/concepts/agent-network.md "Serving and +# infrastructure"). +# +# Same candidate -> smoke -> promote shape as api/cloudbuild.yaml: the new +# revision is deployed without traffic under the `candidate` tag, smoked on its +# tag URL, and only then given 100 % of the traffic. One difference: the deploy +# step creates the service itself on the very first build. gcloud refuses +# `--no-traffic` for a service that does not exist yet, so when `services +# describe` answers that the service is not found, the step deploys without it; +# there is no traffic to protect before anyone calls the service. Any other +# describe failure stops the build, so a permission or API error can never send +# all traffic to an unsmoked revision. +# +# Run from the repository root (the build context is `.`, filtered by the root +# `.dockerignore`), for example: +# +# gcloud builds submit --config=agents/renderer/cloudbuild.yaml \ +# --gcs-source-staging-dir=gs://anyplot-cloudbuild-eu/source \ +# "--substitutions=^|^_ALLOWED_CALLERS=...|_AUDIENCES=...|_SMOKE_RENDER=true" . +# +# The service account, the IAM bindings and the first build are owner tasks; +# docs/concepts/agent-network.md lists them. +substitutions: + _SERVICE_NAME: anyplot-renderer + _REGION: europe-west4 + _SERVICE_ACCOUNT: anyplot-renderer@anyplot.iam.gserviceaccount.com + # RENDERER_ALLOWED_CALLERS: the emails whose ID token the renderer accepts, + # separated by commas: the anyplot-agents service account, the renderer's own + # service account (the render smoke below calls the candidate as that account) + # and, for `adk web` against the deployed renderer, the owner's account. Empty + # means nobody passes. + _ALLOWED_CALLERS: "" + # RENDERER_AUDIENCES beyond the service URL, separated by commas. The deploy + # step always adds the service's own URL (status.url): Cloud Run accepts no + # other audience for a service account's token, not even a tag URL. Add here + # only the OAuth client ids of the owner's user tokens (agents/README.md). + _AUDIENCES: "" + # "true": the smoke step renders a real plot through the candidate, which + # proves the launcher, the in-memory volume and the harness. It calls the + # candidate as the service's own account (_SERVICE_ACCOUNT), so it needs that + # account in _ALLOWED_CALLERS and with roles/run.invoker on the service, this + # build's account with roles/iam.serviceAccountTokenCreator on it, and the + # service URL in RENDERER_AUDIENCES, so the very first build (which creates + # the service and its URL) runs with "false". + _SMOKE_RENDER: "true" + # "1" keeps an instance warm (spike S: a request to a service scaled to zero + # met a 500 and a 429 for about 17 s); "0" scales to zero between sessions. + _MIN_INSTANCES: "0" + +steps: + - name: "gcr.io/cloud-builders/docker" + args: + - "build" + - "-t" + - "europe-west4-docker.pkg.dev/$PROJECT_ID/anyplot/${_SERVICE_NAME}:$BUILD_ID" + - "-t" + - "europe-west4-docker.pkg.dev/$PROJECT_ID/anyplot/${_SERVICE_NAME}:latest" + - "-f" + - "agents/renderer/Dockerfile" + - "." + id: "build-image" + + - name: "gcr.io/cloud-builders/docker" + args: + - "push" + - "europe-west4-docker.pkg.dev/$PROJECT_ID/anyplot/${_SERVICE_NAME}:$BUILD_ID" + id: "push-image" + waitFor: ["build-image"] + + # Deploy WITHOUT traffic (except on the first build, see the header), tagged + # `candidate`, with a deterministic revision suffix so promote targets exactly + # this build's revision (`b` prefix: the suffix must start with a letter). + # + # The flags, each from spikes S and S2: gen2 plus --sandbox-launcher (a Preview + # feature, hence `gcloud beta`); 2 vCPU and 4 GiB, which serve one sandbox at a + # time safely; one instance, because renders are serial and the run queue of + # the agents service already holds the line; concurrency 4 only so a waiting + # caller (the deployed agents and a developer's adk web) gets the render slot's + # `503 busy` instead of a Cloud Run 429, and so a cancel reaches the instance + # while a render holds the slot; and the run directories on a 512 MiB + # in-memory volume, because a disk fill through /work on the plain filesystem + # crashed the instance at about 3.4 GB (the service refuses to render without + # it: `503 volume_missing`). No secrets, no Cloud SQL, no role. + # + # `--update-env-vars`, never `--set-env-vars`, for the reason api/cloudbuild.yaml + # gives: an out-of-band variable must survive a deploy. + - name: "gcr.io/cloud-builders/gcloud" + entrypoint: "bash" + args: + - "-ceu" + - | + gcloud beta run deploy --help | grep -q -- "--sandbox-launcher" \ + || { echo "this gcloud does not know --sandbox-launcher; use a newer builder image"; exit 1; } + if SELF=$$(gcloud run services describe ${_SERVICE_NAME} --region=${_REGION} --platform=managed --format="value(status.url)" 2>describe.err); then + TRAFFIC=(--no-traffic) + elif grep -qE "Cannot find service|NOT_FOUND" describe.err; then + echo "${_SERVICE_NAME} does not exist yet: creating it (gcloud refuses --no-traffic on a new service)" + SELF="" + TRAFFIC=() + else + cat describe.err + echo "could not read ${_SERVICE_NAME}; refusing to deploy without --no-traffic" + exit 1 + fi + EXTRA="${_AUDIENCES}" + AUDIENCES="$$SELF" + if [ -n "$$EXTRA" ]; then AUDIENCES="$${AUDIENCES:+$$AUDIENCES,}$$EXTRA"; fi + echo "RENDERER_AUDIENCES=$$AUDIENCES" + gcloud beta run deploy ${_SERVICE_NAME} \ + --image=europe-west4-docker.pkg.dev/$PROJECT_ID/anyplot/${_SERVICE_NAME}:$BUILD_ID \ + --region=${_REGION} \ + --platform=managed \ + --execution-environment=gen2 \ + --sandbox-launcher \ + --no-allow-unauthenticated \ + --service-account=${_SERVICE_ACCOUNT} \ + --cpu=2 \ + --memory=4Gi \ + --min-instances=${_MIN_INSTANCES} \ + --max-instances=1 \ + --concurrency=4 \ + --timeout=300 \ + --add-volume=name=runs,type=in-memory,size-limit=512Mi \ + --add-volume-mount=volume=runs,mount-path=/tmp/runs \ + "--update-env-vars=^|^ENVIRONMENT=production|RENDERER_ALLOWED_CALLERS=${_ALLOWED_CALLERS}|RENDERER_AUDIENCES=$$AUDIENCES" \ + "$${TRAFFIC[@]}" \ + --tag=candidate \ + --revision-suffix=b$BUILD_ID + id: "deploy" + waitFor: ["push-image"] + + # Pre-traffic smoke against the CANDIDATE revision on its tag URL. + - name: "gcr.io/cloud-builders/gcloud" + entrypoint: "bash" + args: + - "-ceu" + - | + # The candidate tag must point at THIS build's revision, before and after + # the probes (the same double read as api/cloudbuild.yaml). + candidate() { + gcloud run services describe ${_SERVICE_NAME} --region=${_REGION} --platform=managed --format=json \ + | python3 -c "import json,sys; t=json.load(sys.stdin)['status']['traffic']; c=next(x for x in t if x.get('tag')=='candidate'); print(c['url'], c['revisionName'])" + } + read -r URL REV <<< "$$(candidate)" + test "$$REV" = "${_SERVICE_NAME}-b$BUILD_ID" || { echo "candidate tag moved to $$REV (expected ${_SERVICE_NAME}-b$BUILD_ID)"; exit 1; } + echo "smoke against candidate $$URL ($$REV)" + + # 1. Cloud Run IAM holds: without a token the front end answers 403 and + # the request never reaches the container. A 200 would mean the + # service lost --no-allow-unauthenticated. + code=$$(curl -s --retry 5 --retry-delay 5 -o /dev/null -w '%{http_code}' "$$URL/status") + test "$$code" = "403" || { echo "expected 403 without an ID token, got $$code"; exit 1; } + echo "OK: IAM refuses an anonymous caller" + + # 2. A real render: the launcher, the in-memory volume, the harness and + # the caller check, end to end. The token's audience is the SERVICE + # URL even though the request goes to the tag URL: Cloud Run accepts + # no other audience. The first request after a deploy may meet a cold + # instance, hence the retries. + if [ "${_SMOKE_RENDER}" = "true" ]; then + SERVICE_URL=$$(gcloud run services describe ${_SERVICE_NAME} --region=${_REGION} --platform=managed --format="value(status.url)") + # The smoke calls the candidate as the service's own account. A build + # cannot mint an ID token for its own identity (gcloud refuses + # `--include-email` for it, and Cloud Build's metadata server has no + # `identity` endpoint; builds da6f53f5 and a9d3ac7f on 2026-10-10), so it + # impersonates the service account, which needs + # roles/iam.serviceAccountTokenCreator on that account for the build's + # account, roles/run.invoker on the service for the service account, and + # the service account in _ALLOWED_CALLERS (the bootstrap grants all three). + TOKEN=$$(gcloud auth print-identity-token --impersonate-service-account="${_SERVICE_ACCOUNT}" \ + --audiences="$$SERVICE_URL" --include-email 2>impersonate.err) \ + || { cat impersonate.err; echo "could not mint the smoke token: grant roles/iam.serviceAccountTokenCreator on ${_SERVICE_ACCOUNT} to this build's account"; exit 1; } + curl -fsS --retry 5 --retry-delay 5 --retry-all-errors -H "Authorization: Bearer $$TOKEN" -o status.json "$$URL/status" \ + || { echo "/status failed with this build's token: check _ALLOWED_CALLERS, roles/run.invoker and RENDERER_AUDIENCES ($$SERVICE_URL)"; exit 1; } + python3 -c "import json; s=json.load(open('status.json')); assert s['name']=='anyplot-renderer' and s['sandbox'] and s['volume'] is True, s; print('OK: /status', s['version'], s['mem_available_mb'], 'MiB available, volume in place')" + python3 - <<'EOF' + import json + source = "\n".join([ + "import matplotlib.pyplot as plt", + "fig, ax = plt.subplots(figsize=(16, 9), dpi=200)", + "ax.plot([1, 2, 3, 4], [3, 1, 4, 2], linewidth=6)", + "fig.savefig('plot-light.png')", + ]) + job = {"job_id": "smoke", "language": "python", "library": "matplotlib", "source": source, + "data_csv": "x\n1\n", "themes": ["light"], "timeout_s": 60} + json.dump(job, open("job.json", "w")) + EOF + curl -fsS --retry 3 --retry-delay 10 --retry-all-errors --max-time 200 -H "Authorization: Bearer $$TOKEN" \ + -H "Content-Type: application/json" --data @job.json -o render.json "$$URL/render" \ + || { echo "the smoke render request failed"; exit 1; } + python3 -c "import json; r=json.load(open('render.json')); o=r['outputs']['light']; assert o['exit_code']==0 and o['png_base64'], {k: o[k] for k in ('exit_code','reason','attempts','wall_s')}; print('OK: smoke render', o['wall_s'], 's,', o['attempts'], 'attempt(s)')" + else + echo "render smoke skipped (_SMOKE_RENDER=${_SMOKE_RENDER})" + fi + + read -r _ REV_AFTER <<< "$$(candidate)" + test "$$REV_AFTER" = "$$REV" || { echo "candidate tag moved to $$REV_AFTER mid-smoke (started on $$REV) - not promoting"; exit 1; } + echo "smoke OK" + id: "smoke" + waitFor: ["deploy"] + + # Route 100 % of the traffic to EXACTLY the revision this build smoked. + - name: "gcr.io/cloud-builders/gcloud" + args: + - "run" + - "services" + - "update-traffic" + - "${_SERVICE_NAME}" + - "--region=${_REGION}" + - "--platform=managed" + - "--to-revisions=${_SERVICE_NAME}-b$BUILD_ID=100" + id: "promote" + waitFor: ["smoke"] + + # `:latest` moves only after this build promoted (see api/cloudbuild.yaml). + - name: "gcr.io/cloud-builders/docker" + args: + - "push" + - "europe-west4-docker.pkg.dev/$PROJECT_ID/anyplot/${_SERVICE_NAME}:latest" + id: "push-latest" + waitFor: ["promote"] + + - name: "gcr.io/cloud-builders/gcloud" + args: + - "run" + - "services" + - "describe" + - "${_SERVICE_NAME}" + - "--region=${_REGION}" + - "--platform=managed" + - "--format=value(status.url)" + id: "get-url" + waitFor: ["promote"] + +images: + - "europe-west4-docker.pkg.dev/$PROJECT_ID/anyplot/${_SERVICE_NAME}:$BUILD_ID" + - "europe-west4-docker.pkg.dev/$PROJECT_ID/anyplot/${_SERVICE_NAME}:latest" + +options: + logging: CLOUD_LOGGING_ONLY + +timeout: "1200s" diff --git a/agents/renderer/executor.py b/agents/renderer/executor.py new file mode 100644 index 0000000000..0dd4525b69 --- /dev/null +++ b/agents/renderer/executor.py @@ -0,0 +1,697 @@ +"""The sandbox executor: one `sandbox do` per theme, with a kill path, a run watchdog and one retry. + +Every theme runs in a sandbox of its own, in a run directory of its own: + + /usr/local/gcp/bin/sandbox do --sandbox-name r-- + --mount type=bind,source=/tmp/runs/r--/work,destination=/work -w /work + --env PATH=/app/.venv/bin:/usr/local/bin:/usr/bin:/bin --env HOME=/tmp --env ANYPLOT_THEME= ... + -- /app/.venv/bin/python -I /opt/anyplot/harness.py plot.py + +started with `asyncio.create_subprocess_exec` (never a shell) in a session of its own. +Each rule below comes from spike S or S2 (docs/concepts/agent-network.md, "Render"): + +* **No `--write`, no `--allow-egress`.** `--write` makes the sandbox's root filesystem + writable, and with it a nested egress attempt got one step further (S2). `build_argv` + asserts that neither flag is ever present. +* **A complete environment.** A sandbox sees only `HOME`, `LC_CTYPE` and the `--env` + values, with no `PATH`, so `PATH` is passed explicitly. +* **The run directory under `/tmp/runs/`.** `/work` is the only host directory a + sandbox can write; the image and every host file outside `/tmp` are readable from + inside, while sibling run directories are not (spike G). On Cloud Run `/tmp/runs` + is a size-limited in-memory volume, because a disk fill through `/work` crashed the + whole instance at about 3.4 GB (S2); `runs_volume_ok` lets the service refuse to + render without it. A sandbox's private `/tmp` is a tmpfs of its own, bounded only + by the memory floor below (or bound into the run directory with `RENDERER_BIND_TMP`). +* **A run watchdog.** Every `RENDERER_WATCH_INTERVAL_S` the host measures the live run + directory and kills the sandbox once it holds more than `RENDERER_RUN_BUDGET_MB` + (reason `disk_budget`), more than `MAX_RUN_ENTRIES` entries, or a tree it cannot + measure: deeper than `MAX_RUN_DEPTH` or past `PATH_MAX` (`file_budget`). It also + reads `MemAvailable` and kills the sandbox below `RENDERER_KILL_MEM_AVAILABLE_MB` + (`memory`), because a sandbox's private `/tmp` counts as instance memory and no + flag bounds it (S2). The directory budgets are checked once more after the code + exits on its own, so a run that writes past them between two samples and exits + before the next is still refused, with nothing of it collected. +* **The kill path.** On a timeout, the watchdog, a cancellation or an unexpected + error: `os.killpg` with SIGKILL on the launcher's process group, then + `sandbox delete --force `. `delete` exits 0 even when it did nothing (S2), so + the kill counts as confirmed only when the launcher process exits. One that does + not exit within `RENDERER_KILL_GRACE_S` is kept as `stuck`, and every render is + refused with `503 stuck` until it is gone (the service then restarts its instance). + A launcher that exits non-zero on its own also gets a `delete`, in case a failed + launch left a half-created sandbox. +* **One retry with a new sandbox name** when the launcher failed before the harness + started: a non-zero exit with no `HARNESS` start line on stdout. S2 saw one of 26 + renders end in 0.4 s with `failed to exec in container ... exit status 137` and no + output. A render whose harness started failed in the plot code and is not retried; + a second launcher failure is reported with reason `launcher`. +* **Bounded output.** stderr keeps its last `STDERR_LIMIT` bytes and stdout its first + and last few KiB, however much the code writes. Output files are read without + following links (`O_NOFOLLOW`, then `fstat`) and under the wire caps, and the probe + is cleaned (`wire.clean_probe`) before it reaches the response. +* **Host errors are refusals.** An `OSError` on the host (the run directory cannot be + written, the launcher cannot be started) is `Unavailable("io")`, a `503`, never a + plot failure. + +Strictly serial: the service holds one render slot (`main.py`), and the themes of a +job run one after the other. No ADK import. +""" + +import asyncio +import base64 +import contextlib +import json +import logging +import math +import os +import secrets +import shutil +import signal +import stat +import time +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path +from typing import Any, Protocol + +from .settings import RendererSettings +from .wire import ( + MAX_PNG_BYTES, + MAX_PROBE_BYTES, + MAX_STDERR_CHARS, + RenderRequest, + RunReason, + Theme, + ThemeRun, + clean_probe, +) + + +logger = logging.getLogger(__name__) + +SANDBOX_PATH = "/app/.venv/bin:/usr/local/bin:/usr/bin:/bin" +FORBIDDEN_FLAGS = ("--allow-egress", "--write") +WORKDIR = "/work" +CODE_FILE = "plot.py" +DATA_FILE = "data.csv" +LAUNCHER_FAILURE = "failed to exec in container" +HARNESS_START = 'HARNESS {"event": "start"}' +HARNESS_END = 'HARNESS {"event": "end"' +STDERR_LIMIT = 64 * 1024 +STDOUT_HEAD = 4 * 1024 +STDOUT_TAIL = 4 * 1024 +READ_CHUNK = 64 * 1024 +MAX_RUN_ENTRIES = 10_000 +"""Files and directories one run may create; many tiny files cost memory on the volume too.""" +MAX_RUN_DEPTH = 32 +"""Directory levels below the run directory the watchdog measures; a deeper tree counts as over its limit.""" +MAX_USAGE = 1_000_000.0 +"""Largest peak memory (MiB) or CPU time (s) accepted from the harness's end line, which the code can forge.""" +DELETE_TIMEOUT_S = 30.0 +READER_GRACE_S = 5.0 +MIN_RETRY_S = 1.0 +"""A retry needs at least this much of the job's time left.""" +MEMINFO = Path("/proc/meminfo") + + +class Unavailable(Exception): + """The renderer cannot take a render now; answered as `503 {"detail": code}`.""" + + def __init__(self, code: str) -> None: + super().__init__(code) + self.code = code + + +class OutputRejected(ValueError): + """An output file the host refuses to read: not a regular file, or over its cap (`size` and `limit` in bytes).""" + + def __init__(self, message: str, *, size: int | None = None, limit: int | None = None) -> None: + super().__init__(message) + self.size = size + self.limit = limit + + +@dataclass(frozen=True) +class Stop: + """Why the host stopped a run, with the value that broke the limit and the limit (see `ThemeRun.measured`).""" + + reason: RunReason + measured: int | None = None + limit: int | None = None + + +@dataclass(frozen=True) +class TreeUsage: + """What a run directory holds; `measurable` is False for a tree the watchdog could not walk.""" + + bytes: int + entries: int + measurable: bool = True + + +def mem_available_mb(path: Path = MEMINFO) -> int | None: + """`MemAvailable` in MiB: the memory signal, because the cgroup's `memory.current` is unreadable (S2).""" + try: + for line in path.read_text(encoding="ascii").splitlines(): + if line.startswith("MemAvailable:"): + return int(line.split()[1]) // 1024 + except (OSError, ValueError, IndexError): + return None + return None + + +def runs_volume_ok(runs_dir: Path, max_bytes: int, *, root: Path = Path("/")) -> bool: + """Whether `runs_dir` is a mount of its own of at most `max_bytes`: the in-memory volume, not the root filesystem. + + Without the deploy's volume flags the run directories would land on the + instance's root filesystem, where a disk fill took the whole instance down (S2). + """ + try: + if os.stat(runs_dir).st_dev == os.stat(root).st_dev: + return False + return shutil.disk_usage(runs_dir).total <= max_bytes + except OSError: + return False + + +async def read_tail(stream: asyncio.StreamReader | None, limit: int) -> bytes: + """Read `stream` to its end and keep only its last `limit` bytes.""" + if stream is None: + return b"" + tail = bytearray() + while chunk := await stream.read(READ_CHUNK): + tail += chunk + if len(tail) > limit: + del tail[:-limit] + return bytes(tail) + + +async def read_head_tail(stream: asyncio.StreamReader | None, head: int, tail: int) -> tuple[bytes, bytes]: + """Read `stream` to its end and keep its first `head` and its last `tail` bytes.""" + if stream is None: + return b"", b"" + first = bytearray() + last = bytearray() + while chunk := await stream.read(READ_CHUNK): + if len(first) < head: + first += chunk[: head - len(first)] + last += chunk + if len(last) > tail: + del last[:-tail] + return bytes(first), bytes(last) + + +def tree_size(root: Path, max_bytes: int, max_entries: int, max_depth: int = MAX_RUN_DEPTH) -> TreeUsage: + """Bytes and entries under `root`, without following links; stops early once a limit is passed. + + A directory deeper than `max_depth` below `root`, or one that cannot be listed or + measured for any reason but its removal (a path longer than `PATH_MAX` among them), + makes the tree unmeasurable, which the watchdog counts as over its limit: a subtree + the walk skipped would otherwise hold bytes nobody counts. + """ + total = 0 + entries = 0 + pending = [(root, 0)] + while pending: + current, depth = pending.pop() + try: + with os.scandir(current) as listing: + for entry in listing: + entries += 1 + try: + info = entry.stat(follow_symlinks=False) + except FileNotFoundError: + continue # removed while it was listed + except OSError: + return TreeUsage(total, entries, measurable=False) + total += info.st_size + if stat.S_ISDIR(info.st_mode): + if depth + 1 > max_depth: + return TreeUsage(total, entries, measurable=False) + pending.append((Path(entry.path), depth + 1)) + if total > max_bytes or entries > max_entries: + return TreeUsage(total, entries) + except (FileNotFoundError, NotADirectoryError): + continue # removed or replaced while the walk reached it + except OSError: + return TreeUsage(total, entries, measurable=False) + return TreeUsage(total, entries) + + +@dataclass(frozen=True) +class ExecutorConfig: + """Everything the executor needs, taken from `RendererSettings` (tests build it directly).""" + + sandbox_binary: str + python: str + harness: str + runs_dir: Path + mpl_seed: str + mplconfig: str + run_budget_bytes: int + watch_interval_s: float + rlimit_cpu_s: int + rlimit_fsize_mb: int + rlimit_as_mb: int + rlimit_nproc: int + kill_grace_s: float + bind_tmp: bool + kill_mem_available_mb: int = 0 + """The watchdog's `MemAvailable` floor in MiB; 0 switches the memory check off.""" + + @classmethod + def from_settings(cls, settings: RendererSettings) -> "ExecutorConfig": + return cls( + sandbox_binary=settings.sandbox_binary, + python=settings.python, + harness=settings.harness, + runs_dir=Path(settings.runs_dir), + mpl_seed=settings.mpl_seed, + mplconfig=settings.mplconfig, + run_budget_bytes=settings.run_budget_mb * 1024 * 1024, + watch_interval_s=settings.watch_interval_s, + rlimit_cpu_s=settings.rlimit_cpu_s, + rlimit_fsize_mb=settings.rlimit_fsize_mb, + rlimit_as_mb=settings.rlimit_as_mb, + rlimit_nproc=settings.rlimit_nproc, + kill_grace_s=settings.kill_grace_s, + bind_tmp=settings.bind_tmp, + kill_mem_available_mb=settings.kill_mem_available_mb, + ) + + +def check_tree(root: Path, config: ExecutorConfig) -> Stop | None: + """The run directory against the byte and entry budgets: the watchdog's look, and the last look after exit.""" + usage = tree_size(root, config.run_budget_bytes, MAX_RUN_ENTRIES) + if not usage.measurable: + return Stop("file_budget", None, MAX_RUN_ENTRIES) + if usage.bytes > config.run_budget_bytes: + return Stop("disk_budget", usage.bytes, config.run_budget_bytes) + if usage.entries > MAX_RUN_ENTRIES: + return Stop("file_budget", usage.entries, MAX_RUN_ENTRIES) + return None + + +def check_run(root: Path, config: ExecutorConfig, memory: Callable[[], int | None]) -> Stop | None: + """One watchdog look at a live run: its directory against the budgets, then `MemAvailable` against the floor.""" + verdict = check_tree(root, config) + if verdict is not None: + return verdict + floor = config.kill_mem_available_mb + if floor: + available = memory() + if available is not None and available < floor: + return Stop("memory", available, floor) + return None + + +async def watch_run(root: Path, config: ExecutorConfig, memory: Callable[[], int | None]) -> Stop: + """Return the first verdict of `check_run` that stops the run.""" + while True: + await asyncio.sleep(config.watch_interval_s) + verdict = await asyncio.to_thread(check_run, root, config, memory) + if verdict is not None: + return verdict + + +def read_regular(path: Path, max_bytes: int) -> bytes | None: + """The bytes of a regular file the run left, or None when it is missing. + + Opened with `O_NOFOLLOW` (a symlink fails) and `O_NONBLOCK` (a FIFO cannot block + the host), then checked with `fstat` on the open descriptor, so what is read is + exactly what was checked. + """ + try: + descriptor = os.open(path, os.O_RDONLY | os.O_NOFOLLOW | os.O_NONBLOCK) + except FileNotFoundError: + return None + except OSError as exc: + raise OutputRejected(f"{path.name} could not be opened as a regular file ({type(exc).__name__})") from None + try: + info = os.fstat(descriptor) + if not stat.S_ISREG(info.st_mode): + raise OutputRejected(f"{path.name} is not a regular file") + if info.st_size > max_bytes: + raise OutputRejected( + f"{path.name} is {info.st_size} bytes, more than the {max_bytes}-byte limit", + size=info.st_size, + limit=max_bytes, + ) + chunks = bytearray() + while len(chunks) <= max_bytes and (chunk := os.read(descriptor, READ_CHUNK)): + chunks += chunk + finally: + os.close(descriptor) + if len(chunks) > max_bytes: + raise OutputRejected( + f"{path.name} grew past the {max_bytes}-byte limit while it was read", size=len(chunks), limit=max_bytes + ) + return bytes(chunks) + + +def sandbox_env(config: ExecutorConfig, theme: Theme, cpu_s: int) -> dict[str, str]: + """The complete environment inside the sandbox: nothing of the host's reaches it.""" + env = { + "PATH": SANDBOX_PATH, + "HOME": "/tmp", + "ANYPLOT_THEME": theme, + "MPLBACKEND": "Agg", + "MPLCONFIGDIR": config.mplconfig, + "ANYPLOT_MPL_SEED": config.mpl_seed, + "PYTHONDONTWRITEBYTECODE": "1", + # One BLAS thread: Agg renders on one core, and every thread reserves address + # space under RLIMIT_AS and counts toward RLIMIT_NPROC. + "OPENBLAS_NUM_THREADS": "1", + "OMP_NUM_THREADS": "1", + "ANYPLOT_RLIMIT_CPU_S": str(cpu_s), + "ANYPLOT_RLIMIT_FSIZE_MB": str(config.rlimit_fsize_mb), + } + if config.rlimit_as_mb: + env["ANYPLOT_RLIMIT_AS_MB"] = str(config.rlimit_as_mb) + if config.rlimit_nproc: + env["ANYPLOT_RLIMIT_NPROC"] = str(config.rlimit_nproc) + return env + + +def build_argv(config: ExecutorConfig, name: str, run_dir: Path, theme: Theme, cpu_s: int) -> list[str]: + """The `sandbox do` command of one theme; asserted to carry neither `--write` nor `--allow-egress`.""" + argv = [ + config.sandbox_binary, + "do", + "--sandbox-name", + name, + "--mount", + f"type=bind,source={run_dir / 'work'},destination={WORKDIR}", + ] + if config.bind_tmp: + argv += ["--mount", f"type=bind,source={run_dir / 'tmp'},destination=/tmp"] + argv += ["-w", WORKDIR] + for key, value in sandbox_env(config, theme, cpu_s).items(): + argv += ["--env", f"{key}={value}"] + flags = list(argv) + argv += ["--", config.python, "-I", config.harness, CODE_FILE] + if any(part.split("=", 1)[0] in FORBIDDEN_FLAGS for part in flags): + raise AssertionError("a sandbox must never get --write or --allow-egress") + return argv + + +@dataclass +class Attempt: + """One launch of one theme's sandbox.""" + + name: str + exit_code: int | None + timed_out: bool = False + reason: RunReason | None = None + measured: int | None = None + limit: int | None = None + stdout_head: str = "" + stdout_tail: str = "" + stderr: str = "" + png: bytes | None = None + probe: dict[str, Any] | None = None + wall_s: float = 0.0 + + @property + def harness_started(self) -> bool: + return HARNESS_START in self.stdout_head.splitlines() + + @property + def launcher_failed(self) -> bool: + """The launcher failed before the harness printed its start line: worth one retry.""" + return self.exit_code not in (0, None) and self.reason is None and not self.harness_started + + def usage(self) -> dict[str, float]: + """The harness's own peak memory and CPU time, from its last end line (advisory). + + The code under test shares stdout and can print a forged end line, so only a + finite number between 0 and `MAX_USAGE` is taken; anything else is left out. + """ + for line in reversed(self.stdout_tail.splitlines()): + if not line.startswith(HARNESS_END): + continue + try: + report = json.loads(line.split(" ", 1)[1]) + except (ValueError, IndexError, RecursionError): + return {} + if not isinstance(report, dict): + return {} + usage: dict[str, float] = {} + for key in ("max_rss_mb", "cpu_s"): + value = report.get(key) + if isinstance(value, bool) or not isinstance(value, int | float): + continue + try: + number = float(value) + except OverflowError: + continue + if math.isfinite(number) and 0.0 <= number <= MAX_USAGE: + usage[key] = number + return usage + return {} + + def to_wire(self, theme: Theme, attempts: int) -> ThemeRun: + usage = self.usage() + return ThemeRun( + theme=theme, + exit_code=self.exit_code, + timed_out=self.timed_out, + png_base64=None if self.png is None else _b64(self.png), + probe=self.probe, + stderr_tail=self.stderr[-MAX_STDERR_CHARS:], + wall_s=round(self.wall_s, 3), + attempts=attempts, + reason=self.reason, + measured=self.measured, + limit=self.limit, + max_rss_mb=usage.get("max_rss_mb"), + cpu_s=usage.get("cpu_s"), + ) + + +def _b64(data: bytes) -> str: + return base64.b64encode(data).decode("ascii") + + +async def _settle[T](task: "asyncio.Task[T]", default: T) -> T: + """The reader's result once the process is gone, or `default` when the pipe stays open.""" + try: + return await asyncio.wait_for(task, READER_GRACE_S) + except TimeoutError: + return default + + +class Executor(Protocol): + """What the service needs from an executor; tests substitute a fake.""" + + @property + def stuck(self) -> int: ... + + @property + def available(self) -> bool: ... + + async def run_theme(self, job: RenderRequest, theme: Theme) -> ThemeRun: ... + + +class SandboxExecutor: + """Runs one theme of a job per call in a fresh Cloud Run sandbox.""" + + def __init__( + self, + config: ExecutorConfig, + *, + clock: Callable[[], float] = time.monotonic, + memory: Callable[[], int | None] = mem_available_mb, + ) -> None: + self.config = config + self.clock = clock + self.memory = memory + self._stuck: list[asyncio.subprocess.Process] = [] + + @property + def stuck(self) -> int: + """Launcher processes that outlived their kill and are still alive.""" + self._stuck = [process for process in self._stuck if process.returncode is None] + return len(self._stuck) + + @property + def available(self) -> bool: + return os.access(self.config.sandbox_binary, os.X_OK) + + async def run_theme(self, job: RenderRequest, theme: Theme) -> ThemeRun: + """Render one theme; a launcher failure before the harness started is retried once.""" + if not self.available: + raise Unavailable("sandbox_unavailable") + if self.stuck: + raise Unavailable("stuck") + cpu_s = max(1, min(self.config.rlimit_cpu_s, math.ceil(job.timeout_s))) + started = self.clock() + attempt = await self._attempt(job, theme, cpu_s, job.timeout_s) + attempts = 1 + if attempt.launcher_failed: + remaining = job.timeout_s - (self.clock() - started) + _log_attempt(attempt, theme, retry=remaining >= MIN_RETRY_S) + if remaining >= MIN_RETRY_S: + attempt = await self._attempt(job, theme, cpu_s, remaining) + attempts = 2 + if attempt.launcher_failed: + attempt.reason = "launcher" + _log_attempt(attempt, theme, retry=False) + return attempt.to_wire(theme, attempts) + + async def _attempt(self, job: RenderRequest, theme: Theme, cpu_s: int, timeout_s: float) -> Attempt: + name = f"r-{secrets.token_hex(6)}-{theme}" + run_dir = self.config.runs_dir / name + try: + self._prepare(run_dir, job) + return await self._run(name, run_dir, theme, cpu_s, timeout_s) + except OSError as exc: + # A full volume, a failed fork: the host could not run the job, which is + # not the code's fault. `_run` has stopped any sandbox it started. + logger.error(json.dumps({"event": "sandbox_io_error", "sandbox": name, "error": type(exc).__name__})) + raise Unavailable("io") from None + finally: + shutil.rmtree(run_dir, ignore_errors=True) + + def _prepare(self, run_dir: Path, job: RenderRequest) -> None: + """The run directory with the code and the data; the sandbox mounts its `work` directory.""" + work = run_dir / "work" + work.mkdir(parents=True) + # The sandbox is uid 0 behind gVisor's gofer, and the spikes ran with these + # modes; the host runs no other user, so nothing else can reach the files. + os.chmod(run_dir, 0o777) + os.chmod(work, 0o777) + if self.config.bind_tmp: + tmp = run_dir / "tmp" + tmp.mkdir() + os.chmod(tmp, 0o1777) + (work / CODE_FILE).write_text(job.source, encoding="utf-8") + (work / DATA_FILE).write_text(job.data_csv, encoding="utf-8") + + async def _run(self, name: str, run_dir: Path, theme: Theme, cpu_s: int, timeout_s: float) -> Attempt: + argv = build_argv(self.config, name, run_dir, theme, cpu_s) + started = self.clock() + process = await asyncio.create_subprocess_exec( + *argv, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + start_new_session=True, + ) + stdout_task = asyncio.create_task(read_head_tail(process.stdout, STDOUT_HEAD, STDOUT_TAIL)) + stderr_task = asyncio.create_task(read_tail(process.stderr, STDERR_LIMIT)) + exited = asyncio.create_task(process.wait()) + watchdog = asyncio.create_task(watch_run(run_dir, self.config, self.memory)) + stop: Stop | None = None + try: + done, _ = await asyncio.wait({exited, watchdog}, timeout=timeout_s, return_when=asyncio.FIRST_COMPLETED) + if exited not in done: + stop = watchdog.result() if watchdog in done else Stop("timeout") + await self._stop(name, process) + else: + # The code exited on its own. The watchdog samples between sleeps, so a + # run that wrote past the budgets and exited before the next sample + # escaped it; one last look refuses that run like a live one. A failed + # launch may also leave a half-created sandbox; `delete` is idempotent. + stop = await asyncio.to_thread(check_tree, run_dir, self.config) + if stop is not None or process.returncode != 0: + await self._delete(name) + except BaseException: + # A cancellation (the client went away, or `/render/{job_id}/cancel`) or an + # unexpected error: stop the sandbox before _attempt removes its run + # directory, then let it through. + await asyncio.shield(self._stop(name, process)) + stdout_task.cancel() + stderr_task.cancel() + raise + finally: + watchdog.cancel() + exited.cancel() + wall = self.clock() - started + head, tail = await _settle(stdout_task, (b"", b"")) + stderr = await _settle(stderr_task, b"") + attempt = Attempt( + name=name, + exit_code=None if stop else process.returncode, + timed_out=stop is not None and stop.reason == "timeout", + reason=None if stop is None else stop.reason, + measured=None if stop is None else stop.measured, + limit=None if stop is None else stop.limit, + stdout_head=head.decode("utf-8", errors="replace"), + stdout_tail=tail.decode("utf-8", errors="replace"), + stderr=stderr.decode("utf-8", errors="replace"), + wall_s=wall, + ) + if stop is None or stop.reason == "timeout": + self._collect(attempt, run_dir / "work", theme) + return attempt + + def _collect(self, attempt: Attempt, work: Path, theme: Theme) -> None: + """Read the PNG and the probe the run left; a refused PNG is reported, a refused probe is simply absent.""" + try: + attempt.png = read_regular(work / f"plot-{theme}.png", MAX_PNG_BYTES) + except OutputRejected as exc: + attempt.reason = "output_rejected" + attempt.measured = exc.size + attempt.limit = exc.limit + attempt.stderr = str(exc) + try: + attempt.probe = clean_probe(read_regular(work / f"probe-{theme}.json", MAX_PROBE_BYTES)) + except OutputRejected: + attempt.probe = None + + async def _stop(self, name: str, process: asyncio.subprocess.Process) -> None: + """Kill the launcher's process group, delete the sandbox, and confirm by the launcher's exit.""" + with contextlib.suppress(ProcessLookupError, PermissionError): + os.killpg(process.pid, signal.SIGKILL) + await self._delete(name) + try: + await asyncio.wait_for(process.wait(), self.config.kill_grace_s) + except TimeoutError: + if process not in self._stuck: + self._stuck.append(process) + logger.error(json.dumps({"event": "sandbox_stuck", "sandbox": name})) + + async def _delete(self, name: str) -> None: + """`sandbox delete --force `; its exit code proves nothing (it is 0 for an unknown name).""" + try: + deleter = await asyncio.create_subprocess_exec( + self.config.sandbox_binary, + "delete", + "--force", + name, + stdin=asyncio.subprocess.DEVNULL, + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.DEVNULL, + start_new_session=True, + ) + except OSError as exc: + logger.error(json.dumps({"event": "sandbox_delete_failed", "sandbox": name, "error": type(exc).__name__})) + return + try: + await asyncio.wait_for(deleter.wait(), DELETE_TIMEOUT_S) + except TimeoutError: + with contextlib.suppress(ProcessLookupError): + deleter.kill() + await deleter.wait() + + +def _log_attempt(attempt: Attempt, theme: Theme, *, retry: bool) -> None: + """One JSON line per sandbox launch: ids, codes and timings, never stdout, stderr or code.""" + logger.info( + json.dumps( + { + "event": "sandbox", + "sandbox": attempt.name, + "theme": theme, + "exit_code": attempt.exit_code, + "reason": attempt.reason, + "measured": attempt.measured, + "limit": attempt.limit, + "harness_started": attempt.harness_started, + "launcher_message": LAUNCHER_FAILURE in attempt.stderr, + "retry": retry, + "wall_s": round(attempt.wall_s, 3), + "png_bytes": None if attempt.png is None else len(attempt.png), + } + ) + ) diff --git a/agents/renderer/main.py b/agents/renderer/main.py new file mode 100644 index 0000000000..b5e5b710ea --- /dev/null +++ b/agents/renderer/main.py @@ -0,0 +1,437 @@ +"""anyplot-renderer: runs adapted plot code in Cloud Run sandboxes, one at a time. + +The agents service (`agents/main.py`) and a developer's `adk web` both reach this +service through the `remote` render backend (`agents/anyplot/render/backends/remote.py`), +so a local run and the deployed service render in the same sandboxes. Nothing in +`agents/renderer/` imports ADK or `agents.anyplot` (whose package import loads ADK), +so the renderer image installs neither; `wire.py` is the JSON contract both sides +read. The directory has no `__init__.py` on purpose: `adk web agents` lists every +subdirectory of `agents/` that has one as an agent, so this is a namespace package, +like `agents/evals/`. + +| Route | Body | Result | Errors | +|---|---|---|---| +| `POST /render` | `RenderRequest` (`wire.py`): `{job_id, language, library, source, data_csv, themes, timeout_s}` | `RenderResponse`: per theme the exit code, a timeout flag, the stderr tail, the probe, the PNG as base64, the attempts, a reason with the measured value and the limit; the slot wait and total time; `MemAvailable` | `401 unauthenticated`, `403 forbidden`, `409 job_conflict` (another payload for a `job_id` and themes in flight or just finished), `422 invalid_job`, `499 cancelled` or `499 client_gone` (nobody reads it), `500 internal`, `503 busy` (no slot within `RENDERER_SLOT_WAIT_S`), `503 low_memory`, `503 stuck`, `503 sandbox_unavailable`, `503 volume_missing`, `503 io` | +| `POST /render/{job_id}/cancel` | | `{"cancelled": n}`: the renders of that job it stopped | `401`, `403`, `422 invalid_job` | +| `GET /status` | | `RendererStatus`: name, version, revision, in flight, waiting, stuck launchers, `MemAvailable`, whether the launcher exists, whether the run volume is in place | `401`, `403` | + +All routes sit behind the caller check (`auth.py`) and Cloud Run IAM. There is no +`/healthz`: Google's front end intercepts that path (spike S), and Cloud Run's +default TCP startup probe is enough for a service with no dependencies. + +**One render at a time.** The service holds one render slot, and the themes of a job +render one after the other inside it, so at most one sandbox exists on the instance: +spikes S and S2 showed that one 4 GiB instance serves one sandbox safely and that +many sandboxes crash the whole instance. A request waits at most +`RENDERER_SLOT_WAIT_S` for the slot (`503 busy`), and a render is refused while +`MemAvailable` is below `RENDERER_MIN_MEM_AVAILABLE_MB` (`503 low_memory`), which is +the memory signal S2 asked for because the cgroup's memory counter is unreadable. + +**`job_id` and the themes are the idempotency key.** The `remote` backend and the +deploy smoke replay a `POST /render` after an ambiguous failure (a connection error, +a front-end 429 or 5xx), and the first request may have reached this service all the +same. A replay with the same payload while that render runs joins it and gets the +same answer; a replay after it finished gets the stored answer (the last `DONE_CACHE` +successful renders are kept); a replay with another payload for the same `job_id` +and themes is refused with `409 job_conflict`. The key includes the themes because +the backend sends one request per theme under the job's id (`render/serial.py`). +A job's code never runs twice because of a retry. + +**A render stops when its caller does.** uvicorn does not cancel a handler whose +client went away, so the route runs the render as a task and races it against the +connection's `http.disconnect`; the last caller to leave, or a +`POST /render/{job_id}/cancel` from the `remote` backend when its own render was +cancelled or timed out (for a front end that does not pass the disconnect on), cancels +the task, and the executor's kill path stops the sandbox and frees the slot. + +**Fail closed on the instance.** On Cloud Run the run directories must sit on the +size-limited volume (`runs_volume_ok`), or every render answers `503 volume_missing`. +A launcher that outlived its kill makes every later render `503 stuck`, so the +service then ends its own process (SIGTERM, after answering), and Cloud Run starts +a fresh instance for the next request. + +The service only runs code and returns what the run left: the agents service's host +gates (R1-R3) and probe gates judge it (`agents/anyplot/render/gates.py`). Nothing of +a job (code, data, stderr, PNG) is logged, and an unexpected error is logged by its +class and answered `500 internal`, because its message can hold the job's content; +the log carries ids, exit codes, reasons and timings. A request validation error +answers `invalid_job` without echoing the body, because the body is the user's code +and data. + +Run locally (without a launcher every render answers `503 sandbox_unavailable`): +`ENVIRONMENT=development uv run uvicorn agents.renderer.main:app --port 8003`. +""" + +import asyncio +import contextlib +import hashlib +import importlib.metadata +import json +import logging +import os +import signal +import time +from collections import OrderedDict +from collections.abc import AsyncIterator, Callable +from dataclasses import dataclass +from functools import lru_cache +from pathlib import Path +from typing import Annotated, Any + +from fastapi import Depends, FastAPI, Request +from fastapi import Path as PathParam +from fastapi.exceptions import RequestValidationError +from fastapi.responses import JSONResponse + +from .auth import CallerRefused, check_caller +from .executor import Executor, ExecutorConfig, SandboxExecutor, Unavailable, mem_available_mb, runs_volume_ok +from .settings import RendererSettings, get_settings +from .wire import JOB_ID_PATTERN, RendererStatus, RenderRequest, RenderResponse, RenderTimings, Theme, ThemeRun + + +logger = logging.getLogger("agents.renderer") +VERSION_FILE = Path(__file__).with_name("VERSION") +"""Written by the image build from `pyproject.toml`, because the image does not install the project.""" +_NO_STORE = {"Cache-Control": "no-store"} +EXIT_DELAY_S = 1.0 +"""How long the service waits after scheduling its own exit, so the answer in progress goes out first.""" +DISCONNECT_PAUSE_S = 0.5 +"""Pause after a `receive()` message that is not `http.disconnect`, so the watcher never spins.""" +CLIENT_GONE = 499 +"""The answer to a request that was cancelled or whose client left; nobody reads it.""" +DONE_CACHE = 16 +"""Finished renders kept by `job_id`, so a replay after the answer was lost gets it instead of a second run.""" + + +class JobConflict(Exception): + """A `POST /render` whose key is in flight or just finished with another payload; answered `409 job_conflict`.""" + + +type JobKey = tuple[str, tuple[Theme, ...]] +"""The idempotency key of a request: its `job_id` and its themes, in order.""" + + +def job_key(job: RenderRequest) -> JobKey: + return (job.job_id, tuple(job.themes)) + + +def fingerprint(job: RenderRequest) -> str: + """What makes two `POST /render` the same job: the whole request, hashed (the body itself is never kept twice).""" + return hashlib.sha256(job.model_dump_json().encode("utf-8")).hexdigest() + + +@dataclass +class RunningJob: + """A render in flight and the requests waiting for it; the last one to leave may cancel it.""" + + fingerprint: str + task: "asyncio.Task[RenderResponse]" + waiters: int = 0 + + +def _version() -> str: + try: + return importlib.metadata.version("anyplot") + except importlib.metadata.PackageNotFoundError: + pass + try: + return VERSION_FILE.read_text(encoding="utf-8").strip() or "dev" + except OSError: + return "dev" + + +def _terminate_soon() -> None: + """End this process shortly (uvicorn shuts down gracefully on SIGTERM); Cloud Run starts a fresh instance.""" + asyncio.get_running_loop().call_later(EXIT_DELAY_S, os.kill, os.getpid(), signal.SIGTERM) + + +class RenderSlot: + """The renderer's one render slot: first come, first served, with a bounded wait.""" + + def __init__(self) -> None: + self._lock = asyncio.Lock() + self.waiting = 0 + + @property + def in_flight(self) -> int: + return 1 if self._lock.locked() else 0 + + @contextlib.asynccontextmanager + async def hold(self, wait_s: float) -> AsyncIterator[None]: + """Hold the slot; raises `Unavailable("busy")` when it did not come free within `wait_s`.""" + self.waiting += 1 + try: + async with asyncio.timeout(wait_s): + await self._lock.acquire() + except TimeoutError: + raise Unavailable("busy") from None + finally: + self.waiting -= 1 + try: + yield + finally: + self._lock.release() + + +class Renderer: + """The settings, the executor, the slot and the running renders of one service instance.""" + + def __init__( + self, settings: RendererSettings, executor: Executor, *, terminate: Callable[[], None] = _terminate_soon + ) -> None: + self.settings = settings + self.executor = executor + self.slot = RenderSlot() + self.version = _version() + self.terminate = terminate + self.terminating = False + self.volume: bool | None = None + if settings.cloud_run_service: + self.volume = runs_volume_ok(Path(settings.runs_dir), settings.runs_volume_max_mb * 1024 * 1024) + if not self.volume: + logger.error(json.dumps({"event": "runs_volume_missing", "runs_dir": settings.runs_dir})) + self._running: dict[JobKey, RunningJob] = {} + self._done: OrderedDict[JobKey, tuple[str, RenderResponse]] = OrderedDict() + + def join(self, job: RenderRequest) -> "asyncio.Future[RenderResponse]": + """The render of `job`, started now or already there, with this request counted as a waiter. + + `job_id` and the themes are the idempotency key (see the module docstring): + the same payload joins the render in flight or gets the stored answer of a + finished one, and another payload under the same key raises `JobConflict`. + Every `join` is balanced by one `leave`. + """ + key = job_key(job) + print_ = fingerprint(job) + running = self._running.get(key) + if running is not None: + if running.fingerprint != print_: + raise JobConflict + running.waiters += 1 + logger.info(json.dumps({"event": "render_joined", "job_id": job.job_id, "finished": False})) + return running.task + done = self._done.get(key) + if done is not None: + if done[0] != print_: + raise JobConflict + logger.info(json.dumps({"event": "render_joined", "job_id": job.job_id, "finished": True})) + answer: asyncio.Future[RenderResponse] = asyncio.get_running_loop().create_future() + answer.set_result(done[1]) + return answer + task = asyncio.create_task(self.render(job)) + self._running[key] = RunningJob(print_, task, waiters=1) + task.add_done_callback(lambda finished: self._forget(key, print_, finished)) + return task + + def leave(self, job: RenderRequest) -> bool: + """One waiter of `job` is gone; True when its render still runs and nobody waits for it any more.""" + running = self._running.get(job_key(job)) + if running is None or running.waiters == 0: + return False + running.waiters -= 1 + return running.waiters == 0 and not running.task.done() + + def _forget(self, key: JobKey, print_: str, task: "asyncio.Task[RenderResponse]") -> None: + running = self._running.get(key) + if running is not None and running.task is task: + del self._running[key] + if task.cancelled() or task.exception() is not None: + return # only an answer is worth replaying; a refusal or an error runs again + self._done[key] = (print_, task.result()) + while len(self._done) > DONE_CACHE: + self._done.popitem(last=False) + + def cancel(self, job_id: str) -> int: + """Cancel every running render of `job_id` (one per theme request); the executor's kill path stops its sandbox.""" + tasks = [ + running.task for (id_, _), running in self._running.items() if id_ == job_id and not running.task.done() + ] + for task in tasks: + task.cancel() + return len(tasks) + + async def render(self, job: RenderRequest) -> RenderResponse: + started = time.monotonic() + try: + return await self._render(job, started) + finally: + self._exit_if_stuck() + + async def _render(self, job: RenderRequest, started: float) -> RenderResponse: + if self.volume is False: + raise Unavailable("volume_missing") + async with self.slot.hold(self.settings.slot_wait_s): + waited = time.monotonic() - started + floor = self.settings.min_mem_available_mb + available = mem_available_mb() + if floor and available is not None and available < floor: + raise Unavailable("low_memory") + outputs: dict[Theme, ThemeRun] = {} + for theme in job.themes: + outputs[theme] = await self.executor.run_theme(job, theme) + response = RenderResponse( + job_id=job.job_id, + outputs=outputs, + timings=RenderTimings(slot_wait_s=round(waited, 3), total_s=round(time.monotonic() - started, 3)), + version=self.version, + mem_available_mb=mem_available_mb(), + ) + logger.info( + json.dumps( + { + "event": "render", + "job_id": job.job_id, + "library": job.library, + "themes": list(job.themes), + "exit_codes": {theme: run.exit_code for theme, run in response.outputs.items()}, + "reasons": {theme: run.reason for theme, run in response.outputs.items()}, + "attempts": {theme: run.attempts for theme, run in response.outputs.items()}, + "slot_wait_s": response.timings.slot_wait_s, + "total_s": response.timings.total_s, + "mem_available_mb_before": available, + "mem_available_mb_after": response.mem_available_mb, + } + ) + ) + return response + + def _exit_if_stuck(self) -> None: + """On Cloud Run, a stuck launcher ends this instance: no render could run on it again.""" + if self.terminating or not self.settings.cloud_run_service or not self.executor.stuck: + return + self.terminating = True + logger.error(json.dumps({"event": "renderer_exiting", "why": "stuck_launcher", "stuck": self.executor.stuck})) + self.terminate() + + def status(self) -> RendererStatus: + return RendererStatus( + version=self.version, + revision=self.settings.revision, + in_flight=self.slot.in_flight, + waiting=self.slot.waiting, + stuck=self.executor.stuck, + mem_available_mb=mem_available_mb(), + sandbox=self.executor.available, + volume=self.volume, + ) + + +@lru_cache(maxsize=1) +def get_renderer() -> Renderer: + """The process-wide renderer; tests override this dependency.""" + settings = get_settings() + return Renderer(settings, SandboxExecutor(ExecutorConfig.from_settings(settings))) + + +@contextlib.asynccontextmanager +async def lifespan(_: FastAPI) -> AsyncIterator[None]: + """Fail at startup, not at the first render, when the settings are invalid.""" + logging.basicConfig(level=logging.INFO, format="%(message)s") + get_renderer() + yield + + +app = FastAPI(title="anyplot-renderer", docs_url=None, redoc_url=None, openapi_url=None, lifespan=lifespan) +RendererDep = Annotated[Renderer, Depends(get_renderer)] +caller_checked = [Depends(check_caller)] + + +def _internal() -> JSONResponse: + """The renderer's own 500: a JSON `detail`, so the `remote` backend never mistakes it for the front end's.""" + return JSONResponse(status_code=500, content={"detail": "internal"}, headers=_NO_STORE) + + +@app.exception_handler(CallerRefused) +async def _caller_refused(_: Request, exc: CallerRefused) -> JSONResponse: + return JSONResponse(status_code=exc.status, content={"detail": exc.code}, headers=_NO_STORE) + + +@app.exception_handler(Unavailable) +async def _unavailable(_: Request, exc: Unavailable) -> JSONResponse: + return JSONResponse(status_code=503, content={"detail": exc.code}, headers=_NO_STORE) + + +@app.exception_handler(JobConflict) +async def _job_conflict(_: Request, exc: JobConflict) -> JSONResponse: + return JSONResponse(status_code=409, content={"detail": "job_conflict"}, headers=_NO_STORE) + + +@app.exception_handler(RequestValidationError) +async def _invalid(_: Request, exc: RequestValidationError) -> JSONResponse: + """A fixed answer: FastAPI's default would echo the body, which is the user's code and data.""" + return JSONResponse(status_code=422, content={"detail": "invalid_job"}, headers=_NO_STORE) + + +@app.exception_handler(Exception) +async def _unexpected(_: Request, exc: Exception) -> JSONResponse: + """The last resort outside `/render` (which answers its own errors): the class is logged, never the message.""" + logger.error(json.dumps({"event": "renderer_error", "error": type(exc).__name__})) + return _internal() + + +async def _client_gone(request: Request) -> None: + """Return once the server reports the client's disconnect. + + The body has been read by then, so uvicorn's `receive()` blocks until the + connection closes (or the response is complete, which never happens while the + render runs) and then answers `http.disconnect`. + """ + while True: + message = await request.receive() + if message.get("type") == "http.disconnect": + return + await asyncio.sleep(DISCONNECT_PAUSE_S) + + +async def _leave(renderer: Renderer, job: RenderRequest, task: "asyncio.Future[RenderResponse]") -> None: + """This request is done with the render; the last one to leave cancels it and waits for its sandbox to stop.""" + if renderer.leave(job): + task.cancel() + await asyncio.wait({task}) + + +@app.post("/render", dependencies=caller_checked) +async def render(job: RenderRequest, renderer: RendererDep, request: Request) -> JSONResponse: + """Run every theme of `job` in its own sandbox, one after the other, inside the render slot.""" + task = renderer.join(job) + gone = asyncio.create_task(_client_gone(request)) + waiting: set[asyncio.Future[Any]] = {task, gone} + try: + await asyncio.wait(waiting, return_when=asyncio.FIRST_COMPLETED) + except asyncio.CancelledError: + await _leave(renderer, job, task) + raise + finally: + gone.cancel() + if not task.done(): + await _leave(renderer, job, task) + logger.info(json.dumps({"event": "render_abandoned", "job_id": job.job_id, "why": "client_gone"})) + return JSONResponse(status_code=CLIENT_GONE, content={"detail": "client_gone"}, headers=_NO_STORE) + renderer.leave(job) + if task.cancelled(): + logger.info(json.dumps({"event": "render_abandoned", "job_id": job.job_id, "why": "cancelled"})) + return JSONResponse(status_code=CLIENT_GONE, content={"detail": "cancelled"}, headers=_NO_STORE) + error = task.exception() + if isinstance(error, Unavailable): + return await _unavailable(request, error) + try: + if error is not None: + raise error + content = task.result().model_dump(mode="json") + except Exception as exc: # the class only: a message can carry the job's code, data or output + logger.error(json.dumps({"event": "render_error", "job_id": job.job_id, "error": type(exc).__name__})) + return _internal() + return JSONResponse(content=content, headers=_NO_STORE) + + +@app.post("/render/{job_id}/cancel", dependencies=caller_checked) +async def cancel(job_id: Annotated[str, PathParam(pattern=JOB_ID_PATTERN)], renderer: RendererDep) -> JSONResponse: + """Stop the renders of `job_id`: the `remote` backend's call when its own render was cancelled or timed out.""" + cancelled = renderer.cancel(job_id) + logger.info(json.dumps({"event": "render_cancel", "job_id": job_id, "cancelled": cancelled})) + return JSONResponse(content={"cancelled": cancelled}, headers=_NO_STORE) + + +@app.get("/status", dependencies=caller_checked) +async def status(renderer: RendererDep) -> JSONResponse: + return JSONResponse(content=renderer.status().model_dump(mode="json"), headers=_NO_STORE) diff --git a/agents/renderer/settings.py b/agents/renderer/settings.py new file mode 100644 index 0000000000..6162eede69 --- /dev/null +++ b/agents/renderer/settings.py @@ -0,0 +1,126 @@ +"""Runtime settings of the renderer service, read from `RENDERER_*` environment variables. + +`ENVIRONMENT` (shared with the API and the agents, no prefix) and the Cloud Run +variables `K_SERVICE` and `K_REVISION` are read without the prefix. The defaults are +the deployed values, so an unset variable never weakens a limit. + +| Variable | Default | Meaning | +|---|---|---| +| `RENDERER_ALLOWED_CALLERS` | empty | Emails whose ID token may call the service (comma-separated): the agents service account, and the owner's account for `adk web` | +| `RENDERER_AUDIENCES` | empty | Accepted ID-token `aud` values (comma-separated): the service URL (the deploy step adds it), plus the OAuth client id of a developer's user token (see agents/README.md). Cloud Run requires the service URL as the audience even for a request to a tag URL | +| `RENDERER_SANDBOX_BINARY` | `/usr/local/gcp/bin/sandbox` | The Cloud Run sandbox launcher | +| `RENDERER_PYTHON` | `/app/.venv/bin/python` | The interpreter inside the sandbox (the image's own venv) | +| `RENDERER_HARNESS` | `/opt/anyplot/harness.py` | The probe harness inside the sandbox | +| `RENDERER_RUNS_DIR` | `/tmp/runs` | Root of the run directories: the size-limited in-memory volume | +| `RENDERER_MPL_SEED` | `/opt/mplconfig` | The baked matplotlib cache the harness copies into the sandbox | +| `RENDERER_MPLCONFIG` | `/tmp/mpl` | `MPLCONFIGDIR` inside the sandbox (its private `/tmp`) | +| `RENDERER_RUN_BUDGET_MB` | `64` | Bytes one run directory may hold before the watchdog kills the sandbox | +| `RENDERER_WATCH_INTERVAL_S` | `0.05` | How often the watchdog measures a live run directory | +| `RENDERER_SLOT_WAIT_S` | `30` | Longest wait for the render slot before `503 busy` | +| `RENDERER_MIN_MEM_AVAILABLE_MB` | `1024` | Below this `MemAvailable` a render is refused with `503 low_memory`; 0 switches the floor off | +| `RENDERER_KILL_MEM_AVAILABLE_MB` | `512` | Below this `MemAvailable` during a run the watchdog kills the sandbox (reason `memory`); 0 switches it off | +| `RENDERER_RUNS_VOLUME_MAX_MB` | `1024` | On Cloud Run, `RENDERER_RUNS_DIR` must be a mount of its own of at most this size, or every render answers `503 volume_missing` | +| `RENDERER_RLIMIT_CPU_S` | `60` | CPU seconds of the sandboxed process (also capped by the job's timeout) | +| `RENDERER_RLIMIT_FSIZE_MB` | `50` | Largest file the sandboxed process may write | +| `RENDERER_RLIMIT_AS_MB` | `1024` | Address space of the sandboxed process; 0 leaves it unset | +| `RENDERER_RLIMIT_NPROC` | `64` | Processes and threads of the sandboxed process; 0 leaves it unset | +| `RENDERER_KILL_GRACE_S` | `10` | How long to wait for the launcher process to exit after a kill | +| `RENDERER_BIND_TMP` | `false` | Bind a directory of the run over the sandbox's `/tmp`, so `/tmp` counts toward the run budget (spike S2, untested) | +| `ENVIRONMENT` | `production` | `development` skips the caller check and is refused on Cloud Run | +""" + +from functools import lru_cache +from typing import Annotated, Any, Self + +from pydantic import Field, NonNegativeInt, PositiveFloat, PositiveInt, field_validator, model_validator +from pydantic_settings import BaseSettings, NoDecode, SettingsConfigDict + + +def _split_list(value: Any) -> Any: + """A comma-separated string (the documented form) or a list; an empty string is an empty list.""" + if isinstance(value, str): + return [item.strip() for item in value.split(",") if item.strip()] + return value + + +class RendererSettings(BaseSettings): + """Settings of the anyplot-renderer service.""" + + model_config = SettingsConfigDict( + env_prefix="RENDERER_", case_sensitive=False, extra="ignore", validate_by_name=True, validate_by_alias=True + ) + + allowed_callers: Annotated[list[str], NoDecode] = [] + """Emails allowed to call the service (`RENDERER_ALLOWED_CALLERS`). Empty means nobody outside development.""" + + audiences: Annotated[list[str], NoDecode] = [] + """Accepted ID-token audiences (`RENDERER_AUDIENCES`). Empty means nobody outside development.""" + + sandbox_binary: str = "/usr/local/gcp/bin/sandbox" + python: str = "/app/.venv/bin/python" + harness: str = "/opt/anyplot/harness.py" + runs_dir: str = "/tmp/runs" + mpl_seed: str = "/opt/mplconfig" + mplconfig: str = "/tmp/mpl" + + run_budget_mb: PositiveInt = 64 + """Per-run byte budget (`RENDERER_RUN_BUDGET_MB`). A run legitimately leaves a PNG of at most + 10 MiB, a probe and the two inputs; the 512 MiB volume is shared by every run directory.""" + + watch_interval_s: PositiveFloat = 0.05 + slot_wait_s: PositiveFloat = 30.0 + min_mem_available_mb: NonNegativeInt = 1024 + kill_mem_available_mb: NonNegativeInt = 512 + """The watchdog's memory floor during a run (`RENDERER_KILL_MEM_AVAILABLE_MB`): a sandbox's + private `/tmp` counts as instance memory and no flag bounds it (spike S2), so a run that + pushes `MemAvailable` below this is killed before it takes the instance down.""" + runs_volume_max_mb: PositiveInt = 1024 + """Largest size the run-directory volume may report on Cloud Run (`RENDERER_RUNS_VOLUME_MAX_MB`); + the deploy mounts 512 MiB. A larger or shared filesystem means the volume flags were lost.""" + rlimit_cpu_s: PositiveInt = 60 + rlimit_fsize_mb: PositiveInt = 50 + rlimit_as_mb: NonNegativeInt = 1024 + rlimit_nproc: NonNegativeInt = 64 + kill_grace_s: PositiveFloat = 10.0 + bind_tmp: bool = False + + environment: str = Field(default="production", validation_alias="ENVIRONMENT") + """Deployment environment (`ENVIRONMENT`, no prefix). Unset is `production`, so the caller check stays on.""" + + cloud_run_service: str | None = Field(default=None, validation_alias="K_SERVICE") + revision: str | None = Field(default=None, validation_alias="K_REVISION") + + @field_validator("allowed_callers", "audiences", mode="before") + @classmethod + def _parse_list(cls, value: Any) -> Any: + return _split_list(value) + + @field_validator("environment") + @classmethod + def _normalise_environment(cls, value: str) -> str: + return value.strip().lower() + + @field_validator("sandbox_binary", "python", "harness", "runs_dir", "mpl_seed", "mplconfig") + @classmethod + def _absolute(cls, value: str) -> str: + if not value.startswith("/"): + raise ValueError(f"{value!r} must be an absolute path") + return value + + @model_validator(mode="after") + def _consistent(self) -> Self: + if self.is_development and self.cloud_run_service: + raise ValueError( + "ENVIRONMENT=development is refused on Cloud Run (K_SERVICE is set): it would skip the caller check" + ) + return self + + @property + def is_development(self) -> bool: + return self.environment == "development" + + +@lru_cache(maxsize=1) +def get_settings() -> RendererSettings: + """The process-wide settings, read once from the environment.""" + return RendererSettings() diff --git a/agents/renderer/wire.py b/agents/renderer/wire.py new file mode 100644 index 0000000000..d274503139 --- /dev/null +++ b/agents/renderer/wire.py @@ -0,0 +1,180 @@ +"""The renderer's JSON contract: the body of `POST /render` and its answer. + +The request is the agents' `RenderJob` (`agents/anyplot/render/contract.py`) field for +field; the answer is its `RenderResult`, one `ThemeRun` per theme, with the PNG as +base64 and the timings added. The renderer service (`main.py`) validates requests +with these models and the `remote` backend parses answers with them, so both sides +read one definition. + +No ADK and no `agents.anyplot` import: importing that package loads ADK, which the +renderer image does not install. The limits that also exist on the agents side +(`MAX_PNG_BYTES`, `MAX_PROBE_BYTES`, `MAX_STDERR_CHARS`, the job id pattern) are +pinned equal by `tests/unit/agents/renderer/test_wire.py`. +""" + +import json +import math +from typing import Any, Literal + +from pydantic import BaseModel, ConfigDict, Field, field_validator + + +Theme = Literal["light", "dark"] +JOB_ID_PATTERN = r"^[A-Za-z0-9_-]{1,64}$" +"""The agents' `RENDER_ID_PATTERN`: a job id is opaque and never becomes a path or a sandbox name.""" +LIBRARY_PATTERN = r"^[a-z0-9_-]{1,32}$" +MAX_SOURCE_CHARS = 128 * 1024 +"""The run form: the validator parses at most 48 KB, the loader and the normaliser add little.""" +MAX_DATA_CSV_CHARS = 2 * 1024 * 1024 +"""The canonical `data.csv`: the parser takes at most 200 KB of pasted text, and canonical +dates and decimals grow it by well under a factor of ten.""" +MAX_TIMEOUT_S = 120.0 +MAX_PNG_BYTES = 10 * 1024 * 1024 +"""Largest PNG the renderer returns; the agents' R2 gate (`render/png.py`) applies the same cap.""" +MAX_PROBE_BYTES = 256 * 1024 +MAX_PROBE_DEPTH = 16 +"""Deepest nesting a probe may have; the harness writes three levels.""" +MAX_PROBE_INT = 2**53 +"""Largest integer magnitude a probe may hold: what a JSON number keeps exactly as a float.""" +MAX_STDERR_CHARS = 2_000 +"""The stderr tail a theme returns; the agents' `ThemeOutput` keeps the same number of characters.""" + +RunReason = Literal["timeout", "disk_budget", "file_budget", "memory", "launcher", "output_rejected"] +"""Why a theme has no usable output, when the renderer knows more than the exit code: +`timeout` (killed at the job's time limit), `disk_budget` (killed for writing more bytes than +the per-run budget), `file_budget` (killed for creating more files and directories than the +per-run limit, or a tree too deep to measure), `memory` (killed because the instance's +`MemAvailable` fell below the renderer's floor during the run), `launcher` (the sandbox +launcher failed twice before the harness started), `output_rejected` (the PNG is not a +regular file or is over its cap). `measured` and `limit` carry the numbers where there are some.""" + + +def clean_probe(raw: bytes | None) -> dict[str, Any] | None: + """`probe-.json` as a JSON object the wire can carry, or None. + + The code under test can write this file, so it is untrusted: `NaN` and + `Infinity` become null (the harness writes them for a text at an undefined + position), and a probe that is not an object, nests deeper than + `MAX_PROBE_DEPTH`, or holds a number that a float cannot carry exactly is + dropped whole, because serialising it would fail the response (a nesting of + 256 levels) or the agents side's parse (about 200 levels, or an integer too + large for a float). + """ + if not raw: + return None + try: + loaded = json.loads(raw, parse_constant=lambda _: None) + except (ValueError, RecursionError): + return None + if not isinstance(loaded, dict) or not _probe_value_ok(loaded, 1): + return None + return loaded + + +def _probe_value_ok(value: Any, depth: int) -> bool: + if depth > MAX_PROBE_DEPTH: + return False + if isinstance(value, dict): + return all(_probe_value_ok(item, depth + 1) for item in value.values()) + if isinstance(value, list): + return all(_probe_value_ok(item, depth + 1) for item in value) + if isinstance(value, bool) or value is None or isinstance(value, str): + return True + if isinstance(value, int): + return abs(value) <= MAX_PROBE_INT + if isinstance(value, float): + return math.isfinite(value) + return False + + +class RenderRequest(BaseModel): + """One code version to render in every theme of `themes`, one sandbox per theme, in order.""" + + model_config = ConfigDict(extra="forbid") + + job_id: str = Field(pattern=JOB_ID_PATTERN) + language: Literal["python"] + library: str = Field(pattern=LIBRARY_PATTERN) + source: str = Field(min_length=1, max_length=MAX_SOURCE_CHARS) + data_csv: str = Field(max_length=MAX_DATA_CSV_CHARS) + themes: list[Theme] = Field(min_length=1, max_length=2) + timeout_s: float = Field(gt=0, le=MAX_TIMEOUT_S) + + @field_validator("themes") + @classmethod + def _distinct(cls, value: list[Theme]) -> list[Theme]: + if len(set(value)) != len(value): + raise ValueError("themes must be distinct") + return value + + +class ThemeRun(BaseModel): + """What one theme's sandbox left behind, before any gate looked at it.""" + + model_config = ConfigDict(extra="forbid") + + theme: Theme + exit_code: int | None + """The exit code of the sandboxed process; None when the renderer killed it.""" + timed_out: bool = False + png_base64: str | None = None + """`plot-.png` as the code wrote it (base64), at most `MAX_PNG_BYTES` before encoding.""" + probe: dict[str, Any] | None = None + """`probe-.json` from the harness's savefig probe, when it is a JSON object.""" + stderr_tail: str = Field(default="", max_length=MAX_STDERR_CHARS) + wall_s: float = Field(default=0.0, ge=0) + attempts: int = Field(default=1, ge=1, le=2) + """Sandbox launches for this theme: 2 when the first launcher call failed before the harness started.""" + reason: RunReason | None = None + measured: int | None = None + """The value that broke a limit: bytes (`disk_budget`, an oversized PNG for `output_rejected`), + files and directories (`file_budget`; None for a tree too deep to measure), or MiB of + `MemAvailable` (`memory`). None for the other reasons.""" + limit: int | None = None + """The limit `measured` broke, in the same unit.""" + max_rss_mb: float | None = None + """Peak resident memory the harness reported for itself (advisory: the code shares its stdout).""" + cpu_s: float | None = None + + +class RenderTimings(BaseModel): + """Where the request's time went.""" + + model_config = ConfigDict(extra="forbid") + + slot_wait_s: float = Field(ge=0) + """Time spent waiting for the renderer's one render slot.""" + total_s: float = Field(ge=0) + + +class RenderResponse(BaseModel): + """The raw outputs of every theme of one job.""" + + model_config = ConfigDict(extra="forbid") + + job_id: str = Field(pattern=JOB_ID_PATTERN) + outputs: dict[Theme, ThemeRun] + timings: RenderTimings + version: str + mem_available_mb: int | None = None + """`MemAvailable` of the renderer instance after the job, in MiB (None where unreadable).""" + + +class RendererStatus(BaseModel): + """`GET /status`: what the instance is doing and how much memory it has left.""" + + model_config = ConfigDict(extra="forbid") + + name: Literal["anyplot-renderer"] = "anyplot-renderer" + version: str + revision: str | None = None + in_flight: int + waiting: int + stuck: int + """Sandbox launcher processes that did not exit after a kill; while any is alive, renders answer `503 stuck`.""" + mem_available_mb: int | None + sandbox: bool + """Whether the sandbox launcher binary exists and is executable on this instance.""" + volume: bool | None = None + """Whether `RENDERER_RUNS_DIR` is the size-limited volume it must be: checked on Cloud Run + only (None elsewhere); while it is False, renders answer `503 volume_missing`.""" diff --git a/changelog.d/agents-renderer.md b/changelog.d/agents-renderer.md new file mode 100644 index 0000000000..de9c064969 --- /dev/null +++ b/changelog.d/agents-renderer.md @@ -0,0 +1,50 @@ +### Added + +- **The anyplot-renderer service that runs adapted plot code in Cloud Run + sandboxes.** `agents/renderer/` is a small FastAPI service with + `POST /render`, `POST /render/{job_id}/cancel` and `GET /status` behind + Cloud Run IAM and the same ID-token claims check as the agents service. It + runs every theme of a job in a sandbox of its own, one at a time, built on + what spikes S and S2 measured: no `--write` and no egress, an explicit + `PATH`, the run directories on a 512 MiB in-memory volume that the service + requires on Cloud Run, a host watchdog that kills a run past 64 MiB, + 10,000 files or a 512 MiB `MemAvailable` floor and checks the directory + budgets once more after the code exits, a kill that counts only once the + launcher has exited, one retry when the launcher fails before the harness + starts, bounded stdout and stderr, a cleaned probe, and the rlimits 60 s + CPU, 50 MiB per file, 1 GiB address space and 64 processes. `job_id` and + the requested themes are an idempotency key: a replayed request joins the + render in flight or gets the stored answer, never a second run, and + another payload under the same key is refused with `409 job_conflict`. The last caller to leave, or a cancel, + stops the sandbox, an unexpected error answers a JSON `500 internal` + without its message, and a launcher that outlives its kill ends the + instance so Cloud Run starts a fresh one. + It only runs code; the host gates stay in the agents service. Its image + installs the plotting libraries without ADK, CI builds it and renders a + seaborn plot through the harness before merge, and + `agents/renderer/cloudbuild.yaml` builds it, deploys a candidate (creating + the service on the first build, and only when the service is not found), + smokes it with an anonymous call and a real render, and promotes it. (#12115) +- **A `remote` render backend, now the default.** `AGENT_RENDERER=remote` + sends each render to `AGENT_RENDER_URL` with an ID token from the metadata + server, or in development from `AGENT_RENDER_TOKEN` or the developer's + Application Default Credentials, so a local `adk web` and the deployed + agents service render in the same sandboxes. It rides out a cold start by + retrying connection errors and Cloud Run front-end 429 and 5xx answers + after 1, 3, 8 and 15 seconds, asks the renderer to cancel a render it + abandons, and logs every failure it maps to a renderer error. When the + renderer stopped a run at a limit, the repair feedback names the limit and + the measured value. The in-process `sandbox` backend stays as an unused + stub. (#12115) + +### Changed + +- **The probe harness reports its own start and end.** It prints a + `HARNESS` start line before anything else and a closing line with its peak + memory and CPU time, sets every limit soft and hard alike, takes optional + address-space and process limits, and copies the image's baked matplotlib + font cache into the sandbox before matplotlib loads. (#12115) +- **The caller check reads the header Cloud Run verified.** The agents + service and the renderer read `X-Serverless-Authorization` whenever a + request carries it, because Cloud Run then checks only that header and + passes `Authorization` through unverified. (#12115) diff --git a/docs/concepts/agent-network.md b/docs/concepts/agent-network.md index 31511b3282..bf58a54c2b 100644 --- a/docs/concepts/agent-network.md +++ b/docs/concepts/agent-network.md @@ -1,6 +1,6 @@ # Agent network design -> **Status (2026-10-09):** design, with the runtime core built and running locally. Built: the `agents/` package (ADK pin, `AgentSettings` and the Pydantic contracts in `agents/anyplot/schemas.py`); the deterministic data layer in `agents/anyplot/data/` (the parser, data roles, default bindings and the dataset store described under [Parse](#parse)) and `bindings.apply` (`session_state.apply_bindings`, called by `PUT /v1/sessions/{sid}/bindings` and the `set_bindings` tool); the deterministic code layer in `agents/anyplot/code/` (the two-profile AST validator in `validate.py`, plus the protected regions, the canvas normaliser, the readiness scan, the edit applier, the loader and the export in `regions.py`, `normalise.py`, `readiness.py`, `edits.py`, `loader.py` and `export.py`); the runtime core (the root agent, the adapters and the reviewer, the plot pipeline, the ScopeGuard, Budget and ToolSafety plugins, the render layer with the probe harness, the host gates and the `fake` and `local` Docker backends, the private `/v1` service with its `anyplot/1` stream, the run queue in front of whole pipeline runs, and one-theme renders with an on-demand theme toggle; see `agents/README.md`); the `/debug/agent` BFF router in `api/routers/agent.py` (shipped dark behind `AGENT_ENABLED`); and two shared building blocks, `core/canvas.py` (the canvas gate and the PNG auto-reject checks) and `core/defects.py` (the review feedback grammar). Every agent runs on Claude Haiku 5.5 on Vertex AI by default, with Gemini 3.8 Flash as the second arm behind `AGENT_PROVIDER` (see the Model row under [Goals and fixed decisions](#goals-and-fixed-decisions)). Not built: the `sandbox` render backend (it waits for spike S), the agents image, the deploy, `core/catalogue`, the regression harness and the evals, quick-feedback storage and triage, the React page and the button, the analytics events, and everything in phase 2. The research behind it was verified against ADK v2.11.0 and the Vertex AI documentation on 2026-10-08; re-check version-sensitive facts (model ids, prices, ADK APIs) before you implement a section. +> **Status (2026-10-10):** design, with the runtime core built and running locally and the renderer service built but not deployed. Built: the `agents/` package (ADK pin, `AgentSettings` and the Pydantic contracts in `agents/anyplot/schemas.py`); the deterministic data layer in `agents/anyplot/data/` (the parser, data roles, default bindings and the dataset store described under [Parse](#parse)) and `bindings.apply` (`session_state.apply_bindings`, called by `PUT /v1/sessions/{sid}/bindings` and the `set_bindings` tool); the deterministic code layer in `agents/anyplot/code/` (the two-profile AST validator in `validate.py`, plus the protected regions, the canvas normaliser, the readiness scan, the edit applier, the loader and the export in `regions.py`, `normalise.py`, `readiness.py`, `edits.py`, `loader.py` and `export.py`); the runtime core (the root agent, the adapters and the reviewer, the plot pipeline, the ScopeGuard, Budget and ToolSafety plugins, the render layer with the probe harness, the host gates and the `fake` and `local` Docker backends, the private `/v1` service with its `anyplot/1` stream, the run queue in front of whole pipeline runs, and one-theme renders with an on-demand theme toggle; see `agents/README.md`); the `anyplot-renderer` service in `agents/renderer/` (the sandbox executor with the kill path, the run watchdog and the launcher retry from spikes S and S2, one render slot, cancellation on disconnect, the caller check, its image, its CI image job and its Cloud Build config) and the `remote` render backend that calls it (see [Render](#render)); the `/debug/agent` BFF router in `api/routers/agent.py` (shipped dark behind `AGENT_ENABLED`); and two shared building blocks, `core/canvas.py` (the canvas gate and the PNG auto-reject checks) and `core/defects.py` (the review feedback grammar). Every agent runs on Claude Haiku 5.5 on Vertex AI by default, with Gemini 3.8 Flash as the second arm behind `AGENT_PROVIDER` (see the Model row under [Goals and fixed decisions](#goals-and-fixed-decisions)). Not built: the agents image, the deploy of either service, `core/catalogue`, the regression harness and the evals, quick-feedback storage and triage, the React page and the button, the analytics events, and everything in phase 2. The research behind it was verified against ADK v2.11.0 and the Vertex AI documentation on 2026-10-08; re-check version-sensitive facts (model ids, prices, ADK APIs) before you implement a section. This document describes the agent network that lets a visitor paste their own data on a plot page and get that plot adapted, rendered and reviewed ("Use with my data"), and later helps them find the right plot type. It is built with [Google ADK](https://adk.dev) (Python) on Vertex AI (now branded Gemini Enterprise Agent Platform) inside the `anyplot` GCP project: on Claude Haiku 5.5 by default, with Gemini 3.8 Flash as the second arm. @@ -15,7 +15,7 @@ This document describes the agent network that lets a visitor paste their own da | Data input | Paste text only (CSV, TSV, semicolon CSV, JSON), hard cap 200 KB | | Access | Visible only in local development and for admins, gated exactly like `/debug/*`; not public in phase 1 | | Guardrail scope | Allowed: choosing plots, the user's data for them, adapting, rendering and styling, and questions about catalogue code (styling, exports). Everything else gets a fixed refusal | -| Runtime | A separate Cloud Run service. The managed Vertex code-execution sandbox is Python/JS only with a fixed library set (no kaleido, altair, plotnine, pygal, lets-plot, R, Julia, or Chromium), so it cannot render the catalogue | +| Runtime | Two Cloud Run services of their own: `anyplot-agents` runs the agents, and `anyplot-renderer` runs the code in Cloud Run sandboxes (owner decision of 2026-10-10, so a local `adk web` and the deployed agents service share one renderer). The managed Vertex code-execution sandbox is Python/JS only with a fixed library set (no kaleido, altair, plotnine, pygal, lets-plot, R, Julia, or Chromium), so it cannot render the catalogue | | Model | Owner decision of 2026-10-09: Claude Haiku 5.5 (`claude-haiku-5-5`) on Vertex AI, location `eu`, is the phase-1 default for every agent and for the scope and dataset judge (`AGENT_PROVIDER=anthropic-vertex`). Gemini 3.8 Flash (`gemini-3.8-flash`, with the judge on `gemini-3.5-flash-lite`) stays the second arm behind `AGENT_PROVIDER=gemini`, so the two can be compared on price and quality. The judge follows the provider: `AGENT_JUDGE_MODEL` must be a model of the same family. Every model id is an exact pin, never an alias such as `gemini-flash-latest`: that alias is undocumented on Vertex, its served version is unobservable, and it returned 404 once when its target retired. "3.8" in this document always means the Gemini model, never a Python version | | Python | Python 3.13 everywhere, the same as the pipeline | | Location | The `eu` multi-region from day one. No Gemini 3.x Flash is served in europe-west4; `eu` keeps ML processing in the EU at +10 % price and is compatible with the privacy page's EU claim | @@ -42,13 +42,17 @@ This document describes the agent network that lets a visitor paste their own da Browser ──fetchWithAuth──► anyplot-api /debug/agent/* (require_admin → AdminIdentity, CSRF header, AGENT_ENABLED kill switch) │ ID token (roles/run.invoker) + X-Anyplot-User + X-Request-Id ▼ - anyplot-agents (Cloud Run gen2, --sandbox-launcher, --no-allow-unauthenticated, own service account) + anyplot-agents (Cloud Run gen2, --no-allow-unauthenticated, own service account) FastAPI /v1/* → run queue (1 in flight, 1 start a minute, 600 s maximum wait) → Runner(app=App(plugins=[ScopeGuard, Budget, ToolSafety, ContextFilter(6)])) root "anyplot" ── session tools ── plot_pipeline (Workflow NodeTool, input_schema=PipelineArgs) - @node run_pipeline: adapter → apply + validate → render, one theme (sandbox do) → gates + @node run_pipeline: adapter → apply + validate → render, one theme → gates → reviewer (≤1) → repair (≤1) → Event(output=PlotResult) POST /v1/.../versions/{v}/render → render the other theme → gates (no model call, no queue) + │ every render: `remote` backend, ID token (roles/run.invoker) adk web on a developer machine + ▼ │ the same `remote` backend + anyplot-renderer (Cloud Run gen2, --sandbox-launcher, --no-allow-unauthenticated, zero-role account) ◄┘ + FastAPI POST /render → one render slot → per theme: sandbox do (gVisor, no egress) → raw outputs ``` Only one agent talks to the user. Everything the research shows an LLM does not improve (parsing, default bindings, the data loader, the canvas, safety checks, export) is plain code. @@ -139,7 +143,7 @@ The owner decided the queue, the rate and the serial renders on 2026-10-09, afte | Renders per run | 2 rounds of one theme (`PipelineArgs.theme`, `light` by default) | the pipeline loop | | `plot_pipeline` calls per invocation | 1 | `ToolSafety` | | LLM calls per request | 12: `RunConfig(max_llm_calls=12, streaming_mode=StreamingMode.NONE)`. `ADK_MAX_LLM_CALLS=20` only sets the default for runs without a `RunConfig`, such as `adk web` and evals | the budget plugin and the `RunConfig` | -| Render time | 60 s per render, host-enforced and clamped to the remaining soft deadline | the render backend | +| Render time | 60 s per render, host-enforced and clamped to the remaining soft deadline | the pipeline's clamp and the renderer's kill path (`timeout_s` per theme) | | Request deadline | 180 s through `abort_signal` plus task cancellation on disconnect, counted from the moment the run leaves the queue | the `/messages` route | | Runs in flight per instance | 1 (`AGENT_RUN_CONCURRENCY`) | the run queue | | Run starts | fewer than `AGENT_RUNS_PER_MINUTE` (1) in the last 60 s, a sliding window of start times | the run queue | @@ -179,7 +183,7 @@ Callable only with Cloud Run IAM; the BFF mirrors it under `/debug/agent`. | `DELETE /v1/sessions/{sid}` | Purges the session, the dataset store and the render store | | | any | `404 session_expired` after scale-to-zero or the idle sweeper | | -Headers: `X-Anyplot-User` (the HMAC id from the BFF) and `X-Request-Id`. Caller check: Cloud Run IAM verifies the ID token and forwards the `Authorization` header with the signature replaced, so the app decodes the claims without signature verification and requires `aud` in `{AGENT_SERVICE_URL, candidate-tag URL}` and `email` in `AGENT_ALLOWED_CALLERS`. A missing header is rejected when `ENVIRONMENT=production`; the check is skipped only in development. This is weak while anyplot-api runs as the shared compute service account; a dedicated API service account is an owner task before public use. +Headers: `X-Anyplot-User` (the HMAC id from the BFF) and `X-Request-Id`. Caller check: Cloud Run IAM verifies the ID token and forwards it with the signature replaced, so the app decodes the claims without signature verification and requires `aud` in `AGENT_SERVICE_URLS` and `email` in `AGENT_ALLOWED_CALLERS`. Cloud Run requires the service URL as `aud` even for a request to a tag URL, so `AGENT_SERVICE_URLS` holds the service URL, and a deploy smoke mints its token for that URL while it calls the `candidate` tag. When a request carries `X-Serverless-Authorization`, Cloud Run checks only that header and passes `Authorization` through unverified, so the app then reads `X-Serverless-Authorization` and ignores `Authorization`. A missing header is rejected when `ENVIRONMENT=production`; the check is skipped only in development. This is weak while anyplot-api runs as the shared compute service account; a dedicated API service account is an owner task before public use. ## Contracts @@ -201,18 +205,45 @@ Schemas live in `agents/anyplot/schemas.py`: `ColumnProfile{name≤64, dtype, mi ### Render -`agents/anyplot/render/` defines `RenderJob{job_id, language, library, source, data_csv, themes, timeout_s}` and two protocols: `RuntimeAdapter` (`file_name`, `command` as the CI command verbatim, `env`, `loader_block`, `normalise`, `validate`, `collect`) and `RenderBackend` with the backends `sandbox` (phase 1), `local` (development only; refuses `ENVIRONMENT=production`; always `docker run --rm --network none --read-only --tmpfs /tmp -v :/work /app/.venv/bin/python -I /opt/anyplot/harness.py plot.py` with the production image, fails if Docker is missing, no bare-subprocess fallback), `fake` (unit tests, fixture PNGs) and `remote` (the phase-2 renderer split). +`agents/anyplot/render/` defines `RenderJob{job_id, language, library, source, data_csv, themes, timeout_s}` and two protocols: `RuntimeAdapter` (`file_name`, `command` as the CI command verbatim, `env`, `loader_block`, `normalise`, `validate`, `collect`) and `RenderBackend` with the backends `remote` (phase 1: the `anyplot-renderer` service, `AGENT_RENDERER=remote` with `AGENT_RENDER_URL`), `sandbox` (sandboxes launched inside the agents service; a stub that phase 1 does not use), `local` (development only; refuses `ENVIRONMENT=production`; always `docker run --rm --network none --read-only --tmpfs /tmp -v :/work /app/.venv/bin/python -I /opt/anyplot/harness.py plot.py` with the production renderer image, fails if Docker is missing, no bare-subprocess fallback) and `fake` (unit tests, fixture PNGs). -The sandbox command per theme, started with `asyncio.create_subprocess_exec` and never a shell: +The renderer is a service of its own (owner decision of 2026-10-10), so the deployed agents service and a developer's `adk web` render in the same sandboxes. `agents/renderer/` has no ADK import and no `__init__.py` (`adk web agents` would list it as an agent). `POST /render` takes the `RenderJob` as JSON and answers per theme the exit code, a timeout flag, a stderr tail of at most 2,000 characters, the probe, the PNG as base64 (at most 10 MB), the attempts and a reason (`timeout`, `disk_budget`, `file_budget`, `memory`, `launcher` or `output_rejected`) with the measured value and the limit, plus the slot wait, the total time and `MemAvailable`; `POST /render/{job_id}/cancel` stops that job's render; `GET /status` reports the version, the slot, stuck launchers, `MemAvailable`, whether the launcher exists and whether the run volume is in place. `agents/renderer/wire.py` is the JSON contract that both the service and the `remote` backend read. The renderer only runs code and returns its outputs; the host gates and the probe gates below stay in the agents service, and R1 turns a reason into a line that names the limit and the measured value, so the single repair knows what to change. Every route needs an IAM-forwarded ID token whose `aud` is in `RENDERER_AUDIENCES` (the service URL, which the deploy adds, plus the OAuth client ids of the owner's user tokens) and whose `email` is in `RENDERER_ALLOWED_CALLERS`; the check is the one of `/v1`, reads `X-Serverless-Authorization` whenever it is present, and is skipped only in development. An unexpected error answers `500 {"detail": "internal"}` and is logged by its class only, so the `remote` backend never mistakes it for a front-end error and runs the job again. + +The sandbox command per theme, started with `asyncio.create_subprocess_exec` in a session of its own and never a shell, with a fresh name and a fresh run directory for every launch: ``` -/usr/local/gcp/bin/sandbox do --sandbox-name r-- --write \ - --mount type=bind,source=/tmp/runs/,destination=/work -w /work \ - --env ANYPLOT_THEME= --env MPLBACKEND=Agg --env MPLCONFIGDIR=/opt/mplconfig --env HOME=/tmp \ +/usr/local/gcp/bin/sandbox do --sandbox-name r-- \ + --mount type=bind,source=/tmp/runs/r--/work,destination=/work -w /work \ + --env PATH=/app/.venv/bin:/usr/local/bin:/usr/bin:/bin --env HOME=/tmp --env ANYPLOT_THEME= \ + --env MPLBACKEND=Agg --env MPLCONFIGDIR=/tmp/mpl --env ANYPLOT_MPL_SEED=/opt/mplconfig \ + --env ANYPLOT_RLIMIT_CPU_S=60 --env ANYPLOT_RLIMIT_FSIZE_MB=50 --env ANYPLOT_RLIMIT_AS_MB=1024 --env ANYPLOT_RLIMIT_NPROC=64 ... \ -- /app/.venv/bin/python -I /opt/anyplot/harness.py plot.py ``` -A run renders one theme, the one `PipelineArgs.theme` names, and the host gates judge exactly the themes of the job (an output for another theme is ignored, a missing one fails R1). The theme toggle renders the other theme of a finished version later, as a job of its own with the same code and `data.csv`, and adds its PNG to the version's render. Renders are strictly serial per instance, because spikes S and S2 showed one 4 GiB instance serves one sandbox at a time safely. No backend keeps a semaphore of its own: `Services.backend` wraps whichever backend `make_backend` built in `SerialRenderer` (`render/serial.py`), which renders one theme per slot under `AGENT_RENDER_CONCURRENCY` slots, 1 by default, so the `sandbox`, `local` and `fake` backends are all serial by construction, `adk web` runs included. A freed slot goes to a waiting pipeline render before a waiting theme toggle, so a run waits for at most the render in progress; a render's timeout starts when it runs. The theme toggle records a theme that failed the host gates and renders it at most twice; a renderer that could not run is not recorded. The process has its own group and is killed on timeout, followed by `sandbox delete r--`; the run directory is wiped; a code assertion forbids `--allow-egress`. `harness.py` sets resource limits (CPU and file size; the address-space limit is tuned in the spike), registers a `Figure.savefig` probe that writes `probe-.json`, then calls `runpy.run_path`. +What spikes S and S2 measured in europe-west4, and what the executor (`agents/renderer/executor.py`) does about it: + +| Fact | Consequence | +|---|---| +| The sandbox is gVisor with no egress, no metadata server and only `lo`; `--write` makes its root filesystem writable, and with it a nested egress attempt got one step further | No `--write` and no `--allow-egress`, asserted in code; renders work without `--write` | +| A sandbox's environment holds only `HOME`, `LC_CTYPE` and the `--env` values, with no `PATH` | `PATH` is passed explicitly, with the rest of a complete environment | +| The code runs as uid 0; the image and every host file outside `/tmp` are readable from inside, sibling run directories are not, and each sandbox gets a private `/tmp` tmpfs of about 1.9 GiB | Every run directory and every user byte lives under `/tmp/runs/`; the image holds no secret; the harness copies the baked font cache from `/opt/mplconfig` to `/tmp/mpl` | +| A disk fill through `/work` on the plain filesystem crashed the whole instance at about 3.4 GB | `/tmp/runs` is a 512 MiB in-memory volume, which the service checks at startup on Cloud Run (`503 volume_missing` without it), and a host watchdog measures each live run directory every 50 ms and kills the sandbox above 64 MiB (`disk_budget`) or 10,000 entries (`file_budget`); a tree deeper than 32 levels or past `PATH_MAX`, which a walk by path cannot measure, counts as over its limit | +| Each sandbox's private `/tmp` counts as instance memory, and no flag bounds it | The same watchdog reads `MemAvailable` and kills the sandbox below 512 MiB (`memory`); the remedy for the budget itself is an open decision | +| `RLIMIT_CPU`, `RLIMIT_FSIZE`, `RLIMIT_AS` and `RLIMIT_NPROC` are enforced inside; the defaults bound nothing | The harness sets 60 s, 50 MiB, 1 GiB and 64, soft and hard alike; the host keeps its wall-clock timeout | +| `sandbox delete --force` exits 0 even when it did nothing | On a timeout, the watchdog, a cancellation or an unexpected error: `os.killpg` with SIGKILL, then `sandbox delete --force `; the kill counts only when the launcher process exits, and one that does not exit within 10 s is `stuck`, which refuses further renders with `503 stuck` and, on Cloud Run, ends the service's process so a fresh instance replaces it. A launcher that exits non-zero on its own also gets a `delete` | +| One of 26 renders ended in 0.4 s with `failed to exec in container ... exit status 137` and no output | A non-zero exit without the harness's `HARNESS {"event": "start"}` line is retried once with a new sandbox name; a second one is reported as `launcher`; a failure after the harness started goes to the repair | +| The cgroup's `memory.current` is unreadable | `MemAvailable` from `/proc/meminfo` is logged with every render and refuses a render below 1,024 MiB (`503 low_memory`) | +| Many concurrent sandboxes crash a 4 GiB instance; one is safe | One render slot per renderer, themes one after the other, a 30 s wait for the slot (`503 busy`) | +| `/healthz` never reaches the container | No health route; Cloud Run's TCP startup probe | +| matplotlib renders in about 2.5 s and seaborn in about 3.3 s at p95, at 2 vCPU | `AGENT_RENDER_TIMEOUT_S` (60) is far above a normal render | + +stdout and stderr are read while the sandbox runs and kept bounded (the first and last 4 KiB of stdout, the last 64 KiB of stderr, of which 2,000 characters go back), and the outputs are read with `O_NOFOLLOW` and `fstat`, so a symlink, a FIFO or an oversized file is refused (`output_rejected`). The probe is written by the code under test, so the renderer drops one nested deeper than 16 levels or holding a number a float cannot carry, and turns `NaN` into null; the harness's own peak-memory and CPU line is taken only as finite numbers in range. An `OSError` on the host (a full volume, a failed fork) is `503 io`, never a plot failure. The log carries ids, exit codes, reasons and timings, never code, data, stderr or images; `/var/log/sandbox.log` on the host records every launcher call and is the source of truth for launcher metrics. + +uvicorn does not cancel a request handler whose client went away, so `POST /render` runs the render as a task, races it against the connection's `http.disconnect`, and cancels it when the caller is gone; the executor's kill path then stops the sandbox and frees the slot. Because Cloud Run's front end may not pass a closed connection on, the `remote` backend also posts `/render/{job_id}/cancel` when its own render is cancelled (an abort, the request deadline) or its answer does not come in time. `job_id` and the requested themes are an idempotency key (the backend sends one request per theme under the job's id): the backend's retry after a connection error or a front-end 429 or 5xx may replay a request that did reach the renderer, so a replay with the same payload joins the render in flight or gets the stored answer of a finished one (the last 16), and another payload under the same key is refused with `409 job_conflict`; a job's code never runs twice because of a retry, and the last caller to leave is the one whose departure cancels the render. + +The `remote` backend mints the ID token for `AGENT_RENDER_URL`, the renderer's service URL, from the metadata server on Cloud Run; in development it sends `AGENT_RENDER_TOKEN` when set, or the `id_token` of the developer's Application Default Credentials (`agents/README.md`). It retries a connection error or a Cloud Run front-end 429 or 5xx without the renderer's JSON `detail` after 1, 3, 8 and 15 s within 30 s (spike S saw a 500, then a 429, and a 200 about 17 s after the first request to a service scaled to zero), never the renderer's own refusals. A `launcher` reason becomes `RendererUnavailable`, because it is the renderer's failure and not the code's, so it spends no repair and the theme toggle records no try; every other failure maps to `RendererUnavailable` too, and its message is logged. + +A run renders one theme, the one `PipelineArgs.theme` names, and the host gates judge exactly the themes of the job (an output for another theme is ignored, a missing one fails R1). The theme toggle renders the other theme of a finished version later, as a job of its own with the same code and `data.csv`, and adds its PNG to the version's render. Renders are strictly serial per instance, because spikes S and S2 showed one 4 GiB instance serves one sandbox at a time safely. No backend keeps a semaphore of its own: `Services.backend` wraps whichever backend `make_backend` built in `SerialRenderer` (`render/serial.py`), which renders one theme per slot under `AGENT_RENDER_CONCURRENCY` slots, 1 by default, so every backend is serial by construction, `adk web` runs included; the `remote` backend therefore sends one theme per request. The renderer holds one render slot of its own on top, because the deployed agents service and a developer's `adk web` call it independently. A freed slot goes to a waiting pipeline render before a waiting theme toggle, so a run waits for at most the render in progress; a render's timeout starts when it runs. The theme toggle records a theme that failed the host gates and renders it at most twice; a renderer that could not run is not recorded. `harness.py` (copied into the renderer image at `/opt/anyplot/harness.py`) prints its `HARNESS` start line, sets the resource limits soft and hard alike (CPU seconds and file size always; the address space and the process count when the renderer passes them, because outside a sandbox `RLIMIT_NPROC` counts every process of the user), copies the baked font cache into `MPLCONFIGDIR`, registers a `Figure.savefig` probe that writes `probe-.json`, calls `runpy.run_path`, and prints its peak memory and CPU time in a closing `HARNESS` line. Host gates: R1 (exit code and outputs present) and R2 (PNG hardening: `lstat` with no symlinks, at most 10 MB, magic bytes, Pillow under `MAX_IMAGE_PIXELS`, non-blank with less than 98 % background, re-encode) are blocking; a render that fails either is discarded and never becomes `best`. R3 (canvas within 16 px through `core/canvas.py`, lifted from `.github/workflows/impl-review.yml`) is repair-triggering with a fallback: an R3 miss after normalisation is a defect line for the single repair; if it persists after the repair, the host pads the PNG of the rendered theme to the target canvas (never crops), the padded artifact counts as having passed the host gates, and the result ships as `needs_attention` with the residual line `canvas padded after render ()` and the VQ-05 line for that theme, never silently. A job with both themes pads each theme that missed and keeps the other theme's PNG as it rendered. The theme toggle answers a padded theme as `needs_attention` with the reason `canvas_padded`. The probe gates (G3 clipping, G5 annotation out of view, G7 tick-label overlap, G8 row fidelity) are advisory because in-sandbox data can be tampered with: they can only trigger the single repair or add notes, never fail a render. The defect-line helpers (`DEFECT_RE`, `weakness_class`, `defect_ids`, and the `format_defect` builder) live in `core/defects.py`, the canonical stdlib-only implementation, and the PNG auto-reject checks (AR-04 blank and AR-07 format; R2 calls `check_blank` with its 98 % threshold) live next to the canvas gate in `core/canvas.py`, so the agents image needs no `automation/` or `scripts/` files. `automation/scripts/regen_gate.py` keeps a parity-tested copy of the grammar, because the workflows run it as a single-file copy with the runner's system Python, where `core` is not importable. The pipeline switches to `core.defects` in the later change that also switches the canvas gate and updates the copy sites: the `cp` step in `impl-review.yml`, the `git show` in `impl-generate.yml`, `review_retest.py materialize`, and the overlay in `review-retest.yml`. @@ -238,7 +269,7 @@ Plugin order on `App`, where the first non-None result wins and a plugin never r | Off-topic use, jailbreak, multi-turn drift | A judge on every free-text turn that fails closed, the 2,000-character limit, rejected text never stored, a closed tool set, policy in an untemplated `static_instruction`, the root "never" list, adversarial evals including general-programming requests framed as plot-code questions | | Injection through pasted data | The raw dataset never reaches a prompt or a tool argument; only bounded, sanitised, fenced samples do: at most 3 KB of headers, top values and sample cells for the dataset judge at parse time, and the profile's at most 5 sample rows (cells at most 40 characters) for the root and the adapter; canonical headers; bindings validated on the server; adapter output bound to a schema and linted | | Injection through catalogue code, comments or image labels | Own fencing; validated adapter output; a tool-less reviewer with a fixed-id schema that can trigger at most one bounded repair | -| Code escape | The two-profile AST validator; loader substitution; no executors anywhere; the sandbox (no egress, environment or metadata server, read-only root filesystem); resource limits; a host timeout plus `sandbox delete`; per-job directory wipe; a service account with no data access; the phase-0 probe suite and the sibling-directory spike | +| Code escape | The two-profile AST validator; loader substitution; no executors anywhere; the sandbox (no egress, environment or metadata server, read-only root filesystem) in a renderer service of its own with a zero-role service account; resource limits; a host timeout plus a confirmed kill and `sandbox delete`; a run watchdog (bytes, files and `MemAvailable`) on an in-memory, size-limited run volume that the service requires; per-job directory wipe; the phase-0 probe suite and the sibling-directory spike | | Resource abuse (denial of wallet) | Ledger budgets including the judge; the run queue (one run in flight, one start a minute, at most 10 waiting); one run per user and one `plot_pipeline` per invocation; `max-instances=1`; a CSRF header and Origin check on the cookie-authenticated BFF; a spend cap on `aiplatform.googleapis.com` only (never on `run.googleapis.com`, which would pause `anyplot-api`); the `AGENT_ENABLED` kill switch | | Data leakage | A server-derived HMAC user id; ownership checks on every route; no content in logs or spans (`ADK_CAPTURE_MESSAGE_CONTENT_IN_SPANS=false`, `OTEL_INSTRUMENTATION_GENAI_CAPTURE_MESSAGE_CONTENT` unset); in-memory stores tied to the session lifetime plus an idle sweeper and a DELETE route; the translator drops arguments, tool outputs, thoughts and `error_details`; the `eu` endpoint; PNG only in the UI, no model HTML, no auto-loaded URLs | | Session poisoning or BFF bypass | Our own FastAPI, never `get_fast_api_app`; clients never write state or events; the BFF allowlists bodies; the agents app checks the IAM-forwarded ID-token claims; `ToolCallIntegrityPlugin` once sessions persist | @@ -267,9 +298,16 @@ Testing a newer model version, the other provider's arm, a different judge model ## Serving and infrastructure -- **`anyplot-agents`**, a new Cloud Run service deployed by `agents/cloudbuild.yaml` using the API's candidate-then-smoke-then-promote pattern with `--update-*` flags only. A one-time bootstrap deploy without `--no-traffic` creates the service, because gcloud rejects `--no-traffic` on a new service; the API side stays dark through `AGENT_ENABLED=false`. Flags: `gcloud beta run deploy anyplot-agents --region=europe-west4 --execution-environment=gen2 --sandbox-launcher --service-account=anyplot-agents@anyplot.iam.gserviceaccount.com --no-allow-unauthenticated --cpu=2 --memory=4Gi --min-instances=0 --max-instances=1 --concurrency=20 --timeout=900 --no-traffic --tag=candidate --update-env-vars=^|^...`. No secrets and no Cloud SQL in phase 1. The concurrency and the timeout follow the run queue, not the memory: the queue, not Cloud Run, limits the runs in memory, but every queued run holds an open stream, so the instance must accept the queue's 10 waiting streams, the run in flight and the other routes (`--concurrency=20`), and one turn can last the 600 s maximum wait plus the 180 s deadline (`--timeout=900`). The FastAPI app in `agents/main.py` sets `docs_url=None`. The service hosts both the runner and the sandbox launcher; splitting out a zero-role `anyplot-renderer` (the `remote` backend) is a hard gate before any non-owner use. -- **IAM** (owner tasks): the service account `anyplot-agents@` gets `roles/aiplatform.user` and `roles/telemetry.writer`; the API's runtime identity gets `roles/run.invoker` on the service; the Cloud Build identity gets `roles/iam.serviceAccountUser` on `anyplot-agents@`; the GitHub Workload Identity Federation principal gets `roles/aiplatform.user` for evals. Phase 2 adds `cloudsql.client` with a read-only role `anyplot_agents_ro` (SELECT on specs, impls, libraries and languages only, never `feedback`), `secretAccessor` on single secrets, `storage.objectAdmin` on the private bucket, and `modelarmor.user`. -- **Environment on anyplot-agents:** `ENVIRONMENT=production`, `GOOGLE_CLOUD_PROJECT=anyplot`, `GOOGLE_GENAI_USE_ENTERPRISE=TRUE`, `GOOGLE_CLOUD_LOCATION=eu`, `AGENT_LOCATION=eu` (never europe-west4), `AGENT_PROVIDER=anthropic-vertex`, `AGENT_MODEL=claude-haiku-5-5`, `AGENT_JUDGE_MODEL=claude-haiku-5-5` (the Gemini arm: `AGENT_PROVIDER=gemini`, `AGENT_MODEL=gemini-3.8-flash`, `AGENT_JUDGE_MODEL=gemini-3.5-flash-lite`), `AGENT_LIBRARIES=matplotlib,seaborn`, `AGENT_RENDERER=sandbox`, `AGENT_MAX_LLM_CALLS=12`, `ADK_MAX_LLM_CALLS=20`, `AGENT_REQUEST_TOKEN_BUDGET=80000`, `AGENT_DAILY_TOKEN_BUDGET=1000000`, `AGENT_DAILY_PIPELINE_RUNS=40`, `AGENT_GLOBAL_DAILY_TOKEN_BUDGET=3000000`, `AGENT_RUN_CONCURRENCY=1`, `AGENT_RUNS_PER_MINUTE=1`, `AGENT_QUEUE_MAX_WAIT_S=600`, `AGENT_RENDER_CONCURRENCY=1`, `AGENT_RENDER_TIMEOUT_S=60`, `AGENT_REQUEST_DEADLINE_S=180`, `AGENT_SOFT_DEADLINE_S=140`, `AGENT_ALLOWED_CALLERS=`, `ADK_CAPTURE_MESSAGE_CONTENT_IN_SPANS=false`. On anyplot-api: `AGENT_ENABLED=false` (ships dark; the routes answer 404), `AGENT_SERVICE_URL`, and `AGENT_USER_ID_KEY` from Secret Manager (the BFF also answers 404 while the key is unset, so a deploy never breaks). On the app: `VITE_ENABLE_AGENT_CHAT`, a build-time flag that tree-shakes the chunk. +- **`anyplot-agents`**, a new Cloud Run service deployed by `agents/cloudbuild.yaml` using the API's candidate-then-smoke-then-promote pattern with `--update-*` flags only. A one-time bootstrap deploy without `--no-traffic` creates the service, because gcloud rejects `--no-traffic` on a new service; the API side stays dark through `AGENT_ENABLED=false`. Flags: `gcloud run deploy anyplot-agents --region=europe-west4 --execution-environment=gen2 --service-account=anyplot-agents@anyplot.iam.gserviceaccount.com --no-allow-unauthenticated --cpu=2 --memory=4Gi --min-instances=0 --max-instances=1 --concurrency=20 --timeout=900 --no-traffic --tag=candidate --update-env-vars=^|^...`. No secrets and no Cloud SQL in phase 1. The concurrency and the timeout follow the run queue, not the memory: the queue, not Cloud Run, limits the runs in memory, but every queued run holds an open stream, so the instance must accept the queue's 10 waiting streams, the run in flight and the other routes (`--concurrency=20`), and one turn can last the 600 s maximum wait plus the 180 s deadline (`--timeout=900`). The FastAPI app in `agents/main.py` sets `docs_url=None`. The service runs no code of its own users: it has no `--sandbox-launcher`, and every render goes to `anyplot-renderer` through the `remote` backend. Its 2 vCPU and 4 GiB were sized when it still hosted the sandboxes; a smaller instance is an open decision. +- **`anyplot-renderer`**, a phase-1 Cloud Run service that runs the code (`agents/renderer/`), deployed by `agents/renderer/cloudbuild.yaml` with the same candidate, smoke and promote steps. Its deploy step creates the service on the first build, because gcloud refuses `--no-traffic` for a service that does not exist yet; it does so only when `services describe` reports the service as not found, and stops on any other describe error, so a permission or API error never sends all traffic to an unsmoked revision. Flags: `gcloud beta run deploy anyplot-renderer --region=europe-west4 --execution-environment=gen2 --sandbox-launcher --no-allow-unauthenticated --service-account=anyplot-renderer@anyplot.iam.gserviceaccount.com --cpu=2 --memory=4Gi --min-instances=${_MIN_INSTANCES} --max-instances=1 --concurrency=4 --timeout=300 --add-volume=name=runs,type=in-memory,size-limit=512Mi --add-volume-mount=volume=runs,mount-path=/tmp/runs --no-traffic --tag=candidate --update-env-vars=^|^ENVIRONMENT=production|RENDERER_ALLOWED_CALLERS=...|RENDERER_AUDIENCES=...`, where `_MIN_INSTANCES` is 0 unless a build sets 1 to keep an instance warm. One instance with one render slot, because renders are serial; `--concurrency=4` so a second caller gets the slot's `503 busy` after 30 s instead of a Cloud Run 429, and a cancel reaches the instance while a render holds the slot; the run directories on the in-memory volume (see [Render](#render)). No public domain, no secrets, no Cloud SQL and no role: the service account exists only to be an identity. The process runs as root, because `sandbox do` creates the gVisor sandboxes; the model-written code runs inside them. The smoke step checks that IAM refuses an anonymous caller (403) and, with `_SMOKE_RENDER=true`, checks that `/status` reports the launcher and the volume and renders a matplotlib plot through the candidate as the service's own account, which proves the launcher, the volume and the harness together. The build impersonates `anyplot-renderer@` for that token (`gcloud auth print-identity-token --impersonate-service-account`), because a build cannot mint an ID token for its own identity: gcloud refuses `--include-email` for it and Cloud Build's metadata server has no `identity` endpoint (builds `da6f53f5` and `a9d3ac7f` of 2026-10-10). The token is minted for the service URL while the requests go to the `candidate` tag URL, because Cloud Run accepts no other audience. `.github/workflows/ci-image.yml` builds the same image before merge and renders a seaborn plot through its harness. + +| Service | Runs | Called by | Identity and roles | +|---|---|---|---| +| `anyplot-api` | The site API and the `/debug/agent` BFF | Browsers, through Cloudflare | The default compute service account (a dedicated one before public use) | +| `anyplot-agents` | The ADK runner, the run queue, the gates, the stores | `anyplot-api` (`roles/run.invoker`) | `anyplot-agents@`: `roles/aiplatform.user`, `roles/telemetry.writer` | +| `anyplot-renderer` | Code in Cloud Run sandboxes, one at a time | `anyplot-agents`, the owner's `adk web`, the deploy smoke (`roles/run.invoker`) | `anyplot-renderer@`: no role | +- **IAM** (owner tasks): the service account `anyplot-agents@` gets `roles/aiplatform.user` and `roles/telemetry.writer`; the API's runtime identity gets `roles/run.invoker` on the service; the Cloud Build identity gets `roles/iam.serviceAccountUser` on `anyplot-agents@`; the service account `anyplot-renderer@` gets no role; `anyplot-agents@`, the owner's account and `anyplot-renderer@` itself (the render smoke calls the candidate as that account) get `roles/run.invoker` on `anyplot-renderer`, and the Cloud Build identity gets `roles/iam.serviceAccountUser` and `roles/iam.serviceAccountTokenCreator` on `anyplot-renderer@` (to deploy as it, and to mint the smoke's token as it); the GitHub Workload Identity Federation principal gets `roles/aiplatform.user` for evals. Phase 2 adds `cloudsql.client` with a read-only role `anyplot_agents_ro` (SELECT on specs, impls, libraries and languages only, never `feedback`), `secretAccessor` on single secrets, `storage.objectAdmin` on the private bucket, and `modelarmor.user`. +- **Environment on anyplot-agents:** `ENVIRONMENT=production`, `GOOGLE_CLOUD_PROJECT=anyplot`, `GOOGLE_GENAI_USE_ENTERPRISE=TRUE`, `GOOGLE_CLOUD_LOCATION=eu`, `AGENT_LOCATION=eu` (never europe-west4), `AGENT_PROVIDER=anthropic-vertex`, `AGENT_MODEL=claude-haiku-5-5`, `AGENT_JUDGE_MODEL=claude-haiku-5-5` (the Gemini arm: `AGENT_PROVIDER=gemini`, `AGENT_MODEL=gemini-3.8-flash`, `AGENT_JUDGE_MODEL=gemini-3.5-flash-lite`), `AGENT_LIBRARIES=matplotlib,seaborn`, `AGENT_RENDERER=remote`, `AGENT_RENDER_URL=`, `AGENT_MAX_LLM_CALLS=12`, `ADK_MAX_LLM_CALLS=20`, `AGENT_REQUEST_TOKEN_BUDGET=80000`, `AGENT_DAILY_TOKEN_BUDGET=1000000`, `AGENT_DAILY_PIPELINE_RUNS=40`, `AGENT_GLOBAL_DAILY_TOKEN_BUDGET=3000000`, `AGENT_RUN_CONCURRENCY=1`, `AGENT_RUNS_PER_MINUTE=1`, `AGENT_QUEUE_MAX_WAIT_S=600`, `AGENT_RENDER_CONCURRENCY=1`, `AGENT_RENDER_TIMEOUT_S=60`, `AGENT_REQUEST_DEADLINE_S=180`, `AGENT_SOFT_DEADLINE_S=140`, `AGENT_ALLOWED_CALLERS=`, `ADK_CAPTURE_MESSAGE_CONTENT_IN_SPANS=false`. On anyplot-renderer: `ENVIRONMENT=production`, `RENDERER_ALLOWED_CALLERS=,[,]`, `RENDERER_AUDIENCES=[,]`, never a tag URL, which Cloud Run does not accept as an audience; the limits keep their defaults (`agents/renderer/settings.py`). On anyplot-api: `AGENT_ENABLED=false` (ships dark; the routes answer 404), `AGENT_SERVICE_URL`, and `AGENT_USER_ID_KEY` from Secret Manager (the BFF also answers 404 while the key is unset, so a deploy never breaks). On the app: `VITE_ENABLE_AGENT_CHAT`, a build-time flag that tree-shakes the chunk. - **Sessions and artifacts.** Phase 1 uses `InMemorySessionService`, `InMemoryArtifactService` and in-memory dataset and render stores; they are consistent because `max-instances=1`, and scale-to-zero shows "session expired". Phase 2 moves to `DatabaseSessionService` in a separate database `anyplot_agents` on `anyplot-db` with its own user and an hourly purge of sessions older than 24 h, because `alembic/env.py` has no `include_object` filter and ADK's `create_all` tables would read as drift; artifacts go to `GcsArtifactService` on a private EU bucket with a 1-day lifecycle, never `anyplot-images`. - **BFF** in `api/routers/agent.py`: `APIRouter(prefix="/debug/agent", dependencies=[Depends(require_admin)])`. A small refactor `require_admin_identity()` returns `AdminIdentity(email|None, via)` and `require_admin` wraps it unchanged. `user_id = "adm_" + HMAC(key, email or "token")[:16]`. POST requests need `Content-Type: application/json`, `X-Anyplot-Client: agent-chat/1` and an allowed Origin. ID tokens come from `google.oauth2.id_token.fetch_id_token` (skipped for localhost). The messages route is an async generator declared with `response_class=EventSourceResponse` (which is what gives FastAPI's automatic 15 s pings; Cloudflare returns 524 after 125 s), reads upstream with httpx `aiter_lines()`, assembles complete events, re-validates each event type against the `anyplot/1` allowlist, and yields `ServerSentEvent(event=..., raw_data=...)`; on upstream failure it emits `error{code:"upstream"}`. Routes mirror `/v1` plus `GET eligibility`, `POST sessions/{sid}/feedback`, and the triage routes `GET /debug/agent/cases`, `GET /debug/agent/cases/{id}` and `PATCH /debug/agent/cases/{id}`. The deploy smoke test expects 401 on `/debug/agent/status`. - **SSE protocol `anyplot/1`**, translated from ADK events and never forwarded raw: `ready{v, run_id}`, sent before the run waits in the queue; `status{step:"queued", position, waiting}` while it waits, at once, on every change and every 15 s unchanged (`position` 1 runs next, `waiting` counts every queued entry, this one included); `status{step, attempt}` for the pipeline's steps; `message{text}` (root final text only), `plot{PlotResult}`, `refusal{code, text}`, `error{code, ref}` with codes `capacity` (also for a run that waited the queue's maximum), `deadline`, `guard_unavailable`, `upstream` and `internal`, and `done{llm_calls, tokens}`. The translator tolerates the ADK 2.x `node_info` and `output` fields. A user already over the daily budget gets `ready`, `refusal{code:"budget"}` and `done` without waiting in the queue. Because queued time does not count toward the agents service's deadline, the BFF restarts its own turn budget (`AGENT_REQUEST_TIMEOUT_S`) on every queued status, with the queue's 15 s heartbeat on top because the run may start that long before its first event, and on the first event after the wait. Nothing moves a turn past `AGENT_TURN_MAX_S` (890 s), which stays below anyplot-api's `--timeout=900`: a turn still queued when its run could no longer finish inside that cap ends with `error{code:"capacity"}`, and closing the upstream stream takes it out of the queue before it spends a token. @@ -287,7 +325,8 @@ agents/anyplot/plugins/{ledger.py, scope_guard.py, budget.py, tool_safety.py} agents/anyplot/prompts/{root,adapter,reviewer,scope_judge,data_judge}.md refusals.yaml agents/anyplot/data/{parse.py, roles.py, bindings.py, store.py} # no ADK import; built agents/anyplot/code/{regions.py, normalise.py, readiness.py, edits.py, validate.py, loader.py, export.py} # no ADK import; built -agents/anyplot/render/{contract.py, gates.py, png.py, harness.py, runtimes/python.py, backends/{local.py, sandbox.py, fake.py}} +agents/anyplot/render/{contract.py, gates.py, png.py, harness.py, serial.py, runtimes/python.py, backends/{remote.py, local.py, sandbox.py, fake.py}} +agents/renderer/{main.py, executor.py, auth.py, settings.py, wire.py, Dockerfile, cloudbuild.yaml} # anyplot-renderer; no ADK import, no __init__.py; built agents/evals/{scope.evalset.json, test_config.json, harness/test_flows.py, matrix.py, sync_cases.py, baselines/{claude-haiku-5-5,gemini-3.8-flash}.json, fixtures/cases//{case.json, data.csv} (synthetic only), .cases/ (gitignored, synced from the bucket)} core/catalogue/{__init__.py, queries.py} # shared by api/mcp/server.py and agents/ @@ -301,7 +340,7 @@ tests/unit/agents/* tests/unit/api/test_agent_router.py tests/integration/agen `agents/anyplot/code/` exists since 2026-10-09: `validate.py` (the two-profile AST validator), `regions.py` (the AST spans of the THEME block, the Imprint palette, the placeholder, the savefig calls and the canvas keywords, shared by the other modules), `normalise.py`, `readiness.py`, `edits.py`, `loader.py` and `export.py`, with tests in `tests/unit/agents/code/`. Their docstrings hold the exact rules. The 13 map specs are the specs tagged both `plot_type: map` and `data_type: geospatial`. `readiness.scan` takes the spec id as a keyword argument, because the map rule needs it and the normalised code no longer carries the catalogue header. -Files to modify: `pyproject.toml` (extras `agents = ["google-adk==2.11.0"]` and `agents-eval = ["google-adk[eval]==2.11.0"]`; setuptools include `agents*`; ruff `known-first-party`; coverage source; regenerate `uv.lock` and run `uv lock --check`; assert that the lock never contains `openai-agents`, which also installs a top-level `agents` module), `.dockerignore` (`!agents`, `!prompts/default-style-guide.md`, `!prompts/library`), `core/config.py` (`agent_*` settings with the pinned defaults, `_strip_secret` entries), `api/routers/debug.py`, `api/main.py`, `api/cloudbuild.yaml` (env vars, secret binding, 401 smoke), `automation/scripts/regen_gate.py` (switch to `core/defects.py`, loaded from a sibling copy in script mode, together with its copy sites), `.github/workflows/{ci-tests,ci-lint,ci-image}.yml`, a new `.github/workflows/agents-eval.yml`, `app/src/routes/{index.tsx,paths.ts}`, `global-config.ts`, `vite-env.d.ts`, `SpecPage.tsx`, `SpecDetailView.tsx`, `DebugPage.tsx`, `app/Dockerfile`, `app/cloudbuild.yaml`, `docs/index.md`, `docs/reference/{api,plausible,mcp,repository,database}.md`, `docs/development.md`, `CLAUDE.md` and `.github/copilot-instructions.md` (an `agents/**` routing row, kept in sync), and one `changelog.d/.md` per PR. `agents/Dockerfile` builds from `python:3.13-slim` with `uv sync --frozen --extra agents --extra lib-matplotlib --extra lib-seaborn`, bakes the matplotlib font cache into `MPLCONFIGDIR=/opt/mplconfig`, runs as a non-root user and starts `uvicorn agents.main:app --port 8080`. The `deploy-agents` trigger (an owner task) fires on `agents/**`, `core/**`, `prompts/library/**`, `prompts/default-style-guide.md`, `pyproject.toml` and `uv.lock`. +Files to modify: `pyproject.toml` (extras `agents = ["google-adk==2.11.0"]` and `agents-eval = ["google-adk[eval]==2.11.0"]`; setuptools include `agents*`; ruff `known-first-party`; coverage source; regenerate `uv.lock` and run `uv lock --check`; assert that the lock never contains `openai-agents`, which also installs a top-level `agents` module), `.dockerignore` (`!agents`, `!prompts/default-style-guide.md`, `!prompts/library`), `core/config.py` (`agent_*` settings with the pinned defaults, `_strip_secret` entries), `api/routers/debug.py`, `api/main.py`, `api/cloudbuild.yaml` (env vars, secret binding, 401 smoke), `automation/scripts/regen_gate.py` (switch to `core/defects.py`, loaded from a sibling copy in script mode, together with its copy sites), `.github/workflows/{ci-tests,ci-lint,ci-image}.yml`, a new `.github/workflows/agents-eval.yml`, `app/src/routes/{index.tsx,paths.ts}`, `global-config.ts`, `vite-env.d.ts`, `SpecPage.tsx`, `SpecDetailView.tsx`, `DebugPage.tsx`, `app/Dockerfile`, `app/cloudbuild.yaml`, `docs/index.md`, `docs/reference/{api,plausible,mcp,repository,database}.md`, `docs/development.md`, `CLAUDE.md` and `.github/copilot-instructions.md` (an `agents/**` routing row, kept in sync), and one `changelog.d/.md` per PR. `agents/Dockerfile` builds from `python:3.13-slim` with `uv sync --frozen --extra agents`, runs as a non-root user and starts `uvicorn agents.main:app --port 8080`; the plotting extras and the baked matplotlib font cache (`/opt/mplconfig`) belong to the renderer image, `agents/renderer/Dockerfile`, which installs `--extra lib-matplotlib --extra lib-seaborn` without ADK and copies the harness to `/opt/anyplot/harness.py`. The `deploy-renderer` trigger (an owner task) fires on `agents/renderer/**`, `agents/anyplot/render/harness.py`, `pyproject.toml` and `uv.lock`. The `deploy-agents` trigger (an owner task) fires on `agents/**`, `core/**`, `prompts/library/**`, `prompts/default-style-guide.md`, `pyproject.toml` and `uv.lock`. Reused code: the `api/mcp/server.py` tool bodies move into `core/catalogue/queries.py`; `core/database/repositories.py` (`SpecRepository.get_by_id_with_code`, `ImplRepository.get_code`); `core/utils.strip_noqa_comments`; `core/constants.py` (`LIBRARIES_METADATA`, `LIBRARY_LANGUAGES`, `INTERACTIVE_LIBRARIES`); `core/palette.py`; the defect grammar through `core/defects.py` and the canvas gate and PNG auto-reject checks through `core/canvas.py`; the render invocation shape of `agentic/workflows/modules/regen/render.py` (copied, not imported); `automation/js-render/render.mjs` for the JS runtime later; the per-IP-hash rate-limit pattern of `api/routers/feedback.py` and `api/request_context.client_ip` for the public phase. @@ -346,7 +385,7 @@ The pipeline owns this workstream, not the agent network: it makes catalogue cod Libraries, each one `RuntimeAdapter` enabled through `AGENT_LIBRARIES` after its eval matrix reaches 80 %: first plotnine, altair, lets-plot and pygal (no browser); then plotly and bokeh (Chromium fixes); then JS through `render.mjs` with `window.ANYPLOT_DATA`, a `page.route` deny-all and a tree-sitter deny-list; then R (ragg and a readr loader); then Julia (a precompiled depot, a pinned `JULIA_CPU_TARGET`, and asynchronous submit-and-poll if renders exceed 60 s; Cloud Run jobs and worker pools also support sandboxes). Discovery follows the phase-2 flow above. -Hard gates before any non-owner use: the legal-page PR (`LegalPage.tsx` plus a pinned test per store), the `anyplot-renderer` split, anyplot-api on a dedicated service account, database sessions with a purge, GCS artifacts with a lifecycle, persisted budget counters, Model Armor and `ToolCallIntegrityPlugin`, Turnstile with server-side verification plus a Cloudflare rate rule plus per-IP-hash database quotas plus a signed anonymous identity, routes moved out of `/debug`, and owner decisions on the Vertex 24 h cache and the abuse-logging exception. +Hard gates before any non-owner use: the legal-page PR (`LegalPage.tsx` plus a pinned test per store), anyplot-api on a dedicated service account, database sessions with a purge, GCS artifacts with a lifecycle, persisted budget counters, Model Armor and `ToolCallIntegrityPlugin`, Turnstile with server-side verification plus a Cloudflare rate rule plus per-IP-hash database quotas plus a signed anonymous identity, routes moved out of `/debug`, and owner decisions on the Vertex 24 h cache and the abuse-logging exception. ## Verification @@ -388,6 +427,7 @@ These are manual steps only the owner can do; they are tracked in Todoist. 11. Confirm the GCP agreement type: the online terms mean abuse-monitoring prompt logging for up to 90 days; a Master Agreement is exempt. 12. Decide the fate of issue #5405, which proposed Claude with Find, Navigate and Create modes: supersede it with this document or update it. Register the Plausible properties and goals. 13. Before public use: move `anyplot-api` off the default compute service account, which holds `roles/editor`. +14. Before the first render through the deployed renderer: create the service account `anyplot-renderer@` with no role; grant `roles/iam.serviceAccountUser` and `roles/iam.serviceAccountTokenCreator` on it to the Cloud Build identity; run the first build of `agents/renderer/cloudbuild.yaml` with `_SMOKE_RENDER=false`; grant `roles/run.invoker` on `anyplot-renderer` to `anyplot-agents@`, `anyplot-renderer@` itself and the owner's account; rebuild with `_ALLOWED_CALLERS` (`anyplot-agents@`, `anyplot-renderer@`, the owner's account), `_AUDIENCES` (the OAuth client ids of the owner's user tokens; the deploy adds the service URL itself, and Cloud Run accepts no tag URL as an audience) and `_SMOKE_RENDER=true`; then create the `deploy-renderer` trigger with those substitutions. ## Open decisions @@ -405,13 +445,16 @@ Defaults apply until the owner decides otherwise. - One run or toggle in flight per user, across sessions (stricter than refusing a toggle only while that session runs): decided on 2026-10-10, kept. - Whether the "Create plot" action carries the site theme, so a dark-mode visitor's first render is already dark instead of a light render plus a toggle: not yet; the action renders light. - Feedback-case retention and data inclusion: 180 days in the private bucket; `include_data` on by default for admins and off by default with explicit consent for real users. +- The sandbox's private `/tmp` and memory: spike S2 counts each sandbox's `/tmp` tmpfs (about 1.9 GiB, no flag to shrink it) as instance memory, which on paper puts even one sandbox (1 GiB address space, 1.9 GiB `/tmp`, the 512 MiB volume) above the 3 GiB budget of a 4 GiB instance. Measured on 2026-10-10 through the deployed renderer (a throwaway service, 2 vCPU, 4 GiB, one sandbox at a time): code that writes 40 MiB files into `/tmp` until it is full ends after 4.6 s with `OSError: [Errno 28] No space left on device`, the instance stays up with 3,328 MiB available afterwards (3,451 before), and the memory watchdog never has to fire. Decided: a 4 GiB instance with one sandbox at a time, without `RENDERER_BIND_TMP` (which stays available, untested) and without an 8 GiB instance; the validator (no file writes in user code), the serial slot, the `MemAvailable` floor before a render (1,024 MiB) and the watchdog's floor during one (512 MiB, reason `memory`) stay as the controls. Re-measure before a second render slot. +- How a developer's `adk web` authenticates to the deployed renderer: `AGENT_RENDER_TOKEN` from `gcloud auth print-identity-token` is the path spike S proved against Cloud Run IAM; the token minted from a user's Application Default Credentials carries the ADC OAuth client id as `aud`, and whether Cloud Run IAM accepts it is not yet verified. Either way, the token's client id joins `RENDERER_AUDIENCES` (both client ids are in the bootstrap's `_AUDIENCES`) and the owner's email `RENDERER_ALLOWED_CALLERS`: open until the first deploy checks it. +- Cold starts of the renderer: with `--min-instances=0`, the first render after an idle period meets a cold instance, and spike S saw the front end answer 500, then 429, and a 200 only about 17 s after the first request. The `remote` backend retries such answers after 1, 3, 8 and 15 s within 30 s; whether that is enough, or the renderer runs with `_MIN_INSTANCES=1` (the billing commitments probably absorb its cost), is open until the first deploy measures it. - Premium positioning: [Vision](vision.md) lists "Try with your data" as premium; the licensing row in the decisions table settles that the code stays MIT regardless. ## Risks and mitigations | Risk | Mitigation | |---|---| -| Cloud Run sandboxes are in Preview with undocumented regions, isolation technology and cross-run visibility | Spikes S and G plus the probe suite gate phase 1; the `RenderBackend` protocol; the fallback is a gen2 service with Direct VPC egress deny or `GkeCodeExecutor` with a custom image; the renderer split before public use | +| Cloud Run sandboxes are in Preview with undocumented regions, isolation technology and cross-run visibility | Spikes S and G plus the probe suite gate phase 1; the `RenderBackend` protocol; the fallback is a gen2 service with Direct VPC egress deny or `GkeCodeExecutor` with a custom image; the sandboxes run in `anyplot-renderer`, a service of its own with a zero-role account, from phase 1 | | The model adapts poorly (only proxy evidence so far) | Spike X is the go or no-go with owner blind acceptance; an eligibility filter (statically clean data-generation pairs: matplotlib 65 of 251, seaborn 84 of 267; the dpi normaliser removes the canvas condition; plotly's 102 count only in phase 2); hints from the readiness scan; the normalisation workstream; an honest `needs_attention` status | | Gemini 3.8 Flash, the second arm, is a short-term tier not on the tier list, and `eu` has its own capacity pool | One `AGENT_MODEL` value, `retry_options` with backoff from day one, `error{code:"capacity"}` on persistent 429s, no fallback model unless the owner names one, the lifecycle probe, an eval gate on every bump | | The reviewer is a weak judge (expert correlation about 0.43) | Deterministic gates first, a short checklist, at most one repair, residual defects shown as notes, owner acceptance as the real metric | diff --git a/docs/index.md b/docs/index.md index bb1bef3f68..f091d1550a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -63,7 +63,7 @@ High-level understanding of why things work the way they do. - **[Vision](concepts/vision.md)** - Product mission, the problem we solve, and how we're different - **[Library Expansion Roadmap](concepts/library-expansion.md)** - Multi-language gallery expansion plan and licensing policy -- **[Agent Network](concepts/agent-network.md)** - Design of the "Use with my data" agent network on Google ADK and Vertex AI (Claude Haiku 5.5 by default, Gemini 3.8 Flash as the second arm): agents, pipeline, guardrails, serving, roadmap (as of 2026-10-09, the runtime core runs locally; the sandbox renderer, image, deploy and evals are not built) +- **[Agent Network](concepts/agent-network.md)** - Design of the "Use with my data" agent network on Google ADK and Vertex AI (Claude Haiku 5.5 by default, Gemini 3.8 Flash as the second arm): agents, pipeline, guardrails, serving, roadmap (as of 2026-10-10, the runtime core runs locally and the `anyplot-renderer` sandbox service is built but not deployed; the agents image, the deploy and the evals are not built) - **[Adaptable Code](concepts/adaptable-code.md)** - Design for catalogue implementations that adapt to other data: one `# Data` block ending in a tidy `df`, column constants with role comments, a static checker and a perturbation smoke run, the regen gate's adapt route, and a rollout through the daily regeneration (as of 2026-10-09, design only) --- diff --git a/docs/reference/repository.md b/docs/reference/repository.md index a90e40347e..b0565d231e 100644 --- a/docs/reference/repository.md +++ b/docs/reference/repository.md @@ -101,6 +101,14 @@ anyplot/ │ ├── main.py # anyplot-agents service: the private /v1 API around the ADK runner │ ├── stream.py # anyplot/1 SSE translator and output sanitiser │ ├── evals/fixtures/cases/ # Eval fixture cases (case.json + data.csv), also the adk web seed +│ ├── renderer/ # anyplot-renderer service: runs the code in Cloud Run sandboxes, no ADK import +│ │ ├── main.py # POST /render and GET /status behind the IAM caller check, one render slot +│ │ ├── executor.py # sandbox do per theme: kill path, byte watchdog, launcher retry, bounded output +│ │ ├── auth.py # Caller check on the IAM-forwarded ID-token claims +│ │ ├── settings.py # RendererSettings (RENDERER_* variables) +│ │ ├── wire.py # JSON contract shared with the agents' remote render backend +│ │ ├── Dockerfile # Renderer image: plotting venv, baked font cache, harness +│ │ └── cloudbuild.yaml # Build, candidate deploy, IAM and render smoke, promote │ └── anyplot/ │ ├── agent.py # Root agent "anyplot", ALL_AGENTS registry, App with plugins │ ├── models.py # The only model and client factory (Claude on Vertex AI, Gemini) @@ -110,7 +118,7 @@ anyplot/ │ ├── sub_agents/ # Per-library adapters and the tool-less reviewer │ ├── tools/session.py # Root's session tools and the plot_pipeline NodeTool │ ├── plugins/ # ScopeGuard, Budget, ToolSafety and the request ledger -│ ├── render/ # Render contract, harness, host gates, PNG hardening, backends +│ ├── render/ # Render contract, harness, host gates, PNG hardening, backends (remote, local, fake) │ ├── schemas.py # Pydantic contracts of the plot pipeline │ ├── settings.py # AgentSettings (AGENT_* variables) │ ├── data/ # Deterministic data handling, no ADK import diff --git a/docs/workflows/overview.md b/docs/workflows/overview.md index e2bab133b1..95404ffbb8 100644 --- a/docs/workflows/overview.md +++ b/docs/workflows/overview.md @@ -330,7 +330,7 @@ Located in `.github/workflows/`: | `codeql.yml` | CodeQL scanning (actions, JavaScript/TypeScript, Python) on pushes to main, PRs and a weekly cron; `plots/**` is excluded from triggers and analysis, so pipeline PRs never start a scan | | `ci-lint.yml` | Ruff lint check on PRs | | `ci-tests.yml` | Unit + integration tests on PRs. Its "Run Tests" job also lints the "What a good version looks like" section on every run, even when no Python changed: the contract over every spec (blocking) and the style of the changed specs (warnings) | -| `ci-image.yml` | Builds `api/Dockerfile` and smoke-tests the container before merge — `/health`, the reported version against `pyproject.toml`, the runtime stage's COPY payload, non-root uid — plus hadolint on both Dockerfiles. Skips when only `plots/**` or frontend sources changed | +| `ci-image.yml` | Builds `api/Dockerfile` and smoke-tests the container before merge — `/health`, the reported version against `pyproject.toml`, the runtime stage's COPY payload, non-root uid — plus hadolint on every Dockerfile it builds. Skips when only `plots/**` or frontend sources changed. A third job builds `agents/renderer/Dockerfile` when the renderer, the probe harness or the dependency lock changed: it renders a seaborn plot through the harness under the sandbox's rlimits with no network, checks that the image carries no ADK and reports the `pyproject.toml` version, and runs hadolint | | `notify-deployment.yml` | Records GitHub deployment events for `app` / `api` | | `bot-serving-check.yml` | Daily synthetic monitor: curls the Cloud Run origin with crawler UAs and fails on non-200 or a page that is not the prerendered one. Routes are derived from the `@router.get("/seo-proxy/…")` decorators in `api/routers/seo.py` and the expected spec title from `plots//specification.yaml`, so no literal here can go stale. A failure opens (or comments on) the fixed-title issue **Bot serving check is red**; the next green run closes it — the bot→seo-proxy path is invisible to human traffic and needs its own alarm | | `util-claude.yml` | On-demand `@claude` utility (issue/PR comments) | diff --git a/tests/unit/agents/renderer/__init__.py b/tests/unit/agents/renderer/__init__.py new file mode 100644 index 0000000000..8297199275 --- /dev/null +++ b/tests/unit/agents/renderer/__init__.py @@ -0,0 +1 @@ +"""Tests of the renderer service (agents/renderer/) and the remote render backend that calls it.""" diff --git a/tests/unit/agents/renderer/conftest.py b/tests/unit/agents/renderer/conftest.py new file mode 100644 index 0000000000..4ad81326d5 --- /dev/null +++ b/tests/unit/agents/renderer/conftest.py @@ -0,0 +1,98 @@ +"""Fixtures: a hermetic environment, a fake `sandbox` launcher, and an executor that runs the real harness here.""" + +import json +import os +import sys +from collections.abc import Iterator +from dataclasses import dataclass +from pathlib import Path + +import pytest + +from agents.anyplot.render.runtimes.python import HARNESS_SOURCE +from agents.renderer.executor import ExecutorConfig, SandboxExecutor +from agents.renderer.main import get_renderer +from agents.renderer.settings import get_settings + +from .helpers import FAKE_LAUNCHER + + +@pytest.fixture(autouse=True) +def clean_env(monkeypatch: pytest.MonkeyPatch) -> Iterator[None]: + """No RENDERER_* values, no Cloud Run variables and no ENVIRONMENT from the shell or CI.""" + for name in list(os.environ): + upper = name.upper() + if upper.startswith(("RENDERER_", "FAKE_SANDBOX_")) or upper in ("ENVIRONMENT", "K_SERVICE", "K_REVISION"): + monkeypatch.delenv(name) + get_settings.cache_clear() + get_renderer.cache_clear() + yield + get_settings.cache_clear() + get_renderer.cache_clear() + + +@dataclass +class FakeLauncher: + """The fake `sandbox` binary and its call log.""" + + binary: Path + log: Path + failures: Path + + def calls(self) -> list[list[str]]: + if not self.log.exists(): + return [] + return [json.loads(line) for line in self.log.read_text(encoding="utf-8").splitlines()] + + def launches(self) -> list[str]: + """The sandbox names of every `do` call, in order.""" + return [call[2] for call in self.calls() if call[:1] == ["do"]] + + def deletes(self) -> list[str]: + return [call[2] for call in self.calls() if call[:2] == ["delete", "--force"]] + + def fail_next(self, count: int) -> None: + self.failures.write_text(str(count), encoding="utf-8") + + +@pytest.fixture +def launcher(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> FakeLauncher: + binary = tmp_path / "bin" / "sandbox" + binary.parent.mkdir() + binary.write_text(f"#!{sys.executable}\n{FAKE_LAUNCHER}", encoding="utf-8") + binary.chmod(0o755) + log = tmp_path / "sandbox.log" + failures = tmp_path / "failures" + monkeypatch.setenv("FAKE_SANDBOX_LOG", str(log)) + monkeypatch.setenv("FAKE_SANDBOX_FAIL", str(failures)) + return FakeLauncher(binary, log, failures) + + +@pytest.fixture +def config(tmp_path: Path, launcher: FakeLauncher) -> ExecutorConfig: + """The deployed configuration, except that RLIMIT_AS and RLIMIT_NPROC stay off. + + Outside a sandbox `RLIMIT_NPROC` counts every process of this user, so setting it + here would break the machine running the tests, not the code under test. + """ + return ExecutorConfig( + sandbox_binary=str(launcher.binary), + python=sys.executable, + harness=str(HARNESS_SOURCE), + runs_dir=tmp_path / "runs", + mpl_seed=str(tmp_path / "no-seed"), + mplconfig=str(tmp_path / "mpl"), + run_budget_bytes=8 * 1024 * 1024, + watch_interval_s=0.02, + rlimit_cpu_s=60, + rlimit_fsize_mb=50, + rlimit_as_mb=0, + rlimit_nproc=0, + kill_grace_s=5.0, + bind_tmp=False, + ) + + +@pytest.fixture +def executor(config: ExecutorConfig) -> SandboxExecutor: + return SandboxExecutor(config) diff --git a/tests/unit/agents/renderer/helpers.py b/tests/unit/agents/renderer/helpers.py new file mode 100644 index 0000000000..c1b13853ea --- /dev/null +++ b/tests/unit/agents/renderer/helpers.py @@ -0,0 +1,148 @@ +"""Plain helpers for the renderer tests: jobs, unsigned ID tokens, a fake launcher script and a fake executor.""" + +import asyncio +import base64 +import json +import time +from typing import Any + +from agents.anyplot.render.backends.fake import fixture_png +from agents.renderer.wire import RenderRequest, Theme, ThemeRun + + +URL = "https://anyplot-renderer-239660669828.europe-west4.run.app" +AGENTS_SA = "anyplot-agents@anyplot.iam.gserviceaccount.com" + +PLOT = """\ +import os + +import matplotlib.pyplot as plt + +THEME = os.getenv("ANYPLOT_THEME", "light") +fig, ax = plt.subplots(figsize=(4, 2), dpi=50) +ax.plot([1, 2, 3], [1, 3, 2]) +fig.savefig(f"plot-{THEME}.png") +""" +"""A tiny matplotlib plot: 200x100 pixels, a probe, a few tenths of a second.""" + +CANVAS_PLOT = """\ +import os + +import matplotlib.pyplot as plt + +THEME = os.getenv("ANYPLOT_THEME", "light") +fig, ax = plt.subplots(figsize=(16, 9), dpi=200) +fig.patch.set_facecolor("#FAF8F1" if THEME == "light" else "#1A1A17") +ax.bar(["a", "b", "c", "d"], [3, 1, 4, 2], color="#009E73") +fig.savefig(f"plot-{THEME}.png") +""" +"""A plot on the 3200x1800 catalogue canvas, which the host gates accept.""" + +SLEEPER = "import time\ntime.sleep(60)\n" +"""Code that runs until the renderer stops it.""" + +FAKE_LAUNCHER = """\ +import json +import os +import sys + +args = sys.argv[1:] +log = os.environ.get("FAKE_SANDBOX_LOG") +if log: + with open(log, "a", encoding="utf-8") as handle: + handle.write(json.dumps(args) + "\\n") +if args[:1] == ["delete"]: + sys.exit(0) +if args[:1] != ["do"]: + print("unknown subcommand", file=sys.stderr) + sys.exit(125) +mounts, env, workdir, index = {}, {}, None, 1 +while args[index] != "--": + flag, value = args[index], args[index + 1] + if flag == "--mount": + spec = dict(part.split("=", 1) for part in value.split(",")) + mounts[spec["destination"]] = spec["source"] + elif flag == "-w": + workdir = value + elif flag == "--env": + key, _, item = value.partition("=") + env[key] = item + elif flag != "--sandbox-name": + print(f"unknown flag {flag}", file=sys.stderr) + sys.exit(125) + index += 2 +command = args[index + 1 :] +failures = os.environ.get("FAKE_SANDBOX_FAIL") +if failures and os.path.exists(failures): + with open(failures, encoding="utf-8") as handle: + left = int(handle.read() or 0) + if left > 0: + with open(failures, "w", encoding="utf-8") as handle: + handle.write(str(left - 1)) + print("Error: failed to exec in container: cmd.Wait(exec) failed: exit status 137", file=sys.stderr) + sys.exit(1) +os.chdir(mounts[workdir]) +os.execve(command[0], command, env) +""" +"""A stand-in for `/usr/local/gcp/bin/sandbox`: logs every call, emulates `do` by +changing into the `/work` mount and exec-ing the command with exactly the `--env` +values (so the launcher process is the harness, like a real sandbox's `do`), answers +`delete` with 0 like the real one, and fails `do` without starting anything while +the counter file in `FAKE_SANDBOX_FAIL` is positive (the S2 launcher failure).""" + + +def job(source: str = PLOT, *, themes: tuple[Theme, ...] = ("light",), timeout_s: float = 30.0) -> RenderRequest: + return RenderRequest( + job_id="job123", + language="python", + library="matplotlib", + source=source, + data_csv="x,y\n1,2\n", + themes=list(themes), + timeout_s=timeout_s, + ) + + +def id_token(**claims: Any) -> str: + """An ID token as Cloud Run forwards it: readable claims, the signature replaced.""" + + def part(value: dict[str, Any]) -> str: + return base64.urlsafe_b64encode(json.dumps(value).encode()).decode().rstrip("=") + + payload = {"iss": "https://accounts.google.com", "exp": int(time.time()) + 3600, **claims} + return f"{part({'alg': 'RS256', 'typ': 'JWT'})}.{part(payload)}.SIGNATURE_REMOVED_BY_GOOGLE" + + +class FakeExecutor: + """Fixture runs without a sandbox; records calls and the most runs at once.""" + + def __init__(self) -> None: + self.calls: list[tuple[str, Theme]] = [] + self.active = 0 + self.max_active = 0 + self.gate: asyncio.Event | None = None + self.error: Exception | None = None + self.stuck = 0 + self.available = True + + async def run_theme(self, job: RenderRequest, theme: Theme) -> ThemeRun: + self.active += 1 + self.max_active = max(self.max_active, self.active) + try: + if self.error is not None: + raise self.error + if self.gate is not None: + await self.gate.wait() + await asyncio.sleep(0.01) + self.calls.append((job.job_id, theme)) + return ThemeRun( + theme=theme, + exit_code=0, + png_base64=base64.b64encode(fixture_png(theme)).decode(), + probe={"texts": [], "tick_overlaps": 0, "points": 0}, + stderr_tail="", + wall_s=0.5, + max_rss_mb=150.0, + ) + finally: + self.active -= 1 diff --git a/tests/unit/agents/renderer/test_executor.py b/tests/unit/agents/renderer/test_executor.py new file mode 100644 index 0000000000..6216997c37 --- /dev/null +++ b/tests/unit/agents/renderer/test_executor.py @@ -0,0 +1,467 @@ +"""Tests for agents/renderer/executor.py: the launcher command, the kill path, the retry, the watchdog and the bounds. + +The launcher is the fake in `helpers.FAKE_LAUNCHER`; the code runs under the real +probe harness with this interpreter, so every run here is a real subprocess. +""" + +import asyncio +import base64 +import os +import signal +import stat +import time +from dataclasses import replace +from pathlib import Path +from types import SimpleNamespace + +import pytest + +from agents.renderer import executor as executor_module +from agents.renderer.executor import ( + HARNESS_START, + SANDBOX_PATH, + Attempt, + ExecutorConfig, + OutputRejected, + SandboxExecutor, + Unavailable, + build_argv, + mem_available_mb, + read_head_tail, + read_regular, + read_tail, + runs_volume_ok, + sandbox_env, + tree_size, +) +from agents.renderer.settings import RendererSettings +from agents.renderer.wire import MAX_STDERR_CHARS, ThemeRun + +from .conftest import FakeLauncher +from .helpers import PLOT, SLEEPER, job + + +def envs(argv: list[str]) -> dict[str, str]: + return dict(argv[index + 1].split("=", 1) for index, part in enumerate(argv) if part == "--env") + + +class TestCommand: + def test_argv_runs_the_harness_in_the_work_mount_with_a_complete_environment( + self, config: ExecutorConfig, tmp_path: Path + ) -> None: + run_dir = tmp_path / "r-abc-dark" + argv = build_argv(config, "r-abc-dark", run_dir, "dark", 30) + + assert argv[:4] == [config.sandbox_binary, "do", "--sandbox-name", "r-abc-dark"] + assert f"type=bind,source={run_dir / 'work'},destination=/work" in argv + assert argv[argv.index("-w") + 1] == "/work" + assert argv[argv.index("--") :] == ["--", config.python, "-I", config.harness, "plot.py"] + assert "--write" not in argv and not any(part.startswith("--allow-egress") for part in argv) + values = envs(argv) + assert values["PATH"] == SANDBOX_PATH # a sandbox has no PATH of its own (spike S2) + assert values["HOME"] == "/tmp" and values["ANYPLOT_THEME"] == "dark" and values["MPLBACKEND"] == "Agg" + assert values["MPLCONFIGDIR"] == config.mplconfig and values["ANYPLOT_MPL_SEED"] == config.mpl_seed + assert values["ANYPLOT_RLIMIT_CPU_S"] == "30" and values["ANYPLOT_RLIMIT_FSIZE_MB"] == "50" + assert "ANYPLOT_RLIMIT_AS_MB" not in values and "ANYPLOT_RLIMIT_NPROC" not in values + + def test_deployed_defaults_set_every_rlimit_from_spike_s2(self) -> None: + config = ExecutorConfig.from_settings(RendererSettings()) + values = sandbox_env(config, "light", 60) + + assert config.sandbox_binary == "/usr/local/gcp/bin/sandbox" + assert config.runs_dir == Path("/tmp/runs") and config.run_budget_bytes == 64 * 1024 * 1024 + assert values["ANYPLOT_RLIMIT_CPU_S"] == "60" and values["ANYPLOT_RLIMIT_FSIZE_MB"] == "50" + assert values["ANYPLOT_RLIMIT_AS_MB"] == "1024" and values["ANYPLOT_RLIMIT_NPROC"] == "64" + assert values["MPLCONFIGDIR"] == "/tmp/mpl" and values["ANYPLOT_MPL_SEED"] == "/opt/mplconfig" + + def test_bind_tmp_mounts_a_run_directory_over_tmp(self, config: ExecutorConfig, tmp_path: Path) -> None: + argv = build_argv(replace(config, bind_tmp=True), "r-abc-light", tmp_path / "r", "light", 60) + + assert f"type=bind,source={tmp_path / 'r' / 'tmp'},destination=/tmp" in argv + + def test_forbidden_flags_never_reach_the_launcher( + self, config: ExecutorConfig, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(executor_module, "sandbox_env", lambda *args: {"--write": ""}) + + with pytest.raises(AssertionError, match="--write"): + build_argv(config, "r-abc-light", tmp_path, "light", 60) + + +class TestRender: + async def test_a_theme_renders_with_png_probe_and_usage( + self, executor: SandboxExecutor, launcher: FakeLauncher, config: ExecutorConfig + ) -> None: + run = await executor.run_theme(job(PLOT), "dark") + + assert run.exit_code == 0, run.stderr_tail + assert run.reason is None and run.attempts == 1 and not run.timed_out + assert run.png_base64 is not None and base64.b64decode(run.png_base64).startswith(b"\x89PNG") + assert run.probe is not None and run.probe["canvas"] == [200, 100] + assert run.max_rss_mb is not None and run.max_rss_mb > 0 + assert len(launcher.launches()) == 1 and launcher.launches()[0].endswith("-dark") + assert launcher.deletes() == [] # a clean exit needs no delete + assert envs(launcher.calls()[0])["ANYPLOT_THEME"] == "dark" + assert list(config.runs_dir.iterdir()) == [] # every run directory is removed + + async def test_timeout_kills_the_launcher_and_deletes_the_sandbox( + self, executor: SandboxExecutor, launcher: FakeLauncher, config: ExecutorConfig + ) -> None: + started = time.monotonic() + run = await executor.run_theme(job(SLEEPER, timeout_s=1.5), "light") + + assert run.timed_out and run.exit_code is None and run.reason == "timeout" + assert time.monotonic() - started < 15 + assert launcher.deletes() == launcher.launches() # `delete --force` for the killed sandbox + assert executor.stuck == 0 # the kill was confirmed by the launcher's exit + assert list(config.runs_dir.iterdir()) == [] + + async def test_cancellation_kills_the_sandbox_before_its_directory_goes( + self, executor: SandboxExecutor, launcher: FakeLauncher, config: ExecutorConfig + ) -> None: + task = asyncio.create_task(executor.run_theme(job(SLEEPER), "light")) + for _ in range(200): + if launcher.launches(): + break + await asyncio.sleep(0.05) + await asyncio.sleep(0.2) + task.cancel() + + with pytest.raises(asyncio.CancelledError): + await task + assert launcher.deletes() == launcher.launches() + assert executor.stuck == 0 + assert list(config.runs_dir.iterdir()) == [] + + async def test_a_launcher_that_outlives_its_kill_blocks_renders_until_it_exits( + self, config: ExecutorConfig, launcher: FakeLauncher, monkeypatch: pytest.MonkeyPatch + ) -> None: + executor = SandboxExecutor(replace(config, kill_grace_s=0.2)) + monkeypatch.setattr(executor_module, "READER_GRACE_S", 0.2) + monkeypatch.setattr(executor_module.os, "killpg", lambda pid, sig: None) # the kill has no effect + + run = await executor.run_theme(job(SLEEPER, timeout_s=1.0), "light") + + assert run.timed_out and executor.stuck == 1 + with pytest.raises(Unavailable, match="stuck"): + await executor.run_theme(job(PLOT), "light") + survivor = executor._stuck[0] + os.kill(survivor.pid, signal.SIGKILL) + await survivor.wait() + assert executor.stuck == 0 + + async def test_a_missing_launcher_is_unavailable(self, config: ExecutorConfig, tmp_path: Path) -> None: + executor = SandboxExecutor(replace(config, sandbox_binary=str(tmp_path / "missing"))) + + assert executor.available is False + with pytest.raises(Unavailable, match="sandbox_unavailable"): + await executor.run_theme(job(PLOT), "light") + + +class TestRetry: + async def test_a_launcher_failure_before_the_harness_is_retried_once_with_a_new_name( + self, executor: SandboxExecutor, launcher: FakeLauncher + ) -> None: + launcher.fail_next(1) + + run = await executor.run_theme(job(PLOT), "light") + + assert run.exit_code == 0 and run.attempts == 2 and run.reason is None and run.png_base64 + names = launcher.launches() + assert len(names) == 2 and names[0] != names[1] + + async def test_a_second_launcher_failure_is_reported( + self, executor: SandboxExecutor, launcher: FakeLauncher + ) -> None: + launcher.fail_next(2) + + run = await executor.run_theme(job(PLOT), "light") + + assert run.exit_code == 1 and run.attempts == 2 and run.reason == "launcher" and run.png_base64 is None + assert "failed to exec in container" in run.stderr_tail + assert len(launcher.launches()) == 2 + assert launcher.deletes() == launcher.launches() # a failed launch may leave a half-created sandbox + + async def test_a_failure_in_the_plot_code_is_not_retried( + self, executor: SandboxExecutor, launcher: FakeLauncher + ) -> None: + run = await executor.run_theme(job("raise ValueError('a cell of the data')\n"), "light") + + assert run.exit_code == 1 and run.attempts == 1 and run.reason is None + assert "ValueError" in run.stderr_tail + assert len(launcher.launches()) == 1 and launcher.deletes() == launcher.launches() + + def test_only_a_run_without_the_harness_start_line_counts_as_a_launcher_failure(self) -> None: + assert Attempt("r", exit_code=1).launcher_failed + assert not Attempt("r", exit_code=1, stdout_head=f"{HARNESS_START}\n").launcher_failed + assert not Attempt("r", exit_code=0).launcher_failed + assert not Attempt("r", exit_code=None, timed_out=True, reason="timeout").launcher_failed + assert not Attempt("r", exit_code=None, reason="disk_budget").launcher_failed + + +class TestBounds: + async def test_the_byte_watchdog_kills_a_run_over_its_budget( + self, executor: SandboxExecutor, launcher: FakeLauncher, config: ExecutorConfig + ) -> None: + source = ( + "import time\n" + "with open('fill.bin', 'wb') as handle:\n" + " for _ in range(400):\n" + " handle.write(b'x' * 1024 * 1024)\n" + " handle.flush()\n" + " time.sleep(0.01)\n" + ) + run = await executor.run_theme(job(source), "light") + + assert run.reason == "disk_budget" and run.exit_code is None and not run.timed_out + assert run.png_base64 is None + assert launcher.deletes() == launcher.launches() + assert list(config.runs_dir.iterdir()) == [] + + async def test_a_fast_writer_that_exits_between_two_samples_is_still_refused( + self, config: ExecutorConfig, launcher: FakeLauncher + ) -> None: + """The watchdog sleeps between looks; a run that writes past the budget and exits before the next look must not pass.""" + executor = SandboxExecutor(replace(config, watch_interval_s=30.0)) # the watchdog never samples + source = "open('fill.bin', 'wb').write(b'x' * 12 * 1024 * 1024)\n" + + run = await executor.run_theme(job(source), "light") + + assert run.reason == "disk_budget" and run.limit == config.run_budget_bytes + assert run.measured is not None and run.measured > config.run_budget_bytes + assert run.exit_code is None and run.png_base64 is None and not run.timed_out + assert launcher.deletes() == launcher.launches() + assert list(config.runs_dir.iterdir()) == [] + + async def test_the_watchdog_counts_entries_too( + self, executor: SandboxExecutor, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(executor_module, "MAX_RUN_ENTRIES", 50) + source = "import time\nfor index in range(5000):\n open(f'f{index}', 'w').close()\n time.sleep(0.001)\n" + run = await executor.run_theme(job(source), "light") + + assert run.reason == "file_budget" and run.limit == 50 + assert run.measured is not None and run.measured > 50 + + async def test_the_byte_watchdog_reports_the_measured_size_and_the_budget( + self, executor: SandboxExecutor, config: ExecutorConfig + ) -> None: + source = "import time\nopen('fill.bin', 'wb').write(b'x' * 12 * 1024 * 1024)\ntime.sleep(30)\n" + + run = await executor.run_theme(job(source), "light") + + assert run.reason == "disk_budget" and run.limit == config.run_budget_bytes + assert run.measured is not None and run.measured > config.run_budget_bytes + + async def test_a_tree_past_path_max_counts_as_over_its_limit( + self, executor: SandboxExecutor, launcher: FakeLauncher + ) -> None: + """A walk by full paths cannot see past PATH_MAX; what it cannot measure must not escape the budget.""" + source = ( + "import os, time\nfor _ in range(40):\n os.mkdir('d' * 200)\n os.chdir('d' * 200)\ntime.sleep(30)\n" + ) + + run = await executor.run_theme(job(source), "light") + + assert run.reason == "file_budget" and run.measured is None and run.exit_code is None + assert launcher.deletes() == launcher.launches() + + async def test_the_memory_floor_kills_the_run( + self, config: ExecutorConfig, launcher: FakeLauncher, monkeypatch: pytest.MonkeyPatch + ) -> None: + """A sandbox's private /tmp counts as instance memory (S2), so the watchdog reads MemAvailable too.""" + readings = iter([3000, 3000, 400]) + executor = SandboxExecutor(replace(config, kill_mem_available_mb=512), memory=lambda: next(readings, 400)) + + run = await executor.run_theme(job(SLEEPER), "light") + + assert run.reason == "memory" and (run.measured, run.limit) == (400, 512) + assert run.exit_code is None and not run.timed_out and run.png_base64 is None + assert launcher.deletes() == launcher.launches() + + async def test_a_host_io_error_is_unavailable_and_leaves_nothing( + self, executor: SandboxExecutor, launcher: FakeLauncher, config: ExecutorConfig, monkeypatch: pytest.MonkeyPatch + ) -> None: + def full(self: Path, *args: object, **kwargs: object) -> None: + raise OSError(28, "No space left on device") + + monkeypatch.setattr(Path, "write_text", full) + + with pytest.raises(Unavailable, match="io"): + await executor.run_theme(job(PLOT), "light") + assert launcher.launches() == [] and list(config.runs_dir.iterdir()) == [] + + async def test_a_launcher_that_cannot_start_is_unavailable( + self, executor: SandboxExecutor, monkeypatch: pytest.MonkeyPatch + ) -> None: + async def no_fork(*args: object, **kwargs: object) -> None: + raise BlockingIOError(11, "Resource temporarily unavailable") + + monkeypatch.setattr(executor_module.asyncio, "create_subprocess_exec", no_fork) + + with pytest.raises(Unavailable, match="io"): + await executor.run_theme(job(PLOT), "light") + + async def test_stderr_keeps_only_its_tail(self, executor: SandboxExecutor) -> None: + source = "import sys\nsys.stderr.write('x' * 2_000_000)\nsys.stderr.write('THE-END')\nraise SystemExit(3)\n" + + run = await executor.run_theme(job(source), "light") + + assert run.exit_code == 3 and run.attempts == 1 + assert len(run.stderr_tail) <= MAX_STDERR_CHARS and run.stderr_tail.endswith("THE-END") + + async def test_a_flood_on_stdout_keeps_the_harness_lines(self, executor: SandboxExecutor) -> None: + run = await executor.run_theme(job("print('y' * 3_000_000)\n" + PLOT), "light") + + assert run.exit_code == 0 and run.attempts == 1 and run.png_base64 is not None + assert run.max_rss_mb is not None # the end line survives in the stdout tail + + async def test_a_symlinked_png_is_refused(self, executor: SandboxExecutor) -> None: + source = "import os\nos.symlink('/etc/hostname', 'plot-light.png')\n" + + run = await executor.run_theme(job(source), "light") + + assert run.exit_code == 0 and run.png_base64 is None and run.reason == "output_rejected" + + async def test_a_broken_probe_is_absent(self, executor: SandboxExecutor) -> None: + source = PLOT + "open('probe-light.json', 'w').write('[1, 2')\n" + + run = await executor.run_theme(job(source), "light") + + assert run.png_base64 is not None and run.probe is None + + async def test_an_oversized_png_reports_its_size_and_the_cap( + self, executor: SandboxExecutor, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(executor_module, "MAX_PNG_BYTES", 100) + + run = await executor.run_theme(job(PLOT), "light") + + assert run.reason == "output_rejected" and run.limit == 100 + assert run.measured is not None and run.measured > 100 and run.png_base64 is None + + @pytest.mark.parametrize("depth", [17, 210, 300]) + async def test_a_deeply_nested_probe_is_dropped_and_the_answer_still_serialises( + self, executor: SandboxExecutor, depth: int + ) -> None: + """256 levels broke `model_dump`, about 200 the agents' parse; the renderer drops the probe instead.""" + source = PLOT + f"open('probe-light.json', 'w').write('{{\"a\": ' + '[' * {depth} + ']' * {depth} + '}}')\n" + + run = await executor.run_theme(job(source), "light") + + assert run.png_base64 is not None and run.probe is None + ThemeRun.model_validate_json(run.model_dump_json()) + + @pytest.mark.parametrize( + "report", + [ + '{"event": "end", "max_rss_mb": 1' + "0" * 400 + ', "cpu_s": -5}', + '{"event": "end", "max_rss_mb": 1e999, "cpu_s": NaN}', + '{"event": "end", "max_rss_mb": true, "cpu_s": "12"}', + '{"event": "end", "max_rss_mb": ' + "[" * 5000 + "]" * 5000 + "}", + ], + ) + def test_a_forged_end_line_cannot_break_the_answer(self, report: str) -> None: + """The code shares stdout; a JSON int too large for a float made `to_wire` raise OverflowError (a 500).""" + attempt = Attempt("r", exit_code=0, stdout_tail=f"HARNESS {report}\n") + + run = attempt.to_wire("light", 1) + + assert run.max_rss_mb is None and run.cpu_s is None + assert Attempt("r", 0, stdout_tail='HARNESS {"event": "end", "max_rss_mb": 150.5, "cpu_s": 2}').usage() == { + "max_rss_mb": 150.5, + "cpu_s": 2.0, + } + + +class TestHelpers: + async def test_read_tail_and_head_tail_hold_bounded_bytes(self) -> None: + reader = asyncio.StreamReader() + reader.feed_data(b"a" * 100_000 + b"END") + reader.feed_eof() + assert await read_tail(reader, 10) == b"aaaaaaaEND" + + reader = asyncio.StreamReader() + reader.feed_data(b"HEAD" + b"m" * 100_000 + b"TAIL") + reader.feed_eof() + head, tail = await read_head_tail(reader, 4, 4) + assert (head, tail) == (b"HEAD", b"TAIL") + + def test_read_regular_refuses_links_fifos_directories_and_oversize(self, tmp_path: Path) -> None: + (tmp_path / "ok.png").write_bytes(b"12345") + os.symlink(tmp_path / "ok.png", tmp_path / "link.png") + os.mkfifo(tmp_path / "fifo.png") + (tmp_path / "dir.png").mkdir() + + assert read_regular(tmp_path / "ok.png", 5) == b"12345" + assert read_regular(tmp_path / "missing.png", 5) is None + for name in ("link.png", "fifo.png", "dir.png"): + with pytest.raises(OutputRejected): + read_regular(tmp_path / name, 5) + with pytest.raises(OutputRejected, match="limit"): + read_regular(tmp_path / "ok.png", 4) + + def test_tree_size_does_not_follow_links_and_stops_early(self, tmp_path: Path) -> None: + big = tmp_path / "outside.bin" + big.write_bytes(b"x" * 10_000) + root = tmp_path / "run" + (root / "work" / "nested").mkdir(parents=True) + (root / "work" / "nested" / "a.bin").write_bytes(b"y" * 100) + os.symlink(big, root / "work" / "link") + + usage = tree_size(root, 1_000_000, 1_000) + + listed = [root / "work", root / "work" / "nested", root / "work" / "nested" / "a.bin", root / "work" / "link"] + assert usage.measurable and usage.entries == 4 + assert usage.bytes == sum(os.lstat(path).st_size for path in listed) # the link's own size only + assert tree_size(root, 10, 1_000).bytes > 10 # stops once over budget + + def test_a_tree_too_deep_or_too_long_is_unmeasurable(self, tmp_path: Path) -> None: + root = tmp_path / "run" + deep = root.joinpath(*["d"] * 6) + deep.mkdir(parents=True) + + assert tree_size(root, 1_000_000, 1_000, max_depth=8).measurable + assert not tree_size(root, 1_000_000, 1_000, max_depth=5).measurable + + # Past PATH_MAX a full-path walk fails with ENAMETOOLONG; that must not read as "nothing here". + long_root = tmp_path / "long" + long_root.mkdir() + descriptor = os.open(long_root, os.O_RDONLY) + try: + for _ in range(25): + os.mkdir("n" * 200, dir_fd=descriptor) + child = os.open("n" * 200, os.O_RDONLY, dir_fd=descriptor) + os.close(descriptor) + descriptor = child + finally: + os.close(descriptor) + assert not tree_size(long_root, 1_000_000, 1_000).measurable + + def test_runs_volume_ok_needs_a_small_mount_of_its_own( + self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch + ) -> None: + assert runs_volume_ok(tmp_path, 1 << 40, root=tmp_path) is False # the same filesystem as `/` + assert runs_volume_ok(tmp_path / "missing", 1 << 40) is False + + real_stat = os.stat + + def other_device(path: str | os.PathLike[str], *args: object, **kwargs: object) -> os.stat_result: + result = real_stat(path) + if Path(path) == tmp_path: + values = list(result) + values[stat.ST_DEV] = result.st_dev + 1 + return os.stat_result(values) + return result + + monkeypatch.setattr(executor_module.os, "stat", other_device) + monkeypatch.setattr(executor_module.shutil, "disk_usage", lambda path: SimpleNamespace(total=512 << 20)) + assert runs_volume_ok(tmp_path, 1024 << 20) is True + assert runs_volume_ok(tmp_path, 256 << 20) is False # a mount larger than the volume should be + + def test_mem_available_reads_meminfo(self, tmp_path: Path) -> None: + meminfo = tmp_path / "meminfo" + meminfo.write_text("MemTotal: 4194304 kB\nMemFree: 1 kB\nMemAvailable: 3145728 kB\n") + + assert mem_available_mb(meminfo) == 3072 + assert mem_available_mb(tmp_path / "missing") is None diff --git a/tests/unit/agents/renderer/test_remote_backend.py b/tests/unit/agents/renderer/test_remote_backend.py new file mode 100644 index 0000000000..edf6d629f9 --- /dev/null +++ b/tests/unit/agents/renderer/test_remote_backend.py @@ -0,0 +1,576 @@ +"""Tests for agents/anyplot/render/backends/remote.py against the renderer app, in process. + +The backend talks to `agents.renderer.main.app` through httpx's ASGI transport, so +the caller check, the contract and the slot are the real ones; only the executor is +a fake, except in the end-to-end test, which runs the real executor with the fake +launcher and the real harness. +""" + +import asyncio +from collections.abc import AsyncIterator, Callable, Iterator +from typing import Any + +import google.auth.exceptions +import google.oauth2.credentials +import httpx +import pytest + +from agents.anyplot.render.backends import remote as remote_module +from agents.anyplot.render.backends.remote import IdTokenSource, RemoteBackend, token_expiry +from agents.anyplot.render.contract import RendererUnavailable, RenderJob +from agents.anyplot.render.gates import evaluate +from agents.anyplot.render.serial import SerialRenderer +from agents.renderer.executor import ExecutorConfig, SandboxExecutor +from agents.renderer.main import Renderer, app, get_renderer +from agents.renderer.settings import RendererSettings, get_settings + +from .conftest import FakeLauncher +from .helpers import AGENTS_SA, CANVAS_PLOT, SLEEPER, URL, FakeExecutor, id_token + + +GOOD = id_token(aud=URL, email=AGENTS_SA) +NO_WAIT = (0.0, 0.0, 0.0, 0.0) + + +class StaticTokens: + def __init__(self, token: str = GOOD) -> None: + self.value = token + self.calls = 0 + + async def token(self) -> str: + self.calls += 1 + return self.value + + +class Flaky(httpx.AsyncBaseTransport): + """Fails the first `failures` requests with `error`, then hands over to `inner`.""" + + def __init__(self, inner: httpx.AsyncBaseTransport, failures: int, error: type[httpx.TransportError]) -> None: + self.inner = inner + self.failures = failures + self.error = error + self.requests = 0 + + async def handle_async_request(self, request: httpx.Request) -> httpx.Response: + self.requests += 1 + if self.requests <= self.failures: + raise self.error("connection refused", request=request) + return await self.inner.handle_async_request(request) + + +def render_job(**overrides: Any) -> RenderJob: + values: dict[str, Any] = { + "job_id": "job123", + "language": "python", + "library": "matplotlib", + "source": CANVAS_PLOT, + "data_csv": "x,y\n1,2\n", + "themes": ("light",), + "timeout_s": 30.0, + } + return RenderJob(**{**values, **overrides}) + + +@pytest.fixture +def settings() -> RendererSettings: + return RendererSettings( + environment="production", audiences=[URL], allowed_callers=[AGENTS_SA], min_mem_available_mb=0 + ) + + +@pytest.fixture +def fake() -> FakeExecutor: + return FakeExecutor() + + +@pytest.fixture +def renderer(settings: RendererSettings, fake: FakeExecutor) -> Renderer: + return Renderer(settings, fake) + + +@pytest.fixture +def overrides(settings: RendererSettings, renderer: Renderer) -> Iterator[None]: + app.dependency_overrides[get_settings] = lambda: settings + app.dependency_overrides[get_renderer] = lambda: renderer + yield + app.dependency_overrides.clear() + + +@pytest.fixture +async def backend(overrides: None) -> AsyncIterator[RemoteBackend]: + backend = RemoteBackend( + url=URL, tokens=StaticTokens(), transport=httpx.ASGITransport(app=app), retry_delays_s=NO_WAIT + ) + yield backend + await backend.aclose() + + +def mocked(handler: Callable[[httpx.Request], Any], **kwargs: Any) -> RemoteBackend: + """A backend whose requests `handler` answers (sync or async), with no pause between retries.""" + return RemoteBackend( + url=URL, tokens=StaticTokens(), transport=httpx.MockTransport(handler), **{"retry_delays_s": NO_WAIT, **kwargs} + ) + + +def answer_json(**overrides: Any) -> dict[str, Any]: + """A renderer answer for job123 with one light theme, its fields overridable.""" + light = {"theme": "light", "exit_code": 0, "png_base64": None, **overrides} + return { + "job_id": "job123", + "outputs": {"light": light}, + "timings": {"slot_wait_s": 0, "total_s": 0}, + "version": "x", + } + + +class TestRoundTrip: + async def test_a_render_comes_back_as_a_render_result_the_gates_accept( + self, backend: RemoteBackend, fake: FakeExecutor + ) -> None: + result = await backend.render(render_job(themes=("light", "dark"))) + + assert result.job_id == "job123" and set(result.outputs) == {"light", "dark"} + light = result.outputs["light"] + assert light.exit_code == 0 and light.png is not None and light.png.startswith(b"\x89PNG") + assert light.probe == {"texts": [], "tick_overlaps": 0, "points": 0} and light.wall_s == 0.5 + report = evaluate(result, themes=("light", "dark"), library="matplotlib", rows=1) + assert report.passed_host_gates and report.canvas_ok, report.blocking + assert fake.calls == [("job123", "light"), ("job123", "dark")] + + async def test_behind_the_serial_renderer_each_theme_is_its_own_request( + self, backend: RemoteBackend, fake: FakeExecutor + ) -> None: + serial = SerialRenderer(backend, concurrency=1) + + result = await serial.render(render_job(themes=("light", "dark"))) + + assert set(result.outputs) == {"light", "dark"} + assert isinstance(backend.tokens, StaticTokens) and backend.tokens.calls == 2 + + async def test_end_to_end_with_the_real_executor_and_harness( + self, overrides: None, settings: RendererSettings, config: ExecutorConfig + ) -> None: + app.dependency_overrides[get_renderer] = lambda: Renderer(settings, SandboxExecutor(config)) + backend = RemoteBackend(url=URL, tokens=StaticTokens(), transport=httpx.ASGITransport(app=app)) + try: + result = await backend.render(render_job(themes=("dark",))) + finally: + await backend.aclose() + + report = evaluate(result, themes=("dark",), library="matplotlib", rows=1) + assert report.passed_host_gates and report.canvas_ok, (report.blocking, result.outputs["dark"].stderr_tail) + assert result.outputs["dark"].probe is not None and result.outputs["dark"].probe["canvas"] == [3200, 1800] + + +class TestFailures: + async def test_a_refused_caller_is_unavailable_and_logged( + self, overrides: None, caplog: pytest.LogCaptureFixture + ) -> None: + """The pipeline logs only the exception class, so a wrong audience at bootstrap must show here.""" + backend = RemoteBackend( + url=URL, + tokens=StaticTokens(id_token(aud=URL, email="someone@example.com")), + transport=httpx.ASGITransport(app=app), + ) + with pytest.raises(RendererUnavailable, match="refused this caller"): + await backend.render(render_job()) + assert "the renderer refused this caller (HTTP 403, forbidden)" in caplog.text + await backend.aclose() + + async def test_one_connection_error_is_retried(self, overrides: None) -> None: + transport = Flaky(httpx.ASGITransport(app=app), failures=1, error=httpx.ConnectError) + backend = RemoteBackend(url=URL, tokens=StaticTokens(), transport=transport, retry_delays_s=NO_WAIT) + + result = await backend.render(render_job()) + + assert result.outputs["light"].exit_code == 0 and transport.requests == 2 + await backend.aclose() + + async def test_connection_errors_past_the_retries_are_unavailable( + self, overrides: None, caplog: pytest.LogCaptureFixture + ) -> None: + transport = Flaky(httpx.ASGITransport(app=app), failures=5, error=httpx.ConnectError) + backend = RemoteBackend(url=URL, tokens=StaticTokens(), transport=transport, retry_delays_s=NO_WAIT) + + with pytest.raises(RendererUnavailable, match=r"could not be reached \(ConnectError\)"): + await backend.render(render_job()) + assert transport.requests == 1 + len(NO_WAIT) + assert "could not be reached (ConnectError)" in caplog.text # logged, not only raised + await backend.aclose() + + async def test_a_cold_start_is_ridden_out_with_backoff(self) -> None: + """Spike S after a scale from zero: a 500, then 429s, then the answer, about 17 s in all.""" + answers = [ + httpx.Response(500, text="Server Error", headers={"content-type": "text/html"}), + httpx.Response(429, text="Rate exceeded.", headers={"content-type": "text/html"}), + httpx.Response(429, text="Rate exceeded.", headers={"content-type": "text/html"}), + httpx.Response(200, json=answer_json()), + ] + sleeps: list[float] = [] + + async def no_sleep(delay: float) -> None: + sleeps.append(delay) + + backend = RemoteBackend( + url=URL, + tokens=StaticTokens(), + transport=httpx.MockTransport(lambda request: answers.pop(0)), + sleep=no_sleep, + ) + + result = await backend.render(render_job()) + + assert result.outputs["light"].exit_code == 0 + assert sleeps == [1.0, 3.0, 8.0] # the default schedule + await backend.aclose() + + async def test_retries_stop_at_the_budget(self) -> None: + clock = [0.0] + requests: list[float] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(clock[0]) + clock[0] += 12.0 # each answer arrives 12 s after its request + return httpx.Response(503, text="Service Unavailable", headers={"content-type": "text/html"}) + + async def advance(delay: float) -> None: + clock[0] += delay + + backend = mocked( + handler, retry_delays_s=(1.0, 3.0, 8.0, 15.0), retry_budget_s=30.0, clock=lambda: clock[0], sleep=advance + ) + + with pytest.raises(RendererUnavailable, match="HTTP 503"): + await backend.render(render_job()) + assert requests == [0.0, 13.0, 28.0] # a fourth would start 48 s after the first + await backend.aclose() + + async def test_a_front_end_error_is_retried_but_the_renderers_own_is_not(self) -> None: + answers = [ + httpx.Response(429, text="no available instance", headers={"content-type": "text/html"}), + httpx.Response(503, json={"detail": "busy"}), + ] + seen: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + seen.append(request.headers["authorization"]) + return answers[len(seen) - 1] + + backend = mocked(handler) + + with pytest.raises(RendererUnavailable, match="HTTP 503, busy"): + await backend.render(render_job()) + assert seen == [f"Bearer {GOOD}"] * 2 + await backend.aclose() + + async def test_the_renderers_own_500_is_not_retried(self) -> None: + """A JSON 500 is the renderer's own; retrying it would run a job that broke the renderer twice.""" + calls: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + calls.append(request.url.path) + return httpx.Response(500, json={"detail": "internal"}) + + backend = mocked(handler) + + with pytest.raises(RendererUnavailable, match="HTTP 500, internal"): + await backend.render(render_job()) + assert calls == ["/render"] + await backend.aclose() + + @pytest.mark.parametrize( + ("status", "text", "message", "requests"), + [ + (200, "not json", "does not match the contract", 1), + (500, "Server Error", "HTTP 500", 1 + len(NO_WAIT)), + ], + ) + async def test_malformed_or_failed_answers_are_unavailable( + self, status: int, text: str, message: str, requests: int + ) -> None: + calls: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + calls.append(request.url.path) + return httpx.Response(status, text=text) + + backend = mocked(handler) + + with pytest.raises(RendererUnavailable, match=message): + await backend.render(render_job()) + assert len(calls) == requests + await backend.aclose() + + async def test_a_launcher_failure_is_the_renderers_not_the_codes(self) -> None: + backend = mocked( + lambda request: httpx.Response(200, json=answer_json(exit_code=1, attempts=2, reason="launcher")) + ) + + with pytest.raises(RendererUnavailable, match="launcher failed twice"): + await backend.render(render_job()) + await backend.aclose() + + @pytest.mark.parametrize( + ("reason", "measured", "limit", "expected"), + [ + ( + "disk_budget", + 70 * 1024 * 1024, + 64 * 1024 * 1024, + "wrote 70.0 MiB to its working directory, over the 64.0 MiB limit", + ), + ("file_budget", 10_001, 10_000, "created 10001 files and directories, over the limit of 10000"), + ("file_budget", None, 10_000, "a directory tree too deep to measure"), + ("memory", 400, 512, "free memory fell to 400 MiB, below its 512 MiB floor"), + ( + "output_rejected", + 11 * 1024 * 1024, + 10 * 1024 * 1024, + "plot-light.png is 11.0 MiB, over the 10.0 MiB limit", + ), + ("output_rejected", None, None, "plot-light.png is not a regular file"), + ], + ) + async def test_a_limit_the_renderer_hit_reaches_the_repair_with_its_numbers( + self, reason: str, measured: int | None, limit: int | None, expected: str + ) -> None: + """Without the reason R1 read every stop as "failed with no error output" and spent the repair on it.""" + exit_code = 0 if reason == "output_rejected" else None + answer = answer_json(exit_code=exit_code, reason=reason, measured=measured, limit=limit) + backend = mocked(lambda request: httpx.Response(200, json=answer)) + + result = await backend.render(render_job()) + await backend.aclose() + + output = result.outputs["light"] + assert (output.reason, output.measured, output.limit) == (reason, measured, limit) + report = evaluate(result, themes=("light",), library="matplotlib", rows=1) + assert not report.passed_host_gates and len(report.blocking) == 1 + assert expected in report.blocking[0] and "no error output" not in report.blocking[0] + + async def test_an_answer_for_another_job_is_refused(self) -> None: + def other(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={"job_id": "other", "outputs": {}, "timings": {"slot_wait_s": 0, "total_s": 0}, "version": "x"}, + ) + + stray = RemoteBackend(url=URL, tokens=StaticTokens(), transport=httpx.MockTransport(other)) + with pytest.raises(RendererUnavailable, match="another job"): + await stray.render(render_job()) + await stray.aclose() + + async def test_a_theme_without_an_answer_is_left_for_r1(self) -> None: + def light_only(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={ + "job_id": "job123", + "outputs": {"light": {"theme": "light", "exit_code": 1, "png_base64": "%%%"}}, + "timings": {"slot_wait_s": 0, "total_s": 0}, + "version": "x", + }, + ) + + backend = RemoteBackend(url=URL, tokens=StaticTokens(), transport=httpx.MockTransport(light_only)) + result = await backend.render(render_job(themes=("light", "dark"))) + await backend.aclose() + + assert set(result.outputs) == {"light"} and result.outputs["light"].png is None # undecodable base64 + report = evaluate(result, themes=("light", "dark"), library="matplotlib", rows=1) + assert not report.passed_host_gates + assert any("dark" in line and "no output" in line for line in report.blocking) + + async def test_a_job_outside_the_contract_never_leaves(self) -> None: + calls: list[httpx.Request] = [] + backend = RemoteBackend( + url=URL, + tokens=StaticTokens(), + transport=httpx.MockTransport(lambda r: calls.append(r) or httpx.Response(500)), + ) + + with pytest.raises(RendererUnavailable, match="language"): + await backend.render(render_job(language="r")) + assert calls == [] + await backend.aclose() + + +class TestCancel: + """The front end may not pass a closed connection on, so the backend also asks the renderer to stop.""" + + async def test_a_cancelled_render_asks_the_renderer_to_stop(self) -> None: + started = asyncio.Event() + cancels: list[str] = [] + + async def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/cancel"): + cancels.append(request.url.path) + return httpx.Response(200, json={"cancelled": 1}) + started.set() + await asyncio.Event().wait() + raise AssertionError("the render request never gets an answer") + + backend = mocked(handler) + task = asyncio.create_task(backend.render(render_job())) + await asyncio.wait_for(started.wait(), 5) + task.cancel() + + with pytest.raises(asyncio.CancelledError): + await task + assert cancels == ["/render/job123/cancel"] + await backend.aclose() + + async def test_an_answer_that_does_not_come_in_time_is_cancelled_too(self) -> None: + cancels: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/cancel"): + cancels.append(request.url.path) + return httpx.Response(200, json={"cancelled": 1}) + raise httpx.ReadTimeout("timed out", request=request) + + backend = mocked(handler) + + with pytest.raises(RendererUnavailable, match=r"did not answer in time \(ReadTimeout\)"): + await backend.render(render_job()) + assert cancels == ["/render/job123/cancel"] + await backend.aclose() + + async def test_a_cancel_that_fails_is_only_logged(self, caplog: pytest.LogCaptureFixture) -> None: + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("/cancel"): + raise httpx.ConnectError("refused", request=request) + raise httpx.ReadTimeout("timed out", request=request) + + backend = mocked(handler) + + with pytest.raises(RendererUnavailable, match="did not answer in time"): + await backend.render(render_job()) + assert "the cancel did not reach the renderer (ConnectError)" in caplog.text + await backend.aclose() + + async def test_end_to_end_a_cancelled_render_stops_its_sandbox( + self, overrides: None, settings: RendererSettings, config: ExecutorConfig, launcher: FakeLauncher + ) -> None: + renderer = Renderer(settings, SandboxExecutor(config)) + app.dependency_overrides[get_renderer] = lambda: renderer + backend = RemoteBackend(url=URL, tokens=StaticTokens(), transport=httpx.ASGITransport(app=app)) + task = asyncio.create_task(backend.render(render_job(source=SLEEPER))) + for _ in range(200): + if launcher.launches(): + break + await asyncio.sleep(0.05) + task.cancel() + + with pytest.raises(asyncio.CancelledError): + await task + await backend.aclose() + assert len(launcher.launches()) == 1 and launcher.deletes() == launcher.launches() + assert renderer.slot.in_flight == 0 + + +class FakeIdCredentials: + """What `fetch_id_token_credentials` returns: refresh() sets `token`.""" + + def __init__(self, tokens: list[str]) -> None: + self.tokens = tokens + self.token: str | None = None + self.refreshes = 0 + + def refresh(self, request: object) -> None: + self.token = self.tokens[self.refreshes] + self.refreshes += 1 + + +class FakeUserCredentials(google.oauth2.credentials.Credentials): + """A user's Application Default Credentials: refresh() brings an ID token for the ADC client.""" + + def __init__(self, token: str | None) -> None: + super().__init__(token=None) + self.next_id_token = token + + def refresh(self, request: object) -> None: + self._id_token = self.next_id_token + + +class TestIdTokens: + async def test_a_static_token_is_sent_until_it_expires(self) -> None: + source = IdTokenSource(URL, development=True, static_token=GOOD) + assert await source.token() == GOOD + + expired = IdTokenSource(URL, development=True, static_token=id_token(exp=1)) + with pytest.raises(RendererUnavailable, match="AGENT_RENDER_TOKEN has expired"): + await expired.token() + + async def test_deployed_tokens_come_from_the_metadata_server_and_are_cached( + self, monkeypatch: pytest.MonkeyPatch + ) -> None: + now = 1_800_000_000.0 + first, second = id_token(aud=URL, exp=int(now) + 3600), id_token(aud=URL, exp=int(now) + 7200) + credentials = FakeIdCredentials([first, second]) + audiences: list[str] = [] + + def fetch(audience: str, request: object = None) -> FakeIdCredentials: + audiences.append(audience) + return credentials + + monkeypatch.setattr(remote_module.google_id_token, "fetch_id_token_credentials", fetch) + clock = [now] + source = IdTokenSource(URL, development=False, clock=lambda: clock[0]) + + assert await source.token() == first + assert await source.token() == first # cached + clock[0] = now + 3600 - 120 # inside the refresh margin + assert await source.token() == second + assert audiences == [URL, URL] and credentials.refreshes == 2 + + async def test_development_uses_the_users_adc_id_token(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) + user_token = id_token(aud="764086051850-adc.apps.googleusercontent.com", email="owner@example.com") + monkeypatch.setattr("google.auth.default", lambda: (FakeUserCredentials(user_token), "anyplot")) + + def no_fetch(*args: object, **kwargs: object) -> None: + raise AssertionError("a user's ADC never go through the metadata path") + + monkeypatch.setattr(remote_module.google_id_token, "fetch_id_token_credentials", no_fetch) + + assert await IdTokenSource(URL, development=True).token() == user_token + + async def test_adc_without_an_id_token_is_unavailable(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.delenv("GOOGLE_APPLICATION_CREDENTIALS", raising=False) + monkeypatch.setattr("google.auth.default", lambda: (FakeUserCredentials(None), "anyplot")) + + with pytest.raises(RendererUnavailable, match="carry no ID token"): + await IdTokenSource(URL, development=True).token() + + async def test_missing_credentials_are_unavailable_with_a_hint(self, monkeypatch: pytest.MonkeyPatch) -> None: + def broken(*args: object, **kwargs: object) -> None: + raise google.auth.exceptions.DefaultCredentialsError("no credentials") + + monkeypatch.setattr(remote_module.google_id_token, "fetch_id_token_credentials", broken) + + with pytest.raises(RendererUnavailable, match="DefaultCredentialsError"): + await IdTokenSource(URL, development=False).token() + + def test_token_expiry(self) -> None: + assert token_expiry(id_token(exp=1234)) == 1234.0 + assert token_expiry("not-a-token") is None + assert token_expiry("a.%%%.c") is None + + +def test_tokens_are_minted_off_the_event_loop(monkeypatch: pytest.MonkeyPatch) -> None: + """google-auth blocks; minting must run in a worker thread.""" + threads: list[bool] = [] + + def fetch(audience: str, request: object = None) -> FakeIdCredentials: + try: + asyncio.get_running_loop() + threads.append(False) + except RuntimeError: + threads.append(True) + return FakeIdCredentials([GOOD]) + + monkeypatch.setattr(remote_module.google_id_token, "fetch_id_token_credentials", fetch) + + assert asyncio.run(IdTokenSource(URL, development=False).token()) == GOOD + assert threads == [True] diff --git a/tests/unit/agents/renderer/test_service.py b/tests/unit/agents/renderer/test_service.py new file mode 100644 index 0000000000..abe143f0e0 --- /dev/null +++ b/tests/unit/agents/renderer/test_service.py @@ -0,0 +1,468 @@ +"""Tests for agents/renderer/main.py and auth.py: the caller check, the /render contract, the slot and /status.""" + +import asyncio +import base64 +import json +import time +from collections.abc import AsyncIterator, Iterator +from pathlib import Path +from typing import Any + +import httpx +import pytest + +from agents.renderer import main as main_module +from agents.renderer.auth import decode_claims +from agents.renderer.executor import ExecutorConfig, SandboxExecutor, Unavailable +from agents.renderer.main import Renderer, app, get_renderer +from agents.renderer.settings import RendererSettings, get_settings +from agents.renderer.wire import RendererStatus, RenderResponse + +from .conftest import FakeLauncher +from .helpers import AGENTS_SA, SLEEPER, URL, FakeExecutor, id_token, job + + +GOOD = id_token(aud=URL, email=AGENTS_SA) + + +def body(**overrides: Any) -> dict[str, Any]: + return {**job(themes=("light", "dark")).model_dump(mode="json"), **overrides} + + +@pytest.fixture +def settings() -> RendererSettings: + return RendererSettings( + environment="production", audiences=[URL], allowed_callers=[AGENTS_SA], min_mem_available_mb=0 + ) + + +@pytest.fixture +def fake() -> FakeExecutor: + return FakeExecutor() + + +@pytest.fixture +def renderer(settings: RendererSettings, fake: FakeExecutor) -> Renderer: + return Renderer(settings, fake) + + +@pytest.fixture +def overrides(settings: RendererSettings, renderer: Renderer) -> Iterator[None]: + app.dependency_overrides[get_settings] = lambda: settings + app.dependency_overrides[get_renderer] = lambda: renderer + yield + app.dependency_overrides.clear() + + +@pytest.fixture +async def client(overrides: None) -> AsyncIterator[httpx.AsyncClient]: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=URL) as client: + yield client + + +def auth(token: str = GOOD) -> dict[str, str]: + return {"Authorization": f"Bearer {token}"} + + +class TestCallerCheck: + async def test_no_token_is_unauthenticated(self, client: httpx.AsyncClient) -> None: + for headers in ({}, {"Authorization": "Bearer not-a-jwt"}, {"Authorization": "Basic abc"}): + response = await client.get("/status", headers=headers) + assert response.status_code == 401 and response.json() == {"detail": "unauthenticated"} + + @pytest.mark.parametrize( + "claims", + [ + {"aud": "https://elsewhere.run.app", "email": AGENTS_SA}, + {"aud": URL, "email": "someone@example.com"}, + {"aud": URL}, + {"aud": ["https://elsewhere.run.app"], "email": AGENTS_SA}, + ], + ) + async def test_a_token_for_another_audience_or_caller_is_forbidden( + self, client: httpx.AsyncClient, claims: dict[str, Any] + ) -> None: + response = await client.post("/render", json=body(), headers=auth(id_token(**claims))) + + assert response.status_code == 403 and response.json() == {"detail": "forbidden"} + + async def test_the_agents_service_account_passes(self, client: httpx.AsyncClient) -> None: + assert (await client.get("/status", headers=auth())).status_code == 200 + assert (await client.get("/status", headers=auth(id_token(aud=[URL], email=AGENTS_SA)))).status_code == 200 + + async def test_an_owner_user_token_passes_once_its_client_id_and_email_are_listed( + self, client: httpx.AsyncClient, settings: RendererSettings + ) -> None: + """A user's ID token names the OAuth client of gcloud or ADC as its audience, never the URL.""" + gcloud_client = "32555940559.apps.googleusercontent.com" + token = id_token(aud=gcloud_client, email="owner@example.com") + assert (await client.get("/status", headers=auth(token))).status_code == 403 + + settings.audiences.append(gcloud_client) + settings.allowed_callers.append("owner@example.com") + + assert (await client.get("/status", headers=auth(token))).status_code == 200 + + async def test_development_skips_the_check(self, client: httpx.AsyncClient, settings: RendererSettings) -> None: + settings.environment = "development" + + assert (await client.get("/status")).status_code == 200 + + def test_development_is_refused_on_cloud_run(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("ENVIRONMENT", "development") + monkeypatch.setenv("K_SERVICE", "anyplot-renderer") + + with pytest.raises(ValueError, match="refused on Cloud Run"): + RendererSettings() + + async def test_x_serverless_authorization_wins_over_authorization(self, client: httpx.AsyncClient) -> None: + """With both headers Cloud Run checks only X-Serverless-Authorization; Authorization reaches us unverified.""" + forged = auth(id_token(aud=URL, email=AGENTS_SA)) + invoker = {"X-Serverless-Authorization": f"Bearer {id_token(aud=URL, email='editor@example.com')}"} + + refused = await client.get("/status", headers={**forged, **invoker}) + alone = await client.get("/status", headers={"X-Serverless-Authorization": f"Bearer {GOOD}"}) + empty = await client.get("/status", headers={**forged, "X-Serverless-Authorization": ""}) + + assert refused.status_code == 403 and refused.json() == {"detail": "forbidden"} + assert alone.status_code == 200 + assert empty.status_code == 401 # present but empty: never falls back to Authorization + + def test_decode_claims_ignores_the_signature(self) -> None: + claims = decode_claims(f"Bearer {GOOD}") + assert claims is not None and claims["aud"] == URL and claims["email"] == AGENTS_SA + assert decode_claims(f"bearer {GOOD}") == claims + assert decode_claims("Bearer a.b") is None + assert decode_claims("Bearer a.!!!.c") is None + + +class TestRender: + async def test_the_contract_round_trips(self, client: httpx.AsyncClient, fake: FakeExecutor) -> None: + response = await client.post("/render", json=body(), headers=auth()) + + assert response.status_code == 200 and response.headers["cache-control"] == "no-store" + answer = RenderResponse.model_validate_json(response.content) + assert answer.job_id == "job123" and set(answer.outputs) == {"light", "dark"} + light = answer.outputs["light"] + assert light.exit_code == 0 and light.png_base64 is not None + assert base64.b64decode(light.png_base64).startswith(b"\x89PNG") + assert answer.timings.total_s >= answer.timings.slot_wait_s >= 0 + assert fake.calls == [("job123", "light"), ("job123", "dark")] # one theme after the other + + async def test_an_invalid_job_is_refused_without_echoing_it(self, client: httpx.AsyncClient) -> None: + for bad in ( + body(job_id="../escape", source="SECRET-CELL-VALUE"), + body(language="r", source="SECRET-CELL-VALUE"), + body(themes=["light", "light"], source="SECRET-CELL-VALUE"), + body(timeout_s=600, source="SECRET-CELL-VALUE"), + body(extra="x", source="SECRET-CELL-VALUE"), + ): + response = await client.post("/render", json=bad, headers=auth()) + assert response.status_code == 422 and response.json() == {"detail": "invalid_job"} + assert "SECRET" not in response.text + + async def test_renders_are_serial(self, client: httpx.AsyncClient, fake: FakeExecutor) -> None: + responses = await asyncio.gather( + *(client.post("/render", json=body(job_id=f"job{index}"), headers=auth()) for index in range(3)) + ) + + assert [response.status_code for response in responses] == [200, 200, 200] + assert fake.max_active == 1 and len(fake.calls) == 6 + + async def test_a_request_that_cannot_get_the_slot_is_busy( + self, client: httpx.AsyncClient, fake: FakeExecutor, settings: RendererSettings + ) -> None: + settings.slot_wait_s = 0.05 + fake.gate = asyncio.Event() + first = asyncio.create_task(client.post("/render", json=body(), headers=auth())) + while fake.active == 0: + await asyncio.sleep(0.01) + + second = await client.post("/render", json=body(job_id="other"), headers=auth()) + fake.gate.set() + + assert second.status_code == 503 and second.json() == {"detail": "busy"} + assert (await first).status_code == 200 + + async def test_a_replay_while_the_render_runs_joins_it(self, client: httpx.AsyncClient, fake: FakeExecutor) -> None: + """The backend replays a POST after an ambiguous failure; the first may have arrived. The code runs once.""" + fake.gate = asyncio.Event() + first = asyncio.create_task(client.post("/render", json=body(), headers=auth())) + while fake.active == 0: + await asyncio.sleep(0.01) + second = asyncio.create_task(client.post("/render", json=body(), headers=auth())) + await asyncio.sleep(0.05) + fake.gate.set() + + responses = await asyncio.gather(first, second) + + assert [response.status_code for response in responses] == [200, 200] + assert responses[0].content == responses[1].content + assert fake.calls == [("job123", "light"), ("job123", "dark")] + + async def test_a_replay_after_the_render_finished_gets_the_stored_answer( + self, client: httpx.AsyncClient, fake: FakeExecutor + ) -> None: + first = await client.post("/render", json=body(), headers=auth()) + second = await client.post("/render", json=body(), headers=auth()) + + assert first.status_code == second.status_code == 200 and first.content == second.content + assert fake.calls == [("job123", "light"), ("job123", "dark")] + + async def test_another_payload_under_the_same_job_id_is_a_conflict( + self, client: httpx.AsyncClient, fake: FakeExecutor + ) -> None: + assert (await client.post("/render", json=body(), headers=auth())).status_code == 200 + + response = await client.post("/render", json=body(source="print('other')"), headers=auth()) + + assert response.status_code == 409 and response.json() == {"detail": "job_conflict"} + assert len(fake.calls) == 2 + + async def test_each_theme_request_of_a_job_is_its_own_render( + self, client: httpx.AsyncClient, fake: FakeExecutor + ) -> None: + """The backend sends one request per theme under the job's id (render/serial.py); the key includes the themes.""" + light = await client.post("/render", json=body(themes=["light"]), headers=auth()) + dark = await client.post("/render", json=body(themes=["dark"]), headers=auth()) + + assert light.status_code == dark.status_code == 200 + assert fake.calls == [("job123", "light"), ("job123", "dark")] + + async def test_the_stored_answers_are_bounded_and_refusals_are_not_stored( + self, client: httpx.AsyncClient, fake: FakeExecutor + ) -> None: + fake.error = Unavailable("stuck") + assert (await client.post("/render", json=body(), headers=auth())).status_code == 503 + fake.error = None + assert (await client.post("/render", json=body(), headers=auth())).status_code == 200 # ran again + for index in range(main_module.DONE_CACHE): + assert (await client.post("/render", json=body(job_id=f"later{index}"), headers=auth())).status_code == 200 + calls_before = len(fake.calls) + + assert (await client.post("/render", json=body(), headers=auth())).status_code == 200 + + assert len(fake.calls) == calls_before + 2 # job123 fell out of the cache and rendered again + + async def test_only_the_last_waiter_to_leave_may_cancel(self, renderer: Renderer, fake: FakeExecutor) -> None: + fake.gate = asyncio.Event() + request = job(themes=("light", "dark")) + task = renderer.join(request) + assert renderer.join(request) is task + + assert renderer.leave(request) is False # one waiter is still there + assert renderer.leave(request) is True # the last one: this caller cancels + assert renderer.leave(request) is False # a stray extra leave changes nothing + + task.cancel() + await asyncio.wait({task}) + assert renderer.cancel("job123") == 0 + + async def test_low_memory_refuses_before_any_sandbox( + self, client: httpx.AsyncClient, fake: FakeExecutor, settings: RendererSettings, monkeypatch: pytest.MonkeyPatch + ) -> None: + settings.min_mem_available_mb = 1024 + monkeypatch.setattr(main_module, "mem_available_mb", lambda: 700) + + response = await client.post("/render", json=body(), headers=auth()) + + assert response.status_code == 503 and response.json() == {"detail": "low_memory"} + assert fake.calls == [] + + async def test_an_unreadable_memory_signal_does_not_block( + self, client: httpx.AsyncClient, settings: RendererSettings, monkeypatch: pytest.MonkeyPatch + ) -> None: + settings.min_mem_available_mb = 1024 + monkeypatch.setattr(main_module, "mem_available_mb", lambda: None) + + assert (await client.post("/render", json=body(), headers=auth())).status_code == 200 + + async def test_an_executor_refusal_is_a_503_with_its_code( + self, client: httpx.AsyncClient, fake: FakeExecutor + ) -> None: + fake.error = Unavailable("stuck") + + response = await client.post("/render", json=body(), headers=auth()) + + assert response.status_code == 503 and response.json() == {"detail": "stuck"} + + async def test_an_unexpected_error_is_a_json_500_without_its_message( + self, client: httpx.AsyncClient, fake: FakeExecutor, caplog: pytest.LogCaptureFixture + ) -> None: + """A plain-text 500 read as a front-end error, so the backend re-ran the hostile job.""" + fake.error = OverflowError("SECRET-CELL-VALUE") + + response = await client.post("/render", json=body(), headers=auth()) + + assert response.status_code == 500 and response.json() == {"detail": "internal"} + assert "SECRET" not in response.text and "SECRET" not in caplog.text + assert "OverflowError" in caplog.text + + async def test_the_volume_must_be_in_place_on_cloud_run( + self, client: httpx.AsyncClient, fake: FakeExecutor, renderer: Renderer + ) -> None: + renderer.volume = False + + response = await client.post("/render", json=body(), headers=auth()) + status = RendererStatus.model_validate((await client.get("/status", headers=auth())).json()) + + assert response.status_code == 503 and response.json() == {"detail": "volume_missing"} + assert fake.calls == [] and status.volume is False + + def test_the_volume_is_checked_only_on_cloud_run( + self, settings: RendererSettings, fake: FakeExecutor, monkeypatch: pytest.MonkeyPatch + ) -> None: + checks: list[tuple[Path, int]] = [] + + def check(runs_dir: Path, max_bytes: int) -> bool: + checks.append((runs_dir, max_bytes)) + return False + + monkeypatch.setattr(main_module, "runs_volume_ok", check) + + assert Renderer(settings, fake).volume is None and checks == [] + on_cloud_run = settings.model_copy(update={"cloud_run_service": "anyplot-renderer"}) + assert Renderer(on_cloud_run, fake).volume is False + assert checks == [(Path("/tmp/runs"), 1024 * 1024 * 1024)] + + async def test_a_stuck_launcher_ends_the_instance_on_cloud_run( + self, client: httpx.AsyncClient, fake: FakeExecutor, settings: RendererSettings + ) -> None: + exits: list[bool] = [] + stuck = Renderer( + settings.model_copy(update={"cloud_run_service": "anyplot-renderer"}), + fake, + terminate=lambda: exits.append(True), + ) + stuck.volume = None + app.dependency_overrides[get_renderer] = lambda: stuck + fake.stuck = 1 + fake.error = Unavailable("stuck") + + first = await client.post("/render", json=body(), headers=auth()) + second = await client.post("/render", json=body(), headers=auth()) + + assert first.status_code == second.status_code == 503 + assert exits == [True] # once + + async def test_a_stuck_launcher_off_cloud_run_keeps_the_process( + self, client: httpx.AsyncClient, fake: FakeExecutor, renderer: Renderer + ) -> None: + exits: list[bool] = [] + renderer.terminate = lambda: exits.append(True) + fake.stuck = 1 + + await client.post("/render", json=body(), headers=auth()) + + assert exits == [] + + +class TestCancellation: + """uvicorn does not cancel a handler whose client left; the route must notice and stop the sandbox.""" + + @pytest.fixture + def real(self, settings: RendererSettings, config: ExecutorConfig) -> Iterator[Renderer]: + renderer = Renderer(settings, SandboxExecutor(config)) + app.dependency_overrides[get_settings] = lambda: settings + app.dependency_overrides[get_renderer] = lambda: renderer + yield renderer + app.dependency_overrides.clear() + + async def test_a_client_that_goes_away_stops_the_sandbox(self, real: Renderer, launcher: FakeLauncher) -> None: + payload = json.dumps(job(SLEEPER).model_dump(mode="json")).encode() + received: list[str] = [] + + async def receive() -> dict[str, Any]: + if not received: + received.append("body") + return {"type": "http.request", "body": payload, "more_body": False} + for _ in range(200): + if launcher.launches(): + break + await asyncio.sleep(0.05) + await asyncio.sleep(0.2) + return {"type": "http.disconnect"} + + sent: list[dict[str, Any]] = [] + + async def send(message: dict[str, Any]) -> None: + sent.append(message) + + scope = { + "type": "http", + "asgi": {"version": "3.0"}, + "http_version": "1.1", + "method": "POST", + "scheme": "https", + "path": "/render", + "raw_path": b"/render", + "query_string": b"", + "root_path": "", + "headers": [ + (b"host", b"renderer"), + (b"content-type", b"application/json"), + (b"authorization", f"Bearer {GOOD}".encode()), + ], + "client": ("127.0.0.1", 50000), + "server": ("renderer", 443), + } + started = time.monotonic() + + await asyncio.wait_for(app(scope, receive, send), 30) + + assert time.monotonic() - started < 20 # not the 30 s the sleeper asked for + assert len(launcher.launches()) == 1 and launcher.deletes() == launcher.launches() + assert sent[0]["status"] == 499 + assert real.slot.in_flight == 0 and real.executor.stuck == 0 + + async def test_the_cancel_route_stops_a_running_render(self, real: Renderer, launcher: FakeLauncher) -> None: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=URL) as client: + pending = asyncio.create_task( + client.post("/render", json=job(SLEEPER).model_dump(mode="json"), headers=auth()) + ) + for _ in range(200): + if launcher.launches(): + break + await asyncio.sleep(0.05) + + cancelled = await client.post("/render/job123/cancel", headers=auth()) + answer = await asyncio.wait_for(pending, 20) + again = await client.post("/render/job123/cancel", headers=auth()) + + assert cancelled.status_code == 200 and cancelled.json() == {"cancelled": 1} + assert answer.status_code == 499 and answer.json() == {"detail": "cancelled"} + assert again.json() == {"cancelled": 0} + assert launcher.deletes() == launcher.launches() and real.slot.in_flight == 0 + + async def test_the_cancel_route_checks_the_caller_and_the_id(self, real: Renderer) -> None: + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url=URL) as client: + anonymous = await client.post("/render/job123/cancel") + bad = await client.post("/render/..%2Fescape/cancel", headers=auth()) + + assert anonymous.status_code == 401 + assert bad.status_code in (404, 422) + + +class TestStatus: + async def test_status_reports_the_slot_and_the_memory_signal( + self, client: httpx.AsyncClient, monkeypatch: pytest.MonkeyPatch + ) -> None: + monkeypatch.setattr(main_module, "mem_available_mb", lambda: 3500) + + response = await client.get("/status", headers=auth()) + + status = RendererStatus.model_validate(response.json()) + assert status.name == "anyplot-renderer" and status.version + assert (status.in_flight, status.waiting, status.stuck) == (0, 0, 0) + assert status.mem_available_mb == 3500 and status.sandbox is True + + def test_the_default_renderer_uses_the_sandbox_executor(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("RENDERER_ALLOWED_CALLERS", f"{AGENTS_SA}, owner@example.com") + monkeypatch.setenv("RENDERER_AUDIENCES", URL) + + renderer = get_renderer() + + assert type(renderer.executor).__name__ == "SandboxExecutor" + assert renderer.settings.allowed_callers == [AGENTS_SA, "owner@example.com"] + assert renderer.settings.audiences == [URL] + assert renderer.settings.environment == "production" diff --git a/tests/unit/agents/renderer/test_wire.py b/tests/unit/agents/renderer/test_wire.py new file mode 100644 index 0000000000..e92d216ddb --- /dev/null +++ b/tests/unit/agents/renderer/test_wire.py @@ -0,0 +1,127 @@ +"""Tests for agents/renderer/wire.py and settings.py, and for what the renderer package may import.""" + +import os +import subprocess +import sys +from pathlib import Path + +import pytest +from pydantic import ValidationError + +from agents.anyplot.render import contract, png +from agents.anyplot.schemas import RENDER_ID_PATTERN +from agents.renderer import wire +from agents.renderer.settings import RendererSettings + +from .helpers import job + + +REPO_ROOT = Path(__file__).resolve().parents[4] + + +class TestContract: + def test_limits_shared_with_the_agents_side_are_equal(self) -> None: + """The renderer cannot import agents.anyplot (it loads ADK), so the pins live here.""" + assert wire.MAX_PNG_BYTES == png.MAX_PNG_BYTES + assert wire.MAX_PROBE_BYTES == png.MAX_PROBE_BYTES + assert wire.MAX_STDERR_CHARS == contract.MAX_STDERR_CHARS + assert wire.JOB_ID_PATTERN == RENDER_ID_PATTERN + + def test_a_job_names_distinct_known_themes_and_a_bounded_timeout(self) -> None: + assert job(themes=("light", "dark")).themes == ["light", "dark"] + for overrides in ({"themes": []}, {"themes": ["light", "light"]}, {"themes": ["sepia"]}, {"timeout_s": 0}): + with pytest.raises(ValidationError): + wire.RenderRequest.model_validate({**job().model_dump(), **overrides}) + with pytest.raises(ValidationError): + wire.RenderRequest.model_validate({**job().model_dump(), "timeout_s": wire.MAX_TIMEOUT_S + 1}) + + +class TestCleanProbe: + def test_a_harness_probe_passes_and_nan_becomes_null(self) -> None: + raw = b'{"canvas": [3200, 1800], "texts": [{"box": [1.5, 2, NaN, Infinity]}], "tick_overlaps": 0}' + + assert wire.clean_probe(raw) == { + "canvas": [3200, 1800], + "texts": [{"box": [1.5, 2, None, None]}], + "tick_overlaps": 0, + } + + @pytest.mark.parametrize( + "raw", + [ + None, + b"", + b"[1, 2]", + b"not json", + b"\xff\xfe", + b'{"a": ' + b"[" * 16 + b"]" * 16 + b"}", + b'{"a": ' + b"[" * 5000 + b"]" * 5000 + b"}", + b'{"big": 1' + b"0" * 400 + b"}", + b'{"big": 1e999}', + ], + ) + def test_anything_the_wire_cannot_carry_is_dropped(self, raw: bytes | None) -> None: + assert wire.clean_probe(raw) is None + + def test_the_depth_limit_counts_the_object_itself(self) -> None: + nested = b'{"a": ' + b"[" * 15 + b"]" * 15 + b"}" + + assert wire.clean_probe(nested) is not None + + +class TestPackage: + def test_the_renderer_imports_neither_adk_nor_the_agents_runtime_nor_core(self) -> None: + """The image holds only `agents/renderer/` and the plotting venv: no ADK, no `agents/anyplot/`, no `core/`.""" + code = ( + "import sys\n" + "import agents.renderer.main\n" + "loaded = [name for name in sys.modules if name.startswith(('google.adk', 'agents.anyplot', 'core.'))\n" + " or name == 'core']\n" + "assert not loaded, loaded\n" + ) + env = {**os.environ, "PYTHON_DOTENV_DISABLED": "1"} + done = subprocess.run( + [sys.executable, "-c", code], cwd=REPO_ROOT, env=env, capture_output=True, text=True, timeout=120 + ) + + assert done.returncode == 0, done.stderr[-2000:] + + def test_the_renderer_directory_is_no_adk_agent(self) -> None: + """`adk web agents` lists every subdirectory with an `__init__.py` as an agent.""" + renderer = REPO_ROOT / "agents" / "renderer" + + assert not (renderer / "__init__.py").exists() and not (renderer / "agent.py").exists() + + +class TestSettings: + def test_defaults_are_the_deployed_values(self) -> None: + settings = RendererSettings() + + assert settings.environment == "production" and not settings.is_development + assert settings.allowed_callers == [] and settings.audiences == [] + assert settings.runs_dir == "/tmp/runs" and settings.run_budget_mb == 64 + assert settings.slot_wait_s == 30 and settings.min_mem_available_mb == 1024 + assert settings.kill_mem_available_mb == 512 and settings.runs_volume_max_mb == 1024 + assert (settings.rlimit_cpu_s, settings.rlimit_fsize_mb, settings.rlimit_as_mb, settings.rlimit_nproc) == ( + 60, + 50, + 1024, + 64, + ) + assert settings.bind_tmp is False + + def test_lists_and_values_come_from_the_environment(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("RENDERER_ALLOWED_CALLERS", "a@x.iam.gserviceaccount.com,,b@example.com ") + monkeypatch.setenv("RENDERER_AUDIENCES", "") + monkeypatch.setenv("RENDERER_RUN_BUDGET_MB", "32") + monkeypatch.setenv("K_REVISION", "anyplot-renderer-b123") + + settings = RendererSettings() + + assert settings.allowed_callers == ["a@x.iam.gserviceaccount.com", "b@example.com"] + assert settings.audiences == [] and settings.run_budget_mb == 32 + assert settings.revision == "anyplot-renderer-b123" + + def test_paths_must_be_absolute(self) -> None: + with pytest.raises(ValidationError, match="absolute"): + RendererSettings(runs_dir="runs") diff --git a/tests/unit/agents/runtime/test_render.py b/tests/unit/agents/runtime/test_render.py index 9ecc515e32..ac86fe976d 100644 --- a/tests/unit/agents/runtime/test_render.py +++ b/tests/unit/agents/runtime/test_render.py @@ -20,6 +20,7 @@ from agents.anyplot.render import make_backend from agents.anyplot.render.backends.fake import FakeBackend, FakeOutcome, fixture_png from agents.anyplot.render.backends.local import LocalDockerBackend, read_tail +from agents.anyplot.render.backends.remote import IdTokenSource, RemoteBackend from agents.anyplot.render.backends.sandbox import SandboxBackend from agents.anyplot.render.contract import THEMES, RendererUnavailable, RenderJob, RenderResult, Theme, ThemeOutput from agents.anyplot.render.gates import data_rows, error_summary, evaluate @@ -427,8 +428,8 @@ def test_local_backend_refuses_production_and_missing_docker(self, monkeypatch: with pytest.raises(RendererUnavailable, match="Docker"): LocalDockerBackend(image="x", runtime=PythonRuntime(), environment="development") - async def test_sandbox_backend_waits_for_spike_s(self) -> None: - with pytest.raises(NotImplementedError, match="spike S"): + async def test_in_process_sandbox_backend_is_not_used_in_phase_1(self) -> None: + with pytest.raises(NotImplementedError, match="not used in phase 1"): await SandboxBackend(runtime=PythonRuntime()).render(job()) async def test_fake_backend_scripts(self) -> None: @@ -441,8 +442,11 @@ async def test_fake_backend_scripts(self) -> None: def test_factory(self) -> None: assert isinstance(make_backend(AgentSettings(renderer="fake")), FakeBackend) assert isinstance(make_backend(AgentSettings(renderer="sandbox")), SandboxBackend) - with pytest.raises(RendererUnavailable): + with pytest.raises(RendererUnavailable, match="AGENT_RENDER_URL"): make_backend(AgentSettings(renderer="remote")) + remote = make_backend(AgentSettings(renderer="remote", render_url="https://anyplot-renderer.example.run.app/")) + assert isinstance(remote, RemoteBackend) and remote.url == "https://anyplot-renderer.example.run.app" + assert isinstance(remote.tokens, IdTokenSource) and remote.tokens.audience == remote.url def test_job_validation(self) -> None: with pytest.raises(ValueError): diff --git a/tests/unit/agents/runtime/test_stream.py b/tests/unit/agents/runtime/test_stream.py index a7163533d2..70517cac16 100644 --- a/tests/unit/agents/runtime/test_stream.py +++ b/tests/unit/agents/runtime/test_stream.py @@ -185,8 +185,10 @@ def token(claims: dict) -> str: class FakeRequest: - def __init__(self, authorization: str | None) -> None: + def __init__(self, authorization: str | None, serverless: str | None = None) -> None: self.headers = {"authorization": authorization} if authorization else {} + if serverless is not None: + self.headers["x-serverless-authorization"] = serverless class TestCallerCheck: @@ -216,3 +218,14 @@ def test_production_requires_audience_and_caller(self, monkeypatch: pytest.Monke with pytest.raises(AgentsError) as caught: check_caller(FakeRequest(token(claims) if claims else None)) assert caught.value.status == status + + # With X-Serverless-Authorization present, Cloud Run checks only that header and + # passes Authorization through unverified: a forged Authorization must not count. + invoker = token({**good, "email": "editor@example.com"}) + with pytest.raises(AgentsError) as caught: + check_caller(FakeRequest(token(good), serverless=invoker)) + assert caught.value.status == 403 + check_caller(FakeRequest(None, serverless=token(good))) + with pytest.raises(AgentsError) as caught: + check_caller(FakeRequest(token(good), serverless="")) + assert caught.value.status == 401 diff --git a/tests/unit/agents/test_settings.py b/tests/unit/agents/test_settings.py index a5066669d8..bda6ff3ea3 100644 --- a/tests/unit/agents/test_settings.py +++ b/tests/unit/agents/test_settings.py @@ -38,7 +38,9 @@ def test_defaults_are_the_pinned_production_values(self) -> None: assert settings.service_urls == [] assert settings.dev_fixture is None assert settings.libraries == ["matplotlib", "seaborn"] - assert settings.renderer == "sandbox" + assert settings.renderer == "remote" # phase 1 renders through the anyplot-renderer service + assert settings.render_url is None and settings.render_token is None + assert settings.render_image == "anyplot-renderer:dev" assert settings.max_llm_calls == 12 assert settings.request_token_budget == 80_000 assert settings.daily_token_budget == 1_000_000 @@ -270,6 +272,44 @@ def test_unknown_renderer_is_refused(self) -> None: AgentSettings(renderer="subprocess") +class TestRemoteRendererSettings: + def test_render_url_comes_from_the_environment_without_its_slash(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AGENT_RENDER_URL", "https://anyplot-renderer-239660669828.europe-west4.run.app/") + + assert AgentSettings().render_url == "https://anyplot-renderer-239660669828.europe-west4.run.app" + + def test_blank_render_url_is_unset(self, monkeypatch: pytest.MonkeyPatch) -> None: + monkeypatch.setenv("AGENT_RENDER_URL", " ") + + assert AgentSettings().render_url is None + + @pytest.mark.parametrize( + "url", + ["https://renderer.example/render", "https://user@renderer.example", "ftp://renderer.example", "renderer"], + ) + def test_render_url_is_an_origin(self, url: str) -> None: + with pytest.raises(ValidationError): + AgentSettings(render_url=url) + + def test_plain_http_is_refused_in_production(self) -> None: + with pytest.raises(ValidationError, match="https"): + AgentSettings(render_url="http://localhost:8003", environment="production") + + @pytest.mark.parametrize("environment", ["development", "test"]) + def test_plain_http_is_allowed_in_development_and_test(self, environment: str) -> None: + settings = AgentSettings(render_url="http://localhost:8003", environment=environment) + + assert settings.render_url == "http://localhost:8003" + + def test_render_token_is_development_only(self) -> None: + settings = AgentSettings(render_token="a.b.c", environment="development") + + assert settings.render_token is not None and settings.render_token.get_secret_value() == "a.b.c" + assert "a.b.c" not in repr(settings) + with pytest.raises(ValidationError, match="AGENT_RENDER_TOKEN"): + AgentSettings(render_token="a.b.c", environment="production") + + class TestOtherValidators: def test_unknown_library_is_refused(self) -> None: with pytest.raises(ValidationError, match="unknown libraries"):