test(load): add bounded Docker load-testing harness and issue ledger

Author: NathanFlurry

.gitignore

@@ -81,6 +81,7 @@ scripts/ralph/.codex-last-msg-*
 .claude/scheduled_tasks.lock
 
 # Local caches and stray outputs
+.artifacts/
 crates/execution/assets/v8-bridge.js
 crates/execution/assets/v8-bridge-zlib.js
 crates/execution/.agentos-pyodide-cache/

CLAUDE.md

@@ -4,6 +4,10 @@ AgentOS owns the runtime, kernel, VFS, language execution, registry packages,
 ACP/session layer, AgentOS client APIs, docs, and publish machinery. The
 `secure-exec` repository is now a generated compatibility mirror only.
 
+For future RivetKit work, start with the current documentation index at
+https://rivet.dev/llms.txt and follow the linked page for the surface being
+changed.
+
 ## Boundaries
 
 - Keep AgentOS product versions pinned at `0.0.1` in committed files. Release

docs-internal/load-testing-issues.md

@@ -181,3 +181,130 @@ test-bounded cmd='pnpm test':
 
 test-risky-probe *tests:
 	./.agent/scripts/run-risky-test-probe.sh "$@"
+
+# Build the exact-workspace image used by every load-test lane. Dockerfile-local
+# ignore rules keep generated artifacts and unrelated website sources out of the
+# build context.
+load-test-image:
+	docker build --file packages/load-tests/Dockerfile --tag agentos-load-tests:local .
+
+# Container-boundary self-test: prove the cgroup envelope, fd ulimit, tmpfs, and
+# network isolation match the flags before trusting any survival verdict. Uses
+# the exact bounded profile of the limit lane.
+load-test-boundary:
+	#!/usr/bin/env bash
+	set -euo pipefail
+	mkdir -p .artifacts/load-tests
+	container="agentos-load-boundary-$$"
+	trap 'docker rm -f "$container" >/dev/null 2>&1 || true' EXIT INT TERM
+	timeout --signal=TERM --kill-after=30s 2m docker run --rm \
+		--name "$container" \
+		--memory=3g --memory-swap=3g --cpus=2 --pids-limit=256 \
+		--ulimit nofile=1024:1024 --network=none \
+		--user "$(id -u):$(id -g)" \
+		--security-opt no-new-privileges --cap-drop=ALL \
+		--tmpfs /tmp:rw,nosuid,nodev,size=512m,mode=1777 \
+		--volume "$PWD/.artifacts/load-tests:/artifacts" \
+		agentos-load-tests:local boundary
+
+# Guest process-limit attack beside a sentinel VM. The workload never runs on
+# the host; the container has hard memory/CPU/PID/fd/swap/time ceilings.
+load-test-limits:
+	#!/usr/bin/env bash
+	set -euo pipefail
+	mkdir -p .artifacts/load-tests
+	container="agentos-load-limits-$$"
+	trap 'docker rm -f "$container" >/dev/null 2>&1 || true' EXIT INT TERM
+	timeout --signal=TERM --kill-after=30s 8m docker run --rm \
+		--name "$container" \
+		--memory=3g --memory-swap=3g --cpus=2 --pids-limit=256 \
+		--ulimit nofile=1024:1024 --network=none \
+		--user "$(id -u):$(id -g)" \
+		--security-opt no-new-privileges --cap-drop=ALL \
+		--tmpfs /tmp:rw,nosuid,nodev,size=512m,mode=1777 \
+		--volume "$PWD/.artifacts/load-tests:/artifacts" \
+		--env LOAD_TEST_PROCESS_LIMIT --env LOAD_TEST_PROCESS_ATTEMPTS \
+		agentos-load-tests:local limits
+
+# High-scale adversarial battery: bounded-but-LARGER cgroup (8 CPU / 8 GiB) so
+# the V8 executor pool (= CPU count) is big enough to actually run hundreds of
+# concurrent VMs. Still a hard-capped container. Runs the `scale` command.
+# Args after the recipe name are passed as the command (default `scale`).
+load-test-scale cmd='scale':
+	#!/usr/bin/env bash
+	set -euo pipefail
+	mkdir -p .artifacts/load-tests
+	container="agentos-load-scale-$$"
+	trap 'docker rm -f "$container" >/dev/null 2>&1 || true' EXIT INT TERM
+	timeout --signal=TERM --kill-after=30s 25m docker run --rm \
+		--name "$container" \
+		--user "$(id -u):$(id -g)" \
+		--memory=8g --memory-swap=8g --cpus=8 --pids-limit=2048 \
+		--ulimit nofile=8192:8192 --network=none \
+		--security-opt no-new-privileges --cap-drop=ALL \
+		--tmpfs /tmp:rw,nosuid,nodev,size=2g,mode=1777 \
+		--volume "$PWD/.artifacts/load-tests:/artifacts" \
+		--env LOAD_TEST_VM_COUNT --env LOAD_TEST_CONCURRENCY --env LOAD_TEST_CYCLES \
+		--env LOAD_TEST_EXEC_CONCURRENCY --env LOAD_TEST_MATRIX_ONLY \
+		agentos-load-tests:local "{{ cmd }}"
+
+# Full deterministic adversarial limit matrix (processes, fds, sockets,
+# filesystem bytes) beside a sentinel, same bounded cgroup as the limit lane.
+load-test-limits-matrix:
+	#!/usr/bin/env bash
+	set -euo pipefail
+	mkdir -p .artifacts/load-tests
+	container="agentos-load-matrix-$$"
+	trap 'docker rm -f "$container" >/dev/null 2>&1 || true' EXIT INT TERM
+	timeout --signal=TERM --kill-after=30s 10m docker run --rm \
+		--name "$container" \
+		--user "$(id -u):$(id -g)" \
+		--memory=3g --memory-swap=3g --cpus=2 --pids-limit=256 \
+		--ulimit nofile=1024:1024 --network=none \
+		--security-opt no-new-privileges --cap-drop=ALL \
+		--tmpfs /tmp:rw,nosuid,nodev,size=512m,mode=1777 \
+		--volume "$PWD/.artifacts/load-tests:/artifacts" \
+		agentos-load-tests:local limits-matrix
+
+# Sequential, burst, and steady-replacement VM churn with leak gates. This is
+# intentionally more generous than the limit lane but remains a bounded cgroup.
+load-test-churn:
+	#!/usr/bin/env bash
+	set -euo pipefail
+	mkdir -p .artifacts/load-tests
+	container="agentos-load-churn-$$"
+	trap 'docker rm -f "$container" >/dev/null 2>&1 || true' EXIT INT TERM
+	timeout --signal=TERM --kill-after=30s 20m docker run --rm \
+		--name "$container" \
+		--memory=4g --memory-swap=4g --cpus=3 --pids-limit=384 \
+		--ulimit nofile=2048:2048 --network=none \
+		--user "$(id -u):$(id -g)" \
+		--security-opt no-new-privileges --cap-drop=ALL \
+		--tmpfs /tmp:rw,nosuid,nodev,size=1g,mode=1777 \
+		--volume "$PWD/.artifacts/load-tests:/artifacts" \
+		--env LOAD_TEST_CYCLES --env LOAD_TEST_BATCH --env LOAD_TEST_SETTLE_MS \
+		--env LOAD_TEST_RSS_SLOPE_BYTES --env LOAD_TEST_RSS_TOTAL_BYTES \
+		--env LOAD_TEST_PSS_TOTAL_BYTES \
+		agentos-load-tests:local churn
+
+# The external Compute load generator is also containerized. Unlike the local
+# lanes it needs egress to the Rivet APIs, but retains hard resource ceilings.
+load-test-compute:
+	#!/usr/bin/env bash
+	set -euo pipefail
+	mkdir -p .artifacts/load-tests
+	container="agentos-load-compute-$$"
+	trap 'docker rm -f "$container" >/dev/null 2>&1 || true' EXIT INT TERM
+	timeout --signal=TERM --kill-after=30s 20m docker run --rm \
+		--name "$container" \
+		--memory=1g --memory-swap=1g --cpus=1 --pids-limit=128 \
+		--ulimit nofile=1024:1024 \
+		--user "$(id -u):$(id -g)" \
+		--security-opt no-new-privileges --cap-drop=ALL \
+		--tmpfs /tmp:rw,nosuid,nodev,size=128m,mode=1777 \
+		--volume "$PWD/.artifacts/load-tests:/artifacts" \
+		--env RIVET_ENDPOINT --env RIVET_PUBLIC_ENDPOINT --env RIVET_RUN_URL \
+		--env COMPUTE_STEPS --env COMPUTE_HOLD_MS --env COMPUTE_SCALE_DOWN_MS \
+		--env COMPUTE_ACTOR_NAME --env COMPUTE_CREATE_CONCURRENCY \
+		--env COMPUTE_READY_TIMEOUT_MS --env COMPUTE_CLEANUP_DEADLINE_MS \
+		agentos-load-tests:local compute-load

docs-internal/load-testing.md

@@ -0,0 +1,60 @@
+# syntax=docker/dockerfile:1.7
+
+# Rust >= 1.91.1 is required by the aws-sdk / smithy dependency tree; pin to the
+# workspace's toolchain (matches the canonical host: cargo 1.97).
+FROM rust:1.97-bookworm AS build
+
+ARG NODE_VERSION=24.4.1
+ARG TARGETARCH
+
+RUN apt-get update \
+	&& apt-get install -y --no-install-recommends ca-certificates clang cmake curl libssl-dev pkg-config python3 xz-utils \
+	&& rm -rf /var/lib/apt/lists/*
+
+RUN case "$TARGETARCH" in \
+		amd64) node_arch=x64 ;; \
+		arm64) node_arch=arm64 ;; \
+		*) echo "unsupported TARGETARCH=$TARGETARCH" >&2; exit 1 ;; \
+	esac \
+	&& curl -fsSL "https://nodejs.org/dist/v${NODE_VERSION}/node-v${NODE_VERSION}-linux-${node_arch}.tar.xz" \
+		| tar -xJ --strip-components=1 -C /usr/local
+
+RUN corepack enable && corepack prepare pnpm@10.13.1 --activate
+
+WORKDIR /app
+COPY . .
+
+# Full install so every package in the build chain has its dev toolchain (tsc,
+# tsup). The chain is heavier than the cargo build only in disk, not wall-clock.
+RUN --mount=type=cache,target=/root/.local/share/pnpm/store \
+	pnpm install --frozen-lockfile
+# The exact-workspace release sidecar the VMs run against. Cargo's registry and
+# target dir are cache-mounted so editing TypeScript (which busts `COPY . .`)
+# does not force a full recompile. The target dir is a cache mount and is not
+# part of the image layer, so the binary is copied to a persisted path.
+RUN --mount=type=cache,target=/usr/local/cargo/registry \
+	--mount=type=cache,target=/app/target \
+	cargo build --release -p agentos-sidecar \
+	&& mkdir -p /app/release-bin \
+	&& cp /app/target/release/agentos-sidecar /app/release-bin/agentos-sidecar
+# Build only the TS packages the load runner needs, in dependency order, without
+# the WASM toolchain (see packages/load-tests/build-deps.sh for why that is
+# sound for node-only, defaultSoftware:false workloads).
+RUN bash packages/load-tests/build-deps.sh
+
+FROM node:24-bookworm-slim
+
+RUN apt-get update \
+	&& apt-get install -y --no-install-recommends ca-certificates tini \
+	&& rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+COPY --from=build /app /app
+
+ENV AGENTOS_SIDECAR_BIN=/app/release-bin/agentos-sidecar
+ENV LOAD_TEST_ARTIFACT_DIR=/artifacts
+ENV RIVET_PORT=3000
+
+EXPOSE 3000
+ENTRYPOINT ["/usr/bin/tini", "--", "node", "--expose-gc", "packages/load-tests/dist/cli.js"]
+CMD ["compute-server"]

justfile

@@ -0,0 +1,15 @@
+.git
+.jj
+.artifacts
+**/node_modules
+**/dist
+**/.turbo
+target
+website
+toolchain/vendor
+toolchain/c/build
+toolchain/c/vendor
+toolchain/c/libs
+toolchain/c/sysroot
+toolchain/c/.cache
+software/*/bin

packages/load-tests/Dockerfile

@@ -0,0 +1,41 @@
+#!/usr/bin/env bash
+# Build exactly the TypeScript packages the load-test image needs, in
+# dependency order, WITHOUT the WASM toolchain.
+#
+# Why this exists (see docs-internal/load-testing.md handoff notes): the load
+# workloads only ever run `node` inside `defaultSoftware:false` VMs, so they
+# never read a `.aospkg` binary. That means we can build the whole chain with
+# `tsc`/`tsup` and skip the heavy toolchain steps:
+#   - `@agentos-software/*` leaves only export a `packagePath` string; importing
+#     them never touches the (unbuilt) WASM binary, so `tsc` alone suffices.
+#   - runtime-core's `copy-commands` (WASM binaries) and `build:protocol`
+#     (regenerates already-present generated sources) are skipped.
+#   - core's `build:protocols` is skipped for the same reason.
+#
+# Run from the repo root. Requires deps installed (`pnpm install`) and, for
+# runtime-core's vm-config generation, a working cargo toolchain.
+set -euo pipefail
+
+echo "==> building @agentos-software chain (tsc, no WASM binaries)"
+# `@agentos-software/common` (the sole default-software import in core) plus its
+# eight leaf packages and the manifest types package.
+for pkg in manifest coreutils sed grep gawk findutils diffutils tar gzip common; do
+	pnpm --filter "@agentos-software/${pkg}" exec tsc
+done
+
+echo "==> building @rivet-dev/agentos-runtime-core (tsc, no copy-commands)"
+# build:vm-config (cargo test) and build:protocol are skipped: their generated
+# sources (src/generated/*.ts) are already present in the tree, so this stage
+# needs no cargo — keeping the TS build layer independent of the Rust toolchain.
+pnpm --filter @rivet-dev/agentos-runtime-core exec tsc
+
+echo "==> building @rivet-dev/agentos-core (tsc)"
+pnpm --filter @rivet-dev/agentos-core exec tsc
+
+echo "==> building @rivet-dev/agentos actor bundle (tsup)"
+pnpm --filter @rivet-dev/agentos build:actor
+
+echo "==> building @rivet-dev/agentos-load-tests"
+pnpm --dir packages/load-tests build
+
+echo "==> load-test dependency build complete"

packages/load-tests/Dockerfile.dockerignore

@@ -0,0 +1,23 @@
+{
+	"name": "@rivet-dev/agentos-load-tests",
+	"version": "0.0.1",
+	"private": true,
+	"type": "module",
+	"scripts": {
+		"build": "tsc",
+		"check-types": "tsc --noEmit",
+		"load:limits": "node --expose-gc dist/cli.js limits",
+		"load:churn": "node --expose-gc dist/cli.js churn",
+		"compute:serve": "node dist/cli.js compute-server",
+		"compute:load": "node dist/cli.js compute-load"
+	},
+	"dependencies": {
+		"@rivet-dev/agentos": "workspace:*",
+		"@rivet-dev/agentos-core": "workspace:*",
+		"rivetkit": "catalog:rivetkit"
+	},
+	"devDependencies": {
+		"@types/node": "^22.10.2",
+		"typescript": "^5.9.2"
+	}
+}

packages/load-tests/build-deps.sh

@@ -0,0 +1,38 @@
+#!/usr/bin/env bash
+# Orchestrate the bounded local load-test lanes and collect their verdicts.
+# Every lane runs inside its `just` Docker wrapper (never on the host). Prints
+# one JSON verdict line per run; the caller records these in the run ledger.
+#
+# Usage: bash packages/load-tests/run-local-lanes.sh [reps]
+set -uo pipefail
+reps="${1:-3}"
+here="$(cd "$(dirname "$0")/../.." && pwd)"
+cd "$here"
+
+run() {
+	local label="$1"; shift
+	echo "### ${label}"
+	if "$@"; then
+		echo "### ${label} recipe exit=0"
+	else
+		echo "### ${label} recipe exit=$? (see verdict line / artifact)"
+	fi
+}
+
+run "boundary" just load-test-boundary
+
+for i in $(seq 1 "$reps"); do
+	run "limits(process) rep ${i}" just load-test-limits
+done
+
+for i in $(seq 1 "$reps"); do
+	run "limits-matrix rep ${i}" just load-test-limits-matrix
+done
+
+# Churn runs a few extra reps to calibrate the provisional RSS/PSS ceilings.
+churn_reps="${2:-5}"
+for i in $(seq 1 "$churn_reps"); do
+	run "churn rep ${i}" just load-test-churn
+done
+
+echo "### all local lanes complete"