refactor(sandbox): host URLs, local grants and the trust CA read ON_THIS_MACHINE, so a VM provider on this machine needs no core edits

AGENTS.md

@@ -96,7 +96,7 @@ and deserialized through a registry.
 | `task/`, `task_step/` | A `Task` holds its `TaskStep`s inline; `Task.run()` executes them as a DAG. `depends_on` (None means all prior steps) gates a step, independent steps run concurrently, `fail_task_on_error` makes a failure fatal or tolerated, `retry_config` rolls a failed span back through the step journal and re-dispatches it. Built-in steps live in `task_step/task_steps/` (`deploy_env`, `deploy_agent`, `prompt_agent`, the verifiers under `verifiers/`, and more). |
 | `store/` | Four store ABCs with local and cloud implementations: `DocumentStore` (SQLite, MongoDB), `ObjectStore` (filesystem, S3, Cloud Storage), `ImageStore` (local OCI registry, ECR), `SecretStore` (env vars or file, AWS Secrets Manager, Google Cloud Secret Manager). `VersionedEntityStore` implements the shared versioned get/put logic, `QueryBuilder` is the immutable chained query API, `store/base.py` holds the error types. A new backend must pass the conformance kits in `tst/store/`. |
 | `config/` | The `Config` singleton (`get_config`, `configure`, `reset_config`) in `config/runtime.py`, file discovery in `config/loader.py`, and `load_impl`, which resolves `module:Class` pointers. `agent_env.store` re-exports the config names for compatibility. |
-| `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `tensorlake` (the `tensorlake` extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store) and `EnvironmentGatewayProvider`, which renders a docker-compose for the gateway and its MCP servers inside the sandbox; `providers/env_state/` holds env-state providers (`local_postgres` built in). |
+| `providers/` | `providers/sandbox_providers/` holds the sandbox providers `local`, `modal`, `modal_vm`, `e2b`, `sail_vm` (the `sail` extra), `tensorlake` (the `tensorlake` extra); `[sandbox] default` and `agent_default` accept a comma-separated fallback chain, whose deploys skip a provider that can't run them. A sandbox provider declares what it runs (`ON_THIS_MACHINE`, and for VMs on this machine the address their requests to it arrive at, `REACHES_HOST_AT`; `CONTAINER_ACCEPTS`, `PRIVATE_NETWORK`; `SUPPORTED_SANDBOX_MODES`, derived from whether it implements `create_sandbox` and `create_vm`; and `SANDBOX_ACCEPTS`, the images an agent's sandbox runs, which defaults to those a VM loads when it implements `create_vm` and to `CONTAINER_ACCEPTS` otherwise, so only a provider that implements `create_vm` yet gives an agent a container, like the local one, declares it), each sandbox carries the `platform` of the images it runs (linux/amd64 unless its provider sets another when it creates it; None, any, unchecked, for a local sandbox), and code outside `providers/sandbox_providers/` reads those, never which provider it has. `providers/env_providers/` holds the environment providers: `EnvironmentProvider` (an env's containers and state store) and `EnvironmentGatewayProvider`, which renders a docker-compose for the gateway and its MCP servers inside the sandbox; `providers/env_state/` holds env-state providers (`local_postgres` built in). |
 | `a2a_agent/` | The `A2AAgent` entity (`a2a_agent`), its stores and the validator steps. The protocol package provides the agent-side framework. |
 | `runner/` | The `[runner]` seam: `Runner.submit()` returns `(run_id, instance_id)`; `LocalRunner` is built in. |
 | `explorer/` | Optional local web UI: `agent-env up`, needs the `explorer` extra, binds loopback `:8234`. |

src/agent_env/providers/sandbox_providers/local_sandbox.py

@@ -44,6 +44,7 @@
     SANDBOX_MODE_VM,
     Accepts,
     SandboxProvider,
+    on_this_machine_provider,
     refuse_unenforceable_policy,
 )
 from agent_env.store.object_store.local.tls import local_ca
@@ -259,6 +260,10 @@ def _rewrite_app_arg(self, arg: str) -> str:
     def _rewrite_app_script(self, script: str) -> str:
         return _APP_PATH_PATTERN.sub(lambda _: str(self._work_dir), script)
 
+    async def copy_trust_dir_into_container(self, trust_dir: Path, container: str, destination: str) -> None:
+        """Directly: the container is on this machine's Docker, and this sandbox's shell would rewrite the path."""
+        await asyncio.to_thread(_copy_into_container, trust_dir, container, destination)
+
     async def terminate(self) -> None:
         """Tear down whatever this sandbox is running.
 
@@ -504,10 +509,11 @@ def local_grant_trust() -> Path | None:
 
 
 async def start_trusting(sandbox: VmSandbox, container: str, trust_dir: Path) -> None:
-    """Copy ``trust_dir`` into the created ``container`` where ``LOCAL_TRUST_ENV`` points, then start it. A container
-    that cannot be given the files or started is removed, so its name is free for the next attempt."""
+    """Copy ``trust_dir`` into the created ``container`` where ``LOCAL_TRUST_ENV`` points, as the sandbox's host can
+    reach it, then start it. A container that cannot be given the files or started is removed, so its name is free
+    for the next attempt."""
     try:
-        await asyncio.to_thread(_copy_into_container, trust_dir, container, _TRUST_DIR)
+        await sandbox.copy_trust_dir_into_container(trust_dir, container, _TRUST_DIR)
         await sandbox.exec_script(f"docker start {shlex.quote(container)} > /dev/null")
     except Exception:
         await sandbox.exec_script(f"docker rm -f {shlex.quote(container)} >/dev/null 2>&1 || true")
@@ -682,9 +688,10 @@ def get_external_url(cls, url: str) -> str:
 
 
 def host_url_for(url: str, sandbox_type: Optional[str]) -> str:
-    """``url``, which this machine reaches, as a container on ``sandbox_type`` reaches it: only a local
-    sandbox's containers run on this machine, and they reach its loopback by another name."""
-    return LocalSandboxProvider.get_external_url(url) if sandbox_type == LocalSandbox.type else url
+    """``url``, which this machine reaches, as a container on ``sandbox_type`` reaches it: one on this machine (its
+    provider's ``ON_THIS_MACHINE``) reaches this machine's loopback as ``host.docker.internal``; one elsewhere reaches
+    the URL as it is, or not at all."""
+    return LocalSandboxProvider.get_external_url(url) if on_this_machine_provider(sandbox_type) else url
 
 
 def _is_loopback(host: Optional[str]) -> bool:

src/agent_env/providers/sandbox_providers/sandbox.py

@@ -19,6 +19,7 @@
 from abc import ABC, abstractmethod
 from dataclasses import dataclass, replace
 from enum import Enum
+from pathlib import Path
 from typing import IO, TYPE_CHECKING, Any, AsyncIterator, Callable, ClassVar, Iterable, Optional
 
 from agent_env.config import get_config
@@ -477,6 +478,18 @@ async def docker_cp(self, source: str, destination: str, *, remove_source: bool
                 f"docker cp {source} {destination} failed (exit {exit_code}):\nstdout: {stdout[-1500:]}\nstderr: {stderr[-1500:]}"
             )
 
+    async def copy_trust_dir_into_container(self, trust_dir: Path, container: str, destination: str) -> None:
+        """Install the local transfer CA's files, ``trust_dir`` on this machine, at ``destination`` in ``container`` on
+        this VM's Docker: through the VM, where the container is."""
+        vm_dir = self._staging_path("trust", container)
+        await self.exec_script(f"mkdir -p {shlex.quote(vm_dir)}")
+        try:
+            for path in sorted(trust_dir.iterdir()):
+                await self.write_host_file(await asyncio.to_thread(path.read_bytes), f"{vm_dir}/{path.name}")
+            await self.docker_cp(f"{vm_dir}/.", f"{container}:{destination}")
+        finally:
+            await self.exec_script(f"rm -rf {shlex.quote(vm_dir)}")
+
     async def _copy_into_container(self, vm_path: str, destination_path: str) -> None:
         parent = os.path.dirname(destination_path)
         if parent:

src/agent_env/providers/sandbox_providers/sandbox_provider.py

@@ -171,8 +171,16 @@ def from_config(cls, **config: Any) -> Self:
     # sandbox the images that follow from what it implements (SANDBOX_ACCEPTS).
     #
     # Whether its sandboxes run on this machine: an image only this machine holds (in a registry or object store on
-    # it) reaches them, its containers need this machine's Docker, and a gateway's infra envs are built here for it.
+    # it) reaches them, they reach this machine's loopback as ``host.docker.internal`` (resolved by Docker, or by the
+    # provider's ``extra_hosts``), they trust the local transfer CA, and a gateway's infra envs are built here for
+    # them. They are containers on this machine's Docker, or VMs here (``REACHES_HOST_AT``).
     ON_THIS_MACHINE: ClassVar[bool] = False
+    # For a provider on this machine, the address its sandboxes' requests to this machine's names arrive at: None for
+    # containers on this machine's Docker, which arrive at the address ``host.docker.internal`` maps to, where the
+    # local grant server listens by default; for a VM here whose networking forwards this machine's names to an
+    # address of its own, that address: ``127.0.0.1``, which the server serves as well by default, or another the
+    # server is bound to. From such a VM, this machine's addresses by number aren't reached.
+    REACHES_HOST_AT: ClassVar[str | None] = None
     # The sandbox modes it creates, derived for each provider class from what it implements (``__init_subclass__``):
     # SANDBOX_MODE_CONTAINER when it implements create_sandbox, SANDBOX_MODE_VM when it implements create_vm.
     SUPPORTED_SANDBOX_MODES: ClassVar[frozenset[str]] = frozenset()
@@ -438,6 +446,19 @@ def reachable_url(url: str, *, from_sandbox_type: Optional[str], to_sandbox_type
     return provider_cls.get_external_url(url)
 
 
+def on_this_machine_provider(sandbox_type: Optional[str]) -> Optional[type[SandboxProvider]]:
+    """The provider class registered for ``sandbox_type`` when its sandboxes run on this machine (``ON_THIS_MACHINE``),
+    else None: for no type, an unknown one, or one whose provider can't be loaded here, which reaches nothing of
+    this machine's."""
+    if not sandbox_type:
+        return None
+    try:
+        cls = _sandbox_provider_class(sandbox_type)
+    except (ValueError, ConfigError):
+        return None
+    return cls if cls.ON_THIS_MACHINE else None
+
+
 def registered_sandbox_provider_classes() -> list[type[SandboxProvider]]:
     """Every registered provider class, de-duplicated, in registry order.
 

src/agent_env/store/object_store/local/grant_server.py

@@ -51,6 +51,9 @@
 
 DEFAULT_ADVERTISE_HOST = "host.docker.internal"
 _START_TIMEOUT_SECONDS = 10
+_BIND_ATTEMPTS = 5  # a second socket on one port can lose the port to another process in between
+# Where the platform default is served as well: the address a VM on this machine arrives at (``REACHES_HOST_AT``).
+DEFAULT_ALSO_SERVED = "127.0.0.1"
 _OPS_BY_METHOD = {"GET": "get", "HEAD": "get", "PUT": "put", "POST": "post"}
 _UPLOAD_KEY_FIELD = "key"  # the form fields a policy upload names its key and file by (HttpPostPolicyGrant's path/file fields)
 _UPLOAD_FILE_FIELD = "file"
@@ -59,7 +62,7 @@
 _MAX_PART_HEADER_BYTES = 16 * 1024
 _DOWNLOAD_CHUNK_BYTES = 1024 * 1024
 
-_servers: dict[tuple[str, str], GrantServer] = {}
+_servers: dict[tuple[tuple[str, ...], str], GrantServer] = {}
 _servers_lock = threading.Lock()
 
 
@@ -82,7 +85,11 @@ def __init__(self, status: int, reason: str) -> None:
 def grant_server(bind_host: str | None, advertise_host: str | None) -> GrantServer:
     """This process's grant server for a bind and advertised host, created on first use; neither
     given means this platform's default bind address and ``host.docker.internal``."""
-    key = (bind_host or default_bind_host(), advertise_host or DEFAULT_ADVERTISE_HOST)
+    resolved = bind_host or default_bind_host()
+    # The platform default is served at DEFAULT_ALSO_SERVED too, where a VM on this machine arrives; an explicit
+    # bind host is served as given.
+    also = () if bind_host is not None or resolved == DEFAULT_ALSO_SERVED else (DEFAULT_ALSO_SERVED,)
+    key = ((resolved, *also), advertise_host or DEFAULT_ADVERTISE_HOST)
     with _servers_lock:
         server = _servers.get(key)
         if server is None:
@@ -130,12 +137,38 @@ def default_bind_host() -> str:
         return "127.0.0.1"
 
 
+def _close_all(sockets: list[socket.socket]) -> None:
+    for sock in sockets:
+        sock.close()
+
+
+def _bind_same_port(hosts: tuple[str, ...]) -> list[socket.socket]:
+    """Sockets bound to ``hosts`` on one free port: the first picks it, the rest take it. Another process can take
+    the port on a later host in between, which is retried; a host that can't be bound at all raises as it is."""
+    attempts = _BIND_ATTEMPTS if len(hosts) > 1 else 1
+    for attempt in range(attempts):
+        sockets: list[socket.socket] = []
+        try:
+            for host in hosts:
+                family = socket.AF_INET6 if ipaddress.ip_address(host).version == 6 else socket.AF_INET
+                sockets.append(socket.socket(family, socket.SOCK_STREAM))
+                sockets[-1].bind((host, sockets[0].getsockname()[1] if len(sockets) > 1 else 0))
+            return sockets
+        except OSError:
+            lost_the_port = len(sockets) > 1
+            _close_all(sockets)
+            if not lost_the_port or attempt == attempts - 1:
+                raise
+    raise AssertionError("unreachable")
+
+
 class GrantServer:
     """Serves the grants of the local stores registered with it, over HTTPS on one port. Started by
     the first grant it issues; it runs in a daemon thread for the rest of the process."""
 
-    def __init__(self, bind_host: str, advertise_host: str) -> None:
-        self.bind_host = bind_host
+    def __init__(self, bind_hosts: tuple[str, ...], advertise_host: str) -> None:
+        self.bind_hosts = bind_hosts
+        self.bind_host = bind_hosts[0]  # the one configured or the platform default; any other is served as well
         self.advertise_host = advertise_host
         self._signer = GrantSigner()
         self._stores: dict[str, LocalFilesystemObjectStore] = {}
@@ -191,17 +224,16 @@ def _start(self) -> int:
         with self._lock:
             if self._port is not None:
                 return self._port
-            address = ipaddress.ip_address(self.bind_host)
-            sock = socket.socket(socket.AF_INET6 if address.version == 6 else socket.AF_INET, socket.SOCK_STREAM)
+            sockets: list[socket.socket] = []
             try:
-                sock.bind((self.bind_host, 0))
+                sockets = _bind_same_port(self.bind_hosts)
                 hosts = ["localhost", "host.docker.internal", "127.0.0.1", self.advertise_host]
                 with contextlib.suppress(ValueError):  # a name the CA cannot vouch for would void the certificate
                     check_local_host(self.bind_host)
                     hosts.append(self.bind_host)
                 context = server_context(local_ca(), hosts)
             except (OSError, ValueError) as e:
-                sock.close()
+                _close_all(sockets)
                 raise GrantUnavailableError(f"the local grant server cannot listen on {self.bind_host}: {e}") from e
             config = uvicorn.Config(
                 self._app, lifespan="off", log_config=None, http=_UnloggedH11Protocol,
@@ -212,7 +244,7 @@ def _start(self) -> int:
 
             def serve() -> None:
                 try:
-                    asyncio.run(server.serve(sockets=[sock]))
+                    asyncio.run(server.serve(sockets=sockets))
                 except BaseException as e:  # uvicorn ends a failed startup with SystemExit
                     failure.append(e)
 
@@ -222,15 +254,15 @@ def serve() -> None:
             while not server.started:
                 if failure or not thread.is_alive() or time.monotonic() > deadline:
                     server.should_exit = True
-                    sock.close()
+                    _close_all(sockets)
                     raise GrantUnavailableError(
                         f"the local grant server did not start on {self.bind_host}"
                         + (f": {failure[0]!r}" if failure else "")
                     )
                 time.sleep(0.01)
             self._uvicorn, self._thread = server, thread
-            self._port = sock.getsockname()[1]
-            logger.info("Local grant server listening on %s:%d", self.bind_host, self._port)
+            self._port = sockets[0].getsockname()[1]
+            logger.info("Local grant server listening on %s:%d", " and ".join(self.bind_hosts), self._port)
             return self._port
 
     async def _handle(self, request: Request) -> Response: