Give Core its own health check

deploy/compose/compose.yaml

@@ -52,7 +52,7 @@ services:
     depends_on:
       database: {condition: service_healthy}
     environment:
-      OAC_PUBLIC_URL: &public-url ${OAC_PUBLIC_URL:-http://localhost:8080}
+      OAC_PUBLIC_URL: &public-url ${OAC_PUBLIC_URL:-http://localhost:${OAC_WEB_PORT:-8080}}
       OAC_INSTALLATION_ID_FILE: /run/oac/installation.id
       OAC_DATABASE_URL: postgres://agents_api@database:5432/agents_api?sslmode=disable
       OAC_DATABASE_PASSWORD_FILE: /run/database/password
@@ -82,6 +82,11 @@ services:
         source: data
         target: /state
         volume: {nocopy: true, subpath: state}
+    healthcheck:
+      test: [CMD, /usr/local/bin/oac-core, healthcheck]
+      interval: 5s
+      timeout: 10s
+      retries: 30
 
   web:
     ports:

deploy/compose/test_compose.py

@@ -29,7 +29,7 @@ def rendered_compose(directory):
 
 class ComposeTests(unittest.TestCase):
     @classmethod
-    def render(cls, public_url=None):
+    def render(cls, public_url=None, web_port=None):
         env = dict(os.environ)
         env.pop('OAC_PUBLIC_URL', None)
         env.pop('OAC_HOST', None)
@@ -39,6 +39,8 @@ def render(cls, public_url=None):
         env['OAC_DATA_DIR'] = '/tmp/oac-compose-fixture'
         if public_url is not None:
             env['OAC_PUBLIC_URL'] = public_url
+        if web_port is not None:
+            env['OAC_WEB_PORT'] = web_port
         return json.loads(subprocess.check_output(
             ['docker', 'compose', '--env-file', os.devnull, '-f', str(cls.compose_file),
              'config', '--format', 'json'], env=env))
@@ -70,17 +72,21 @@ def test_compose_uses_private_services_and_ordered_initialization(self):
         self.assertIsNone(services['core']['command'])
         self.assertNotIn('OAC_WEB_INSTALLATION_SOCKET', services['web']['environment'])
         self.assertEqual(services['init']['command'], ['/usr/local/bin/oac', 'init'])
+        # Each service reports its own health, so up --wait waits for Core directly.
+        self.assertEqual(services['core']['healthcheck']['test'], ['CMD', '/usr/local/bin/oac-core', 'healthcheck'])
         self.assertEqual(services['web']['healthcheck']['test'], ['CMD', '/usr/local/bin/oac-web', 'healthcheck'])
         self.assertNotIn('python3', json.dumps(self.compose))
         self.assertEqual(services['init']['environment']['OAC_REVISION'], 'd' * 40)
         for name in ('OAC_EXECUTION_CONCURRENCY', 'OAC_DEFAULT_HARNESS', 'OAC_HARNESSES', 'OAC_WRITE_AUDIT_RETENTION', 'OAC_LOG_LEVEL'):
             self.assertEqual(services['core']['environment'][name], '', name)
 
     def test_public_url_can_be_configured_after_initial_startup(self):
-        for value in (None, '', 'https://oac.example.test', 'http://localhost:9080'):
-            with self.subTest(public_url=value):
-                configured = self.render(value)
-                expected = value or 'http://localhost:8080'
+        for value, port, expected in ((None, None, 'http://localhost:8080'), ('', None, 'http://localhost:8080'),
+                                      (None, '9080', 'http://localhost:9080'), ('', '9080', 'http://localhost:9080'),
+                                      ('https://oac.example.test', '9080', 'https://oac.example.test'),
+                                      ('http://localhost:9080', None, 'http://localhost:9080')):
+            with self.subTest(public_url=value, web_port=port):
+                configured = self.render(value, port)
                 for name, setting in (('core', 'OAC_PUBLIC_URL'), ('web', 'OAC_PUBLIC_URL')):
                     self.assertEqual(configured['services'][name]['environment'][setting], expected)
                 self.assertEqual(

docs/configuration.md

@@ -44,7 +44,7 @@ Model providers are not process settings; see [Default models](#default-models).
 
 | Variable | Default | Meaning |
 | --- | --- | --- |
-| `OAC_PUBLIC_URL` | Required; `compose.yaml` sets `http://localhost:8080` | Origin applications, nodes, sandboxes and self-hosted executors use. See [changing the public URL](#changing-the-public-url) |
+| `OAC_PUBLIC_URL` | Required; `compose.yaml` sets `http://localhost:` followed by `OAC_WEB_PORT` | Origin applications, nodes, sandboxes and self-hosted executors use. See [changing the public URL](#changing-the-public-url) |
 | `OAC_HOST` | `127.0.0.1` | Web bind address published by `compose.yaml`. The installer sets `0.0.0.0` |
 | `OAC_WEB_PORT` | `8080` | Host port of Web |
 | `OAC_LOG_LEVEL` | `info` | `debug`, `info`, `warn` or `error` |
@@ -260,7 +260,7 @@ Core reads its process environment. Compose interpolates `.env` into it and moun
 | Variable | Set from |
 | --- | --- |
 | `OAC_PUBLIC_URL` | Required. The [public URL](#settings). Core validates it once and derives the Agents API base, the daemon WebSocket URL, the self-hosted `remote_url`, the installer downloads, the hosted sandbox address and the deployment's read-only `core_url` from it, never from request headers |
-| `OAC_ADDR` | The image sets `:8091`. Independently started Core defaults to `127.0.0.1:8091` when unset or empty |
+| `OAC_ADDR` | The image sets `:8091`. Independently started Core defaults to `127.0.0.1:8091` when unset or empty. `oac-core healthcheck` probes its `/healthz` on `127.0.0.1` when its host is empty or unspecified |
 | `OAC_DATABASE_URL` | Required. PostgreSQL without a password |
 | `OAC_DATABASE_PASSWORD_FILE` | `/run/database/password`. The URL must then carry no password |
 | `OAC_CREDENTIAL_KEY_FILE` | Required. `/run/oac/credential.key`: a base64-encoded random 32-byte key. Core seals stored credentials with it |
@@ -279,9 +279,9 @@ Compose sets these for Web. Set them yourself only when you run the console with
 
 | Variable | Default | Meaning |
 | --- | --- | --- |
-| `OAC_WEB_ADDR` | `:8080` | Listener address. The healthcheck probes it on `127.0.0.1` when its host is empty or unspecified |
+| `OAC_WEB_ADDR` | `:8080` | Listener address. `oac-web healthcheck` probes its `/healthz` on `127.0.0.1` when its host is empty or unspecified |
 | `OAC_PUBLIC_URL` | Required | The [public URL](#settings): the exact browser-facing origin, HTTP or HTTPS, without a path. Host and origin checks use it; HTTPS makes the session cookie `Secure` |
-| `OAC_WEB_UPSTREAM` | `http://core:8091` | Core's origin, HTTP or HTTPS, without credentials, query or path. The healthcheck probes its `/healthz` |
+| `OAC_WEB_UPSTREAM` | `http://core:8091` | Core's origin, HTTP or HTTPS, without credentials, query or path |
 | `OAC_WEB_CORE_KEY_FILE` | Required | Absolute path of a regular file with no group or other permissions, holding the Core key: at least 32 characters, no whitespace, at most 4 KiB |
 | `OAC_WEB_DIST` | `/www` | Absolute directory of the built console; must contain `index.html` |
 | `OAC_WEB_NODE_PAYLOAD_DIR` | unset | Absolute path of the matched distribution's node payload (the installer's `node-payload/`). Unset, `/node-install/*` is not served and Add node is unavailable |

docs/getting-started/operations.md

@@ -30,7 +30,7 @@ Use these observations for different questions:
 
 | Observation | What it establishes |
 | --- | --- |
-| `docker compose ps` | The database accepts its readiness check |
+| `docker compose ps` | The database accepts its readiness check, and Core and Web each answer their own `/healthz` |
 | Core `/healthz` | Core's process is alive |
 | An authenticated API read | The caller's key works for that resource |
 | Environment connection | The Runtime transport is connected |

docs/zh/configuration.md

@@ -1,7 +1,7 @@
 ---
 title: "配置参考"
 source: docs/configuration.md
-source_hash: d8e517c85fbc4da3e6cc8906d49c8ac9d92c7b571edf964916d4c2d7efbc5c6d
+source_hash: 51d98943e71402c79cf134e757192607c3628944369e869a69a9582e34fa8666
 ---
 
 Core 安装的每项设置都恰好只有一个归属位置，分属以下三类：
@@ -48,7 +48,7 @@ Core 安装的每项设置都恰好只有一个归属位置，分属以下三类
 
 | Variable | Default | Meaning |
 | --- | --- | --- |
-| `OAC_PUBLIC_URL` | 必填；`compose.yaml` 设为 `http://localhost:8080` | 应用、节点、沙箱和自托管执行器使用的源地址。参阅[更改公共 URL](#changing-the-public-url) |
+| `OAC_PUBLIC_URL` | 必填；`compose.yaml` 设为 `http://localhost:` 加 `OAC_WEB_PORT` | 应用、节点、沙箱和自托管执行器使用的源地址。参阅[更改公共 URL](#changing-the-public-url) |
 | `OAC_HOST` | `127.0.0.1` | `compose.yaml` 发布的 Web 绑定地址。安装器设置为 `0.0.0.0` |
 | `OAC_WEB_PORT` | `8080` | Host port of Web |
 | `OAC_LOG_LEVEL` | `info` | `debug`, `info`, `warn` or `error` |
@@ -264,7 +264,7 @@ Core 读取进程环境。Compose 将 `.env` 插值到环境中，并把机密
 | 变量 | 设置来源 |
 | --- | --- |
 | `OAC_PUBLIC_URL` | 必填。[公共 URL](#settings)。Core 只校验一次，并从中派生 Agents API 基地址、守护进程 WebSocket URL、自托管 `remote_url`、安装程序下载地址、托管沙箱地址和部署的只读 `core_url`，绝不从请求标头派生 |
-| `OAC_ADDR` | 安装程序在容器中设置为 `:8091`。独立启动的 Core 在未设置或为空时，默认使用 `127.0.0.1:8091` |
+| `OAC_ADDR` | 安装程序在容器中设置为 `:8091`。独立启动的 Core 在未设置或为空时，默认使用 `127.0.0.1:8091`。主机部分为空或未指定时，`oac-core healthcheck` 在 `127.0.0.1` 上探测其 `/healthz` |
 | `OAC_DATABASE_URL` | 必填。不含密码的 PostgreSQL URL |
 | `OAC_DATABASE_PASSWORD_FILE` | `/run/database/password`。此时 URL 不得包含密码 |
 | `OAC_CREDENTIAL_KEY_FILE` | 必填。`/run/oac/credential.key`：Base64 编码的 32 字节随机密钥。Core 用它加密存储的凭据 |
@@ -283,9 +283,9 @@ Compose 为 Web 设置这些变量。仅在不使用 Compose 运行控制台时
 
 | 变量 | 默认值 | 含义 |
 | --- | --- | --- |
-| `OAC_WEB_ADDR` | `:8080` | 监听地址。主机部分为空或未指定时，健康检查在 `127.0.0.1` 上探测它 |
+| `OAC_WEB_ADDR` | `:8080` | 监听地址。主机部分为空或未指定时，`oac-web healthcheck` 在 `127.0.0.1` 上探测其 `/healthz` |
 | `OAC_PUBLIC_URL` | 必填 | [公共 URL](#settings)：面向浏览器的准确源地址，可以使用 HTTP 或 HTTPS，且不得包含路径。Host 和源地址检查使用此值；HTTPS 会使 Session Cookie 具备 `Secure` 属性 |
-| `OAC_WEB_UPSTREAM` | `http://core:8091` | Core 的源地址，可以使用 HTTP 或 HTTPS，且不得包含凭据、查询参数或路径。健康检查探测其 `/healthz` |
+| `OAC_WEB_UPSTREAM` | `http://core:8091` | Core 的源地址，可以使用 HTTP 或 HTTPS，且不得包含凭据、查询参数或路径 |
 | `OAC_WEB_CORE_KEY_FILE` | 必填 | 常规文件的绝对路径，该文件没有组或其他用户权限，并保存 Core 密钥：至少 32 个字符、不含空白字符、最大 4 KiB |
 | `OAC_WEB_DIST` | `/www` | 已构建控制台的绝对目录；必须包含 `index.html` |
 | `OAC_WEB_NODE_PAYLOAD_DIR` | 未设置 | 所匹配发行版的节点载荷（即安装程序的 `node-payload/`）的绝对路径。未设置时，不提供 `/node-install/*`，且 **添加节点** 不可用 |

docs/zh/getting-started/operations.md

@@ -1,7 +1,7 @@
 ---
 title: "运维"
 source: docs/getting-started/operations.md
-source_hash: 1d79122c41ef06b70dfd592c55d6b474dc53485f4500518871d8e4a7aba43b91
+source_hash: 8a7e237e49368e9704af1a18a8b22b5ff84f00a7eb42bfc182b960ea59f41121
 ---
 
 安装运维人员负责 Core 主机、存储和可用性。节点主机运行各自的服务；参阅[节点](nodes.md)。设置见[配置参考](../configuration.md)。
@@ -32,7 +32,7 @@ docker compose -f ~/.oac/core/compose.yaml ps
 
 | 观察 | 能证明什么 |
 | --- | --- |
-| `docker compose ps` | 数据库接受就绪检查 |
+| `docker compose ps` | 数据库接受就绪检查，Core 和 Web 各自响应自己的 `/healthz` |
 | Core `/healthz` | Core 进程存活 |
 | 经过认证的 API 读取 | 调用者的密钥适用于该资源 |
 | Environment 连接 | Runtime 传输已连接 |

scripts/compose-smoke.py

@@ -16,6 +16,7 @@
 import re
 import shutil
 import signal
+import socket
 import subprocess
 import tempfile
 import urllib.error
@@ -115,8 +116,12 @@ def main():
     images = build_images(directory, project.removeprefix('oac-smoke-'), revision, source_time)
 
     override.write_text(json.dumps({'services': {'init': {'network_mode': 'none'}}}))
+    # A free loopback port, kept for the whole run: the default origin follows it.
+    with socket.socket() as probe:
+        probe.bind(('127.0.0.1', 0))
+        port = str(probe.getsockname()[1])
     env = {**os.environ, 'COMPOSE_PROGRESS': 'plain',
-           'OAC_HOST': '127.0.0.1', 'OAC_WEB_PORT': '0',
+           'OAC_HOST': '127.0.0.1', 'OAC_WEB_PORT': port,
            **{'OAC_IMAGE_' + name.upper(): image for name, image in images.items()}}
     env.pop('OAC_PUBLIC_URL', None)
 
@@ -142,7 +147,7 @@ def client():
             urllib.request.HTTPCookieProcessor(http.cookiejar.CookieJar()))
 
     browser = client()
-    origin = 'http://localhost:8080'
+    origin = 'http://localhost:' + port
     address = ''
 
     def request(path, body=None, headers=None, status=200):
@@ -208,15 +213,13 @@ def terminate(_signum, _frame):
         private_logs(key, project_key)
 
         print('Configuring a reachable URL and recreating containers with the same data directory', flush=True)
-        # Retain the assigned port across recreation, without claiming a fixed host port.
-        env['OAC_WEB_PORT'] = address.rsplit(':', 1)[1]
         env['OAC_PUBLIC_URL'] = address
         compose('down')
         compose('up', '-d', '--wait', '--wait-timeout', '120', timeout=180)
         origin = address
         browser = client()
         assert compose('exec', '-T', 'web', '/usr/local/bin/oac-web', 'core-key').decode().strip() == key, 'Sign-in key changed'
-        request('/', headers={'Host': 'localhost:8080'}, status=403)
+        request('/', headers={'Host': 'localhost:' + port}, status=403)
         request('/console/auth/login', {'core_key': key})
         updated = get('/core/v1/installation')
         assert updated['installation_id'] == facts['installation_id'], 'Installation identity changed'
@@ -244,7 +247,6 @@ def terminate(_signum, _frame):
         urllib.request.urlretrieve(RELEASES + pins['tag'] + '/compose.yaml', published)
         if hashlib.sha256(published.read_bytes()).hexdigest() != pins['compose_sha256']:
             raise ValueError('Pinned release compose.yaml checksum mismatch')
-        env['OAC_WEB_PORT'] = '0'
         env.pop('OAC_PUBLIC_URL')
         released = runner([published], {name: value for name, value in env.items() if not name.startswith('OAC_IMAGE_')})
         released('up', '-d', '--wait', '--wait-timeout', '600', timeout=900)
@@ -265,11 +267,14 @@ def terminate(_signum, _frame):
         compose('run', '--rm', '--no-deps', 'init')
         compose('up', '-d', '--wait', '--wait-timeout', '180', '--pull', 'never', '--remove-orphans', timeout=240)
         address = 'http://' + compose('port', 'web', '8080').decode().strip()
+        # The release's default origin ignored the Web port; this checkout's follows it.
+        origin = 'http://localhost:' + port
         browser = client()
         assert compose('exec', '-T', 'web', '/usr/local/bin/oac-web', 'core-key').decode().strip() == key, 'Upgrade changed the Core key'
         request('/console/auth/login', {'core_key': key})
         upgraded = get('/core/v1/installation')
         assert upgraded['installation_id'] == facts['installation_id'], 'Upgrade changed the installation identity'
+        assert upgraded['public_url'] == origin, 'The upgraded default origin does not follow the Web port'
         assert upgraded['source_commit'] == revision, 'Core does not run this checkout'
         assert json.loads(request('/node-install/manifest.json'))['source_commit'] == revision, 'Web serves another release'
         assert any(p['id'] == project_data['id'] for p in get('/core/v1/projects')['data']), 'Upgrade lost the Project'

services/core/cmd/server/main.go

@@ -25,6 +25,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"net"
 	"net/http"
 	"os"
 	"os/signal"
@@ -81,6 +82,16 @@ func main() {
 		}
 		return
 	}
+	if len(os.Args) == 2 && os.Args[1] == "healthcheck" {
+		if err == nil {
+			err = healthcheck(config.Addr)
+		}
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err.Error())
+			os.Exit(1)
+		}
+		return
+	}
 	if err == nil {
 		err = run(config)
 	}
@@ -90,6 +101,29 @@ func main() {
 	}
 }
 
+// healthcheck reports whether Core's listener answers /healthz. Compose runs it
+// inside the core container, so an empty or unspecified listen host is probed
+// on 127.0.0.1.
+func healthcheck(addr string) error {
+	host, port, err := net.SplitHostPort(addr)
+	if err != nil {
+		return errors.New("OAC_ADDR must be a host:port listen address")
+	}
+	if ip := net.ParseIP(host); host == "" || (ip != nil && ip.IsUnspecified()) {
+		host = "127.0.0.1"
+	}
+	client := &http.Client{Timeout: 3 * time.Second, Transport: &http.Transport{Proxy: nil}}
+	response, err := client.Get("http://" + net.JoinHostPort(host, port) + "/healthz")
+	if err != nil {
+		return err
+	}
+	response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return fmt.Errorf("Core returned HTTP %d", response.StatusCode)
+	}
+	return nil
+}
+
 func run(config processconfig.Config) error {
 	log.Init(config.Log)
 	log.Bg().Info("Core process configuration loaded from the process environment")

services/core/cmd/server/main_test.go

@@ -0,0 +1,25 @@
+package main
+
+import (
+	"net"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestHealthcheckProbesCoreListener(t *testing.T) {
+	core := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.URL.Path != "/healthz" {
+			http.NotFound(w, r)
+		}
+	}))
+	_, port, _ := net.SplitHostPort(core.Listener.Addr().String())
+	// The image listens on every address; the probe uses loopback.
+	if err := healthcheck(":" + port); err != nil {
+		t.Fatalf("Core was reported unhealthy while it answers: %v", err)
+	}
+	core.Close()
+	if err := healthcheck(":" + port); err == nil {
+		t.Fatal("Core was reported healthy after its listener stopped")
+	}
+}

services/core/internal/processconfig/config.go

@@ -1,8 +1,8 @@
 // Package processconfig is the process settings Core loads from its
 // environment. Load reads every variable and every file it names exactly once,
-// applies each default and validates the result; startup and `oac-core
-// check-config` both call it. Errors name the variable and never include its
-// value or the content of a file.
+// applies each default and validates the result; startup, `oac-core
+// check-config` and `oac-core healthcheck` call it. Errors name the variable
+// and never include its value or the content of a file.
 package processconfig
 
 import (

services/web/main.go

@@ -7,7 +7,6 @@ import (
 	"fmt"
 	"net"
 	"net/http"
-	"net/url"
 	"os"
 	"os/signal"
 	"syscall"
@@ -48,8 +47,8 @@ func printCoreKey() error {
 	return nil
 }
 
-// healthcheck reports the installation healthy once Core and Web's own listener
-// answer. Compose runs it inside the web container.
+// healthcheck reports whether Web's own listener answers /healthz. Compose
+// runs it inside the web container; Core has its own healthcheck.
 func healthcheck() error {
 	c, err := loadConfig()
 	if err != nil {
@@ -62,22 +61,14 @@ func healthcheck() error {
 	if ip := net.ParseIP(host); host == "" || (ip != nil && ip.IsUnspecified()) {
 		host = "127.0.0.1"
 	}
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
-	defer cancel()
 	client := &http.Client{Timeout: 3 * time.Second, Transport: &http.Transport{Proxy: nil}}
-	for _, server := range []*url.URL{c.upstream, {Scheme: "http", Host: net.JoinHostPort(host, port)}} {
-		request, err := http.NewRequestWithContext(ctx, http.MethodGet, server.JoinPath("healthz").String(), nil)
-		if err != nil {
-			return err
-		}
-		response, err := client.Do(request)
-		if err != nil {
-			return err
-		}
-		response.Body.Close()
-		if response.StatusCode != http.StatusOK {
-			return fmt.Errorf("%s returned HTTP %d", server.Host, response.StatusCode)
-		}
+	response, err := client.Get("http://" + net.JoinHostPort(host, port) + "/healthz")
+	if err != nil {
+		return err
+	}
+	response.Body.Close()
+	if response.StatusCode != http.StatusOK {
+		return fmt.Errorf("Web returned HTTP %d", response.StatusCode)
 	}
 	return nil
 }