diff --git a/deploy/compose/compose.yaml b/deploy/compose/compose.yaml index 7aca1c7c1..9e46b7930 100644 --- a/deploy/compose/compose.yaml +++ b/deploy/compose/compose.yaml @@ -1,4 +1,4 @@ -# Release template. The publisher pins the initialization image to this release. +# Release template. The publisher pins the initialization image to this release and PostgreSQL to its mirrored upstream digest. # Release images are pinned; OAC_IMAGE_* selects another reference. # Docker owns data permissions on every host platform. x-ingress-image: &ingress-image ${OAC_IMAGE_INGRESS:-__OAC_INIT_IMAGE__} @@ -17,7 +17,7 @@ services: volume: {nocopy: true} database: - image: postgres:16-alpine + image: __OAC_DATABASE_IMAGE__ restart: unless-stopped depends_on: init: {condition: service_completed_successfully} diff --git a/deploy/compose/database-image.txt b/deploy/compose/database-image.txt new file mode 100644 index 000000000..5eb278640 --- /dev/null +++ b/deploy/compose/database-image.txt @@ -0,0 +1 @@ +postgres:16-alpine@sha256:721873c34ceb9f8d8fc265984940dc982404c105f19ad51be9fdc5970a6080ea diff --git a/deploy/compose/test_compose.py b/deploy/compose/test_compose.py index 8c55344d4..a973b9552 100644 --- a/deploy/compose/test_compose.py +++ b/deploy/compose/test_compose.py @@ -21,6 +21,7 @@ def rendered_compose(directory): 'INIT_IMAGE': 'ghcr.io/minimax-ai/openagentcore/ingress@sha256:' + 'e' * 64, 'CORE_IMAGE': 'ghcr.io/minimax-ai/openagentcore/core@sha256:' + 'e' * 64, 'WEB_IMAGE': 'ghcr.io/minimax-ai/openagentcore/web@sha256:' + 'e' * 64, + 'DATABASE_IMAGE': 'ghcr.io/minimax-ai/openagentcore/postgres@sha256:' + 'f' * 64, }) path = Path(directory) / 'compose.yaml' path.write_text(text) @@ -62,7 +63,7 @@ def test_compose_uses_private_services_and_ordered_initialization(self): self.assertNotIn('build', service) if service is not services['web']: self.assertNotIn('ports', service) - self.assertTrue(service['image'] == 'postgres:16-alpine' or service['image'].endswith('@sha256:' + 'e' * 64)) + self.assertTrue(service['image'].startswith('ghcr.io/')) for volume in service.get('volumes', []): self.assertNotIn('docker.sock', json.dumps(volume)) self.assertEqual(volume['type'], 'volume') @@ -80,6 +81,10 @@ def test_compose_uses_private_services_and_ordered_initialization(self): for name in ('OAC_EXECUTION_CONCURRENCY', 'OAC_DEFAULT_HARNESS', 'OAC_HARNESSES', 'OAC_WRITE_AUDIT_RETENTION', 'OAC_LOG_LEVEL'): self.assertEqual(services['core']['environment'][name], '', name) + def test_database_stays_on_postgresql_16_pinned_by_digest(self): + # An upgrade keeps the data volume only within one PostgreSQL major version. + self.assertRegex(render_compose.DATABASE_PIN, r'^postgres:16-alpine@sha256:[0-9a-f]{64}$') + def test_public_url_can_be_configured_after_initial_startup(self): for value, port, expected in ((None, None, 'http://localhost:8080'), ('', None, 'http://localhost:8080'), (None, '9080', 'http://localhost:9080'), ('', '9080', 'http://localhost:9080'), diff --git a/deploy/install-source.py b/deploy/install-source.py index 65aecac33..7eeabb2d0 100644 --- a/deploy/install-source.py +++ b/deploy/install-source.py @@ -80,7 +80,7 @@ def install(bundle, destination, host, port, public_url, upgrade): (stage / '.env').write_text(f"COMPOSE_PROJECT_NAME=oac-{uuid.uuid4().hex[:10]}\nOAC_HOST='{host}'\nOAC_WEB_PORT='{port}'\nOAC_PUBLIC_URL='{public_url}'\n") os.chmod(stage / '.env', 0o600) template = (bundle / 'compose.yaml.template').read_text().replace('__OAC_REVISION__', revision).replace('__OAC_INIT_IMAGE__', images['ingress']) - for name in ('core', 'web'): + for name in ('core', 'web', 'database'): template = template.replace('__OAC_' + name.upper() + '_IMAGE__', images[name]) (stage / 'compose.yaml').write_text(template) # The installer owns only the service images and Web's payload bind; other entries are the operator's. diff --git a/docs/getting-started/install-options.md b/docs/getting-started/install-options.md index 994c9f5ef..9fa35a4d3 100644 --- a/docs/getting-started/install-options.md +++ b/docs/getting-started/install-options.md @@ -14,7 +14,7 @@ On Windows, `irm … | iex` cannot pass flags and the default execution policy b ## Docker Compose and hosting platforms -Use the `compose.yaml` from a release on any [supported Core host](./install.md#prerequisites). The release pins its initialization, Core and Web images and source revision in the [Compose template](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/compose.yaml). PostgreSQL uses `postgres:16-alpine`. It starts PostgreSQL, Core and Web. Web forwards `/v1` and `/api/v1` to Core. Data lives in a named Docker volume. The one-time initialization service generates random secrets there and prepares the node installer; Core applies database migrations when it starts. [Compose configuration](../configuration.md#compose-installations) owns the settings and the data directory. +Use the `compose.yaml` from a release on any [supported Core host](./install.md#prerequisites). The release pins its initialization, Core, Web and PostgreSQL images and source revision. PostgreSQL 16 is the official image, pinned by digest and mirrored to GHCR with the other images. It starts PostgreSQL, Core and Web. Web forwards `/v1` and `/api/v1` to Core. Data lives in a named Docker volume. The one-time initialization service generates random secrets there and prepares the node installer; Core applies database migrations when it starts. [Compose configuration](../configuration.md#compose-installations) owns the settings and the data directory. For a local trial, download `compose.yaml` from a release into an empty directory, then run: @@ -132,6 +132,6 @@ The address changes whenever `cloudflared` restarts; nodes bound to the old addr ## Offline hosts -The installer downloads release files from GitHub Releases with this shell's network settings, including `HTTPS_PROXY`. Docker pulls the images through its daemon, which does not read the shell's proxy variables: when the registries are reachable only through a proxy or mirror, configure [the daemon's proxy](https://docs.docker.com/engine/daemon/proxy/) or a [registry mirror](https://docs.docker.com/docker-hub/image-library/mirror/), then rerun the installer. +The installer downloads release files from GitHub Releases with this shell's network settings, including `HTTPS_PROXY`. Docker pulls the images through its daemon, which does not read the shell's proxy variables. Every image, PostgreSQL included, comes from GHCR (`ghcr.io`); Docker Hub is not needed. When GHCR is reachable only through a proxy or mirror, configure [the daemon's proxy](https://docs.docker.com/engine/daemon/proxy/) or a [registry mirror](https://docs.docker.com/docker-hub/image-library/mirror/), then rerun the installer. A Linux amd64 host without that access installs from the release asset `oac--linux-amd64-offline.tar.gz`, which contains every image and node artifact. Copy it to the host, extract it and run its `install.py` with Python 3.9 or newer; [source installation](../development.md#install-a-checkout-without-publishing-a-release) describes its options. diff --git a/docs/maintainers.md b/docs/maintainers.md index ff7766d54..4f1ac07f1 100644 --- a/docs/maintainers.md +++ b/docs/maintainers.md @@ -35,7 +35,7 @@ make build-core-distribution | `CORE_DISTRIBUTION_BUILD_DIR` | Output directory under `~/.oac`. Default: `~/.oac/build/core-distribution` | | `CORE_DISTRIBUTION_BUILD_NETWORK` | Docker build network: `default`, `host` or `none` | | `CORE_DISTRIBUTION_MICROSANDBOX_ARCHIVE` | Cached microsandbox release archive. Default: `~/.oac/cache/microsandbox-v0.7.8-linux-x86_64.tar.gz`, downloaded when missing | -| `CORE_DISTRIBUTION_DATABASE_IMAGE` | PostgreSQL 16 image; the default is pinned by its linux/amd64 manifest digest | +| `CORE_DISTRIBUTION_DATABASE_IMAGE` | PostgreSQL 16 image; the default is the linux/amd64 manifest of the index in `deploy/compose/database-image.txt` | The build reuses the Core, Web, Runtime, SDK and helper builders. The manifest records the commit, its commit time (`source_time`, which orders releases for upgrades) and source tree, image config and OCI manifest digests, the Runtime OCI manifest digest, the microsandbox runtime and firmware hashes, and the size and SHA-256 of every Runtime and node artifact; native installers carry only their SHA-256 in the [catalog](#native-installers). Output is the control archive and its `.sha256`, the optional offline archive, and the versioned Runtime, node and native installer assets. Nothing is published. Rebuilding into a directory that already holds this commit's distribution is refused. @@ -140,7 +140,7 @@ Distribution and Runtime archives use `pigz` level 6 with at most four compressi ### Container registry -Version releases and manual `build-` drafts publish `ghcr.io/minimax-ai/openagentcore/:`, where `` is `core`, `web`, `runtime` or `ingress`. Core, Web and ingress indexes contain Linux amd64 and arm64 images; Runtime contains Linux amd64. Platform images use `-` tags and are loaded from the release archives. Existing version tags must match the release images and platform set. The publisher verifies every version index before updating `latest` for a stable release; prereleases and drafts leave `latest` unchanged. PostgreSQL uses its upstream image. SemVer build metadata uses `_` in place of `+` in container tags; version strings are limited to 128 characters. After verifying the images, the publisher uploads the release's `compose.yaml` and checksum list. Compose pins Core, Web and ingress by their release index digests, and initialization checks its build revision against the Compose revision. +Version releases and manual `build-` drafts publish `ghcr.io/minimax-ai/openagentcore/:`, where `` is `core`, `web`, `runtime`, `ingress` or `postgres`. Core, Web and ingress indexes contain Linux amd64 and arm64 images; Runtime contains Linux amd64. Platform images use `-` tags and are loaded from the release archives. Existing version tags must match the release images and platform set. The publisher verifies every version index before updating `latest` for a stable release; prereleases and drafts leave `latest` unchanged. `postgres` is the official PostgreSQL index pinned in `deploy/compose/database-image.txt`, copied with all its platforms and its digest; the publisher refuses a registry tag that names another digest. To move PostgreSQL, change that file to a new digest of the same major version, which is the one place the template, the offline bundle and the mirror read. SemVer build metadata uses `_` in place of `+` in container tags; version strings are limited to 128 characters. After verifying the images, the publisher uploads the release's `compose.yaml` and checksum list. Compose pins Core, Web, ingress and PostgreSQL by their release index digests, and initialization checks its build revision against the Compose revision. The combined build/publication job uses `GITHUB_TOKEN` with `packages: write`. On the first publication, GitHub creates each container package as private: a package administrator must change all four packages to **Public** in their package settings before users can pull anonymously. See [GitHub container visibility](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry). Verify an unauthenticated pull after changing visibility. Repository visibility alone does not make a new container package public. diff --git a/docs/zh/getting-started/install-options.md b/docs/zh/getting-started/install-options.md index a70daa900..b1687cc7c 100644 --- a/docs/zh/getting-started/install-options.md +++ b/docs/zh/getting-started/install-options.md @@ -1,7 +1,7 @@ --- title: "安装选项" source: docs/getting-started/install-options.md -source_hash: b3802e02538475497cc07acc8b5a20646cb555e3f77bc03e16eef4f82adac456 +source_hash: dc50c81c07f235146acb81aa965c6d9814dc3e57e43fd9769f98713eb008da4f --- [默认安装](install.md)无需任何选项。本页介绍安装选项、Compose 部署和反向代理配置。 @@ -16,7 +16,7 @@ Windows 上 `irm … | iex` 无法传参,默认执行策略也会拦截下载 ## Docker Compose 与托管平台 {#docker-compose-and-hosting-platforms} -在任一[支持的 Core 主机](install.md#prerequisites)上使用发行版中的 `compose.yaml`。发行流程会在 [Compose 模板](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/compose/compose.yaml)中固定初始化、Core 和 Web 镜像及源码版本。PostgreSQL 使用 `postgres:16-alpine`。它会启动 PostgreSQL、Core 和 Web。Web 把 `/v1` 和 `/api/v1` 转发到 Core。数据保存在 Docker 命名卷中。一次性初始化服务会在该目录中生成随机机密信息并准备节点安装程序;Core 启动时执行数据库迁移。[Compose 配置](../configuration.md#compose-installations)负责管理各项设置和数据目录。 +在任一[支持的 Core 主机](install.md#prerequisites)上使用发行版中的 `compose.yaml`。发行流程会固定初始化、Core、Web 和 PostgreSQL 镜像及源码版本。PostgreSQL 16 使用官方镜像,按摘要固定,并与其他镜像一起镜像到 GHCR。它会启动 PostgreSQL、Core 和 Web。Web 把 `/v1` 和 `/api/v1` 转发到 Core。数据保存在 Docker 命名卷中。一次性初始化服务会在该目录中生成随机机密信息并准备节点安装程序;Core 启动时执行数据库迁移。[Compose 配置](../configuration.md#compose-installations)负责管理各项设置和数据目录。 进行本地试用时,请将发行版的 `compose.yaml` 下载到一个空目录,然后运行: @@ -136,6 +136,6 @@ http://:8443 { ## 离线主机 {#offline-hosts} -安装程序使用当前 shell 的网络设置(包括 `HTTPS_PROXY`)从 GitHub Releases 下载发行文件。Docker 通过其守护进程拉取镜像,守护进程不读取 shell 的代理变量:如果只能经代理或镜像加速器访问容器注册表,请配置[守护进程代理](https://docs.docker.com/engine/daemon/proxy/)或[镜像加速器](https://docs.docker.com/docker-hub/image-library/mirror/),然后重新运行安装程序。 +安装程序使用当前 shell 的网络设置(包括 `HTTPS_PROXY`)从 GitHub Releases 下载发行文件。Docker 通过其守护进程拉取镜像,守护进程不读取 shell 的代理变量。所有镜像(包括 PostgreSQL)都来自 GHCR(`ghcr.io`),不需要 Docker Hub。如果只能经代理或镜像加速器访问 GHCR,请配置[守护进程代理](https://docs.docker.com/engine/daemon/proxy/)或[镜像加速器](https://docs.docker.com/docker-hub/image-library/mirror/),然后重新运行安装程序。 无法访问这些地址的 Linux amd64 主机可使用发行资产 `oac--linux-amd64-offline.tar.gz` 安装,其中包含所有镜像和节点制品。将其复制到该主机并解压,然后用 Python 3.9 或更高版本运行其中的 `install.py`;[源码安装](../development.md#install-a-checkout-without-publishing-a-release)介绍其选项。 diff --git a/docs/zh/maintainers.md b/docs/zh/maintainers.md index 44343b008..397c4566e 100644 --- a/docs/zh/maintainers.md +++ b/docs/zh/maintainers.md @@ -1,7 +1,7 @@ --- title: "构建并发布 OpenAgentCore" source: docs/maintainers.md -source_hash: 06be9f5ebff84ac8d7ec419e965345d0ac7d0064f8c79f35c4c025f73b9b82dd +source_hash: e2ebdce087ab0890f1c28871b09e7301d8efead9a2174d22469e9c8f6e9b2b77 --- 本指南面向负责构建和发布 OpenAgentCore 的维护者。要安装 Core 和 Web,请使用 [安装指南](getting-started/install.md)。安装器代码遵循的规则见 [部署](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/README.md) 和 [节点安装器](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/node/README.md);必需检查见 [CONTRIBUTING](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/CONTRIBUTING.md#required-checks)。 @@ -37,7 +37,7 @@ make build-core-distribution | `CORE_DISTRIBUTION_BUILD_DIR` | `~/.oac` 下的输出目录。默认值:`~/.oac/build/core-distribution` | | `CORE_DISTRIBUTION_BUILD_NETWORK` | Docker 构建网络:`default`、`host` 或 `none` | | `CORE_DISTRIBUTION_MICROSANDBOX_ARCHIVE` | 已缓存的 microsandbox 发布归档。默认值:`~/.oac/cache/microsandbox-v0.7.8-linux-x86_64.tar.gz`,缺失时下载 | -| `CORE_DISTRIBUTION_DATABASE_IMAGE` | PostgreSQL 16 镜像;默认值通过其 linux/amd64 清单摘要固定 | +| `CORE_DISTRIBUTION_DATABASE_IMAGE` | PostgreSQL 16 镜像;默认值为 `deploy/compose/database-image.txt` 中索引的 linux/amd64 清单 | 构建过程会复用 Core、Web、Runtime、SDK 和辅助程序构建器。清单会记录提交、其提交时间(`source_time`,用于在升级时为发布版本排序)和源代码树、镜像配置及 OCI 清单摘要、Runtime OCI 清单摘要、microsandbox 运行时和固件哈希,以及每个 Runtime 和节点构件的大小与 SHA-256;原生安装器在[目录](#native-installers)中仅记录其 SHA-256。输出包括控制归档及其 `.sha256`、可选的离线归档,以及带版本号的 Runtime、节点和原生安装器资源。此过程不会发布任何内容。如果目标目录中已包含此提交的分发包,重建会拒绝执行。 @@ -142,7 +142,7 @@ git push origin v1.2.3 ### 容器注册表 {#container-registry} -版本发布和手动 `build-` 草稿使用 `ghcr.io/minimax-ai/openagentcore/:`,其中 `` 为 `core`、`web`、`runtime` 或 `ingress`。Core、Web 和 ingress 索引包含 Linux amd64 和 arm64 镜像,Runtime 包含 Linux amd64。各平台镜像使用 `-` 标签,从发行归档加载。已有版本标签必须与发行镜像及平台集合一致。发布器校验全部版本索引后,才为稳定版更新 `latest`;预发布版和草稿保持 `latest` 不变。PostgreSQL 使用上游镜像。容器标签中的 SemVer 构建元数据用 `_` 替换 `+`,版本字符串上限为 128 个字符。镜像校验后,发布器上传该版本的 `compose.yaml` 和校验和清单。Compose 用对应发行版的索引摘要固定 Core、Web 和 ingress,初始化时检查其构建版本与 Compose 版本一致。 +版本发布和手动 `build-` 草稿使用 `ghcr.io/minimax-ai/openagentcore/:`,其中 `` 为 `core`、`web`、`runtime`、`ingress` 或 `postgres`。Core、Web 和 ingress 索引包含 Linux amd64 和 arm64 镜像,Runtime 包含 Linux amd64。各平台镜像使用 `-` 标签,从发行归档加载。已有版本标签必须与发行镜像及平台集合一致。发布器校验全部版本索引后,才为稳定版更新 `latest`;预发布版和草稿保持 `latest` 不变。`postgres` 是固定在 `deploy/compose/database-image.txt` 中的官方 PostgreSQL 索引,连同全部平台和原摘要一并复制;注册表中的标签若指向其他摘要,发布器会拒绝。要升级 PostgreSQL,请把该文件改为同一主版本的新摘要,模板、离线包和镜像复制都读取这一处。容器标签中的 SemVer 构建元数据用 `_` 替换 `+`,版本字符串上限为 128 个字符。镜像校验后,发布器上传该版本的 `compose.yaml` 和校验和清单。Compose 用对应发行版的索引摘要固定 Core、Web、ingress 和 PostgreSQL,初始化时检查其构建版本与 Compose 版本一致。 合并的构建/发布作业使用具有 `packages: write` 权限的 `GITHUB_TOKEN`。首次发布时,GitHub 会将每个容器软件包创建为私有:软件包管理员必须先在各自的软件包设置中将全部四个软件包改为 **Public**,用户才能匿名拉取。请参阅 [GitHub container visibility](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry)。更改可见性后,请验证未认证拉取。仅更改仓库可见性并不会使新的容器软件包变为公开。 diff --git a/scripts/build-core-distribution.sh b/scripts/build-core-distribution.sh index df064929c..cb403f83f 100755 --- a/scripts/build-core-distribution.sh +++ b/scripts/build-core-distribution.sh @@ -194,10 +194,19 @@ build_image runtime \ --build-arg "CODEX_IMAGE=${image_tags[0]}" --build-arg "CLAUDE_IMAGE=${image_tags[1]}" \ --build-arg "MCODE_IMAGE=${image_tags[2]}" "$stage/combined" -# Pin the linux/amd64 platform manifest, not the multi-platform tag: the -# containerd store keeps a pulled tag's whole index, whose export holds every -# platform. A platform manifest stays one image in both stores. -database_image="${CORE_DISTRIBUTION_DATABASE_IMAGE:-postgres:16-alpine@sha256:1a66d744c1b459e13b05a8fca341da84cb63383e99ce262210efee5a319d4551}" +# The pin in deploy/compose/database-image.txt names the multi-platform index. +# Resolve its linux/amd64 platform manifest: the containerd store keeps a pulled +# tag's whole index, whose export holds every platform, while a platform +# manifest stays one image in both stores. +database_image="${CORE_DISTRIBUTION_DATABASE_IMAGE:-}" +if [[ -z "$database_image" ]]; then + database_pin="$(cat deploy/compose/database-image.txt)" + database_amd64="$(docker buildx imagetools inspect --raw "$database_pin" | python3 -c ' +import json, sys +print(next(m["digest"] for m in json.load(sys.stdin)["manifests"] + if m["platform"] == {"os": "linux", "architecture": "amd64"}))')" + database_image="${database_pin%@*}@$database_amd64" +fi if [[ ! "$database_image" =~ ^sha256:[0-9a-f]{64}$ ]]; then docker pull --platform linux/amd64 "$database_image" fi diff --git a/scripts/ci_plan.py b/scripts/ci_plan.py index 0ac96cb39..b8c52d489 100644 --- a/scripts/ci_plan.py +++ b/scripts/ci_plan.py @@ -70,7 +70,7 @@ (("packages/claude-sdk-adapter/", "packages/mcode-harness/"), WEB, ("harness", "native", "backend", "distribution")), (("packages/tsconfig/",), (".json",), ("harness", "native")), (("deploy/node/", "scripts/acceptance/"), (".py", ".json", ".sh"), ("distribution",)), - (("deploy/compose/",), (".yaml", ".toml", ".json"), ("distribution", "compose")), + (("deploy/compose/",), (".yaml", ".toml", ".json", ".txt"), ("distribution", "compose")), (("scripts/compose-smoke.py", "scripts/render-compose.py", "deploy/compose/test_compose.py"), (".py",), ("distribution", "compose")), (("deploy/install.sh", "deploy/install-source.py", "deploy/test_install.py", "scripts/publish-core-release.", "scripts/core-distribution-manifest.", "scripts/build-core-distribution.sh", diff --git a/scripts/compose-smoke.py b/scripts/compose-smoke.py index c0c76feb7..163afafbc 100644 --- a/scripts/compose-smoke.py +++ b/scripts/compose-smoke.py @@ -111,6 +111,7 @@ def main(): 'INIT_IMAGE': 'ghcr.io/minimax-ai/openagentcore/ingress@sha256:' + '0' * 64, 'CORE_IMAGE': 'ghcr.io/minimax-ai/openagentcore/core@sha256:' + '0' * 64, 'WEB_IMAGE': 'ghcr.io/minimax-ai/openagentcore/web@sha256:' + '0' * 64, + 'DATABASE_IMAGE': render_compose.DATABASE_PIN, })) override = directory / 'offline-init.json' images = build_images(directory, project.removeprefix('oac-smoke-'), revision, source_time) diff --git a/scripts/publish-core-release.py b/scripts/publish-core-release.py index 0730bf7bc..c480919db 100644 --- a/scripts/publish-core-release.py +++ b/scripts/publish-core-release.py @@ -108,6 +108,27 @@ def registry_image(reference): return manifest, selected +def registry_index_digest(reference): + index = json.loads(subprocess.check_output(["docker", "buildx", "imagetools", "inspect", reference, + "--format", "{{json .Manifest}}"], text=True)) + digest = index.get("digest", "") + if not distribution.DIGEST.fullmatch(digest): + raise ValueError("Invalid registry index digest") + return digest + + +def mirror_database(repository, image_tag): + """Copy the pinned official PostgreSQL index to GHCR, keeping its digest and every platform.""" + source = render_compose.DATABASE_PIN + digest = source.rsplit("@", 1)[1] + reference = "ghcr.io/" + repository.lower() + "/postgres:" + image_tag + if registry_manifest(reference) is None: + subprocess.run(["docker", "buildx", "imagetools", "create", "--tag", reference, source], check=True) + if registry_index_digest(reference) != digest: + raise ValueError("Registry tag names a different PostgreSQL image: " + reference) + return {"tag": reference, "digest": reference.rsplit(":", 1)[0] + "@" + digest} + + def publish_images(assets, repository, revision, tag, floating_latest=False): """Verify both architectures before publishing immutable platform tags and indexes.""" image_tag = tag.replace("+", "_") @@ -183,13 +204,9 @@ def push_image(item): if registry_manifest(reference) is None: subprocess.run(["docker", "buildx", "imagetools", "create", "--tag", reference, *sources], check=True) verify_index(reference, registry_manifest(reference), expected) - index = json.loads(subprocess.check_output(["docker", "buildx", "imagetools", "inspect", reference, - "--format", "{{json .Manifest}}"], text=True)) - digest = index.get("digest", "") - if not distribution.DIGEST.fullmatch(digest): - raise ValueError("Invalid registry index digest") - pinned = reference.rsplit(":", 1)[0] + "@" + digest + pinned = reference.rsplit(":", 1)[0] + "@" + registry_index_digest(reference) result[name] = {"tag": reference, "digest": pinned} + database = mirror_database(repository, image_tag) # Advance floating tags only after all version indexes are verified. if floating_latest: for name, entry in result.items(): @@ -197,6 +214,7 @@ def push_image(item): expected = {arch: value[1] for (n, arch), value in entries.items() if n == name} subprocess.run(["docker", "buildx", "imagetools", "create", "--tag", latest, entry["digest"]], check=True) verify_index(latest, registry_manifest(latest), expected) + result["database"] = database return result @@ -304,6 +322,7 @@ def upload(path): "INIT_IMAGE": images["ingress"]["digest"], "CORE_IMAGE": images["core"]["digest"], "WEB_IMAGE": images["web"]["digest"], + "DATABASE_IMAGE": images["database"]["digest"], }) expected.update({path.name: path.stat().st_size for path in compose_files}) parallel_each(upload, compose_files) diff --git a/scripts/publish-core-release.test.py b/scripts/publish-core-release.test.py index 7ec9ea4e9..b7870eef7 100644 --- a/scripts/publish-core-release.test.py +++ b/scripts/publish-core-release.test.py @@ -54,7 +54,7 @@ def setUp(self): self.addCleanup(stack.close) self.images = stack.enter_context(mock.patch.object(publisher, "publish_images", return_value={ name: {"digest": "ghcr.io/minimax-ai/openagentcore/" + name + "@sha256:" + "ab" * 32} - for name in ("core", "web", "runtime", "ingress")})) + for name in ("core", "web", "runtime", "ingress", "database")})) self.api = stack.enter_context(mock.patch.object(publisher, "api", side_effect=self.response)) def test_registry_failure_leaves_release_draft(self): @@ -170,6 +170,7 @@ def test_compose_defaults_use_published_images_for_every_release_mode(self): for name in ("core", "web", "ingress"): self.assertIn("${OAC_IMAGE_" + name.upper() + ":-" + self.images.return_value[name]["digest"] + "}", compose) + self.assertIn("image: " + self.images.return_value["database"]["digest"] + "\n", compose) self.assertNotIn(":latest", compose) self.assertNotIn("__OAC_", compose) @@ -362,6 +363,7 @@ def setUp(self): self.configs = {'amd64': 'sha256:' + '1' * 64, 'arm64': 'sha256:' + '2' * 64} self.digests = {'amd64': 'sha256:' + '3' * 64, 'arm64': 'sha256:' + '4' * 64} self.index_digest = 'sha256:' + '5' * 64 + self.database_digest = publisher.render_compose.DATABASE_PIN.rsplit('@', 1)[1] self.remote_images = {} for arch in ('amd64', 'arm64'): names = publisher.IMAGE_NAMES if arch == 'amd64' else ('core', 'web', 'ingress') @@ -403,6 +405,8 @@ def output(self, command, **kwargs): if command[1] == 'buildx': # Buildx versions used by hosted runners treat templates beginning # with {{.Manifest as a request for human-readable manifest output. + if '/postgres:' in command[4]: + return json.dumps({'digest': self.database_digest}) if command[-1].startswith('{{.Manifest'): return 'Name: example/core:v1.2.3\nDigest: ' + self.index_digest + '\n' self.assertEqual(command[-2:], ['--format', '{{json .Manifest}}']) @@ -425,6 +429,23 @@ def test_publishes_and_reuses_verified_multiarch_indexes(self): self.assertEqual(len(self.remote_images['ghcr.io/minimax-ai/openagentcore/core:v1.2.3']['manifests']), 2) self.run.reset_mock(); self.publish(); self.assertEqual(self.pushes(), []) + def mirrors(self): + return [c.args[0] for c in self.run.call_args_list if c.args[0][-2].endswith('/postgres:v1.2.3')] + + def test_mirrors_the_pinned_postgresql_index_to_ghcr_by_digest(self): + result = self.publish() + pin = publisher.render_compose.DATABASE_PIN + self.assertRegex(pin, r'^postgres:16-alpine@sha256:[0-9a-f]{64}$') + self.assertEqual(self.mirrors(), [['docker', 'buildx', 'imagetools', 'create', '--tag', + 'ghcr.io/minimax-ai/openagentcore/postgres:v1.2.3', pin]]) + self.assertEqual(result['database']['digest'], 'ghcr.io/minimax-ai/openagentcore/postgres@' + self.database_digest) + self.run.reset_mock(); self.publish(); self.assertEqual(self.mirrors(), []) + + def test_changed_postgresql_digest_in_the_registry_is_refused(self): + self.database_digest = 'sha256:' + '9' * 64 + with self.assertRaisesRegex(ValueError, 'different PostgreSQL image'): + self.publish() + def test_invalid_index_digest_prevents_floating_tag_updates(self): for digest in ('', 'not-a-digest'): with self.subTest(digest=digest): @@ -439,8 +460,8 @@ def test_invalid_index_digest_prevents_floating_tag_updates(self): def test_latest_updates_after_all_version_indexes(self): self.publish(floating_latest=True) creates = [c.args[0][5] for c in self.run.call_args_list if c.args[0][1:4] == ['buildx', 'imagetools', 'create']] - self.assertTrue(all(ref.endswith(':v1.2.3') for ref in creates[:4])) - self.assertTrue(all(ref.endswith(':latest') for ref in creates[4:])) + self.assertTrue(all(ref.endswith(':v1.2.3') for ref in creates[:5])) + self.assertTrue(all(ref.endswith(':latest') for ref in creates[5:])) def test_conflicting_platform_prevents_every_push(self): self.remote_images['ghcr.io/minimax-ai/openagentcore/web:v1.2.3-arm64'] = {'config': {'digest': 'different'}} diff --git a/scripts/render-compose.py b/scripts/render-compose.py index b4e00c248..18c450364 100644 --- a/scripts/render-compose.py +++ b/scripts/render-compose.py @@ -1,7 +1,7 @@ #!/usr/bin/env python3 """Fill the Compose template with one release's node metadata. -The template is deploy/compose/compose.yaml. Initialization, Core and Web images are pinned to the release. A release publishes the rendered file; this script does not run Docker. +The template is deploy/compose/compose.yaml. Initialization, Core and Web images are pinned to the release, and PostgreSQL to the digest in deploy/compose/database-image.txt, which a release mirrors to GHCR. A release publishes the rendered file; this script does not run Docker. """ import hashlib import pathlib @@ -10,7 +10,9 @@ ROOT = pathlib.Path(__file__).resolve().parents[1] TEMPLATE = ROOT / "deploy/compose/compose.yaml" -TOKENS = ("REVISION", "INIT_IMAGE", "CORE_IMAGE", "WEB_IMAGE") +TOKENS = ("REVISION", "INIT_IMAGE", "CORE_IMAGE", "WEB_IMAGE", "DATABASE_IMAGE") +# The one home of the PostgreSQL pin: the official upstream multi-platform index. +DATABASE_PIN = (ROOT / "deploy/compose/database-image.txt").read_text().strip() def render(values): @@ -20,9 +22,11 @@ def render(values): raise ValueError("Missing Compose values: " + ", ".join(missing)) if not re.fullmatch(r"[0-9a-f]{40}", values["REVISION"]): raise ValueError("REVISION must be a full source commit SHA") - for name in TOKENS[1:]: + for name in TOKENS[1:-1]: if not re.fullmatch(r"ghcr\.io/[a-z0-9._/-]+@sha256:[0-9a-f]{64}", values[name]): raise ValueError(name + " must be an immutable GHCR image reference") + if not re.fullmatch(r"[a-z0-9._/:-]+@sha256:[0-9a-f]{64}", values["DATABASE_IMAGE"]): + raise ValueError("DATABASE_IMAGE must be an immutable image reference") text = TEMPLATE.read_text() for name in TOKENS: token = "__OAC_" + name + "__"