diff --git a/deploy/README.md b/deploy/README.md index 40d20aae5..53ce99450 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -20,7 +20,7 @@ An existing directory runs the release named by `OAC_REVISION` in its `compose.y `install.py` prepares a new installation in a `.staging-*` directory inside the installation directory: the node payload, `.env`, `oac` and both Compose files. After `docker compose config` accepts them, it moves them into place, `compose.yaml` last. An earlier failure removes the stage, and the directory too when the run created it, so the same command can be rerun; the next run also removes a stage left by a killed run. A directory with other contents but no `compose.yaml` is refused. -The host `oac` command also implements `apply`, `core-key` and `rotate-core-key`. `apply` runs init, which finishes a first start that stopped before it and refuses a data volume committed to another release, then checks Core configuration before starting services and recreating those that changed. Reading the key executes `oac-web core-key` in Web. Rotation runs in the initialization container, where Linux file ownership is identical on every host, then restarts Core and Web. Start, stop, logs and removal use `docker compose`. The initialization image verifies and copies bundled node metadata without network access. Its receipt authenticates immutable metadata and credentials; the operator-rotatable Core key and its digest remain mutable. No service receives a Docker socket. Initialization emits structured step logs without credential values; view them with `docker compose logs --timestamps init`. +The host `oac` command also implements `apply`, `core-key`, `rotate-core-key` and `uninstall`. `apply` runs init, which finishes a first start that stopped before it and refuses a data volume committed to another release, then checks Core configuration before starting services and recreating those that changed. Reading the key executes `oac-web core-key` in Web. Rotation runs in the initialization container, where Linux file ownership is identical on every host, then restarts Core and Web. Start, stop and logs use `docker compose`; `oac uninstall` removes the installation, its data volume, the lock and the sidecar. The initialization image verifies and copies bundled node metadata without network access. Its receipt authenticates immutable metadata and credentials; the operator-rotatable Core key and its digest remain mutable. No service receives a Docker socket. Initialization emits structured step logs without credential values; view them with `docker compose logs --timestamps init`. Web serves the console and forwards `/v1` and `/api/v1` to Core, so it is the only published service. HTTPS is terminated by the operator's reverse proxy or hosting platform, which routes to `web:8080`; `OAC_PUBLIC_URL` records that origin. diff --git a/deploy/install-source.py b/deploy/install-source.py index fadc69ecd..b4135deef 100644 --- a/deploy/install-source.py +++ b/deploy/install-source.py @@ -6,11 +6,11 @@ import json import os from pathlib import Path +import re import shutil import subprocess import sys import tempfile -from urllib.parse import urlsplit import uuid # The bundle ships the node installer's distribution modules beside this file. @@ -20,8 +20,24 @@ import node_payload +# Matches Core's OAC_PUBLIC_URL rule (publicorigin.New): a canonical http(s) origin with no path or credentials. +PUBLIC_ORIGIN = re.compile(r'https?://(?P(?:[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.)*[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?|\[(?P[0-9a-f:.]+)\])(?::(?P[1-9][0-9]{0,4}))?') + + +def valid_public_origin(value): + match = PUBLIC_ORIGIN.fullmatch(value) + if not match or len(match['host']) > 253 or int(match['port'] or 1) > 65535: + return False + try: + return match['ip6'] is None or bool(ipaddress.IPv6Address(match['ip6'])) + except ValueError: + return False + + def run(failure, *args, **kwargs): """Run a command whose output the user sees; a nonzero exit raises failure.""" + # The installation's .env is the one home of OAC_* and COMPOSE_* settings, so the caller's shell cannot override them. + kwargs['env'] = {name: value for name, value in os.environ.items() if not name.upper().startswith(('OAC_', 'COMPOSE_'))} if subprocess.run(args, **kwargs).returncode: raise ValueError(failure) @@ -90,8 +106,9 @@ def install(bundle, destination, host, port, public_url, upgrade): except ValueError: raise ValueError('compose.override.yaml must stay JSON so the installer can replace its own entries') from None services = override.setdefault('services', {}) - for name, image in (('core', 'core'), ('web', 'web'), ('init', 'ingress'), ('database', 'database')): - services.setdefault(name, {})['image'] = images[image] + # compose.yaml is the one home of the image references; earlier installers also wrote them here. + for name in ('core', 'web', 'init', 'database'): + services.setdefault(name, {}).pop('image', None) web = services['web'] web['volumes'] = [volume for volume in web.get('volumes', []) if not (isinstance(volume, dict) and volume.get('target') == '/node-payload')] web['volumes'].append({'type': 'bind', 'source': str(destination / 'node-payload'), 'target': '/node-payload', 'read_only': True}) @@ -158,9 +175,8 @@ def main(): if not 1 <= args.web_port <= 65535: parser.error('web port must be between 1 and 65535') if args.public_url is not None: - url = urlsplit(args.public_url) - if url.scheme not in ('http', 'https') or not url.hostname or url.username or url.password or url.path not in ('', '/') or url.query or url.fragment or any(c in args.public_url for c in "\r\n'\\$"): - parser.error('public URL must be an HTTP(S) origin without credentials') + if not valid_public_origin(args.public_url): + parser.error('public URL must be a canonical http or https origin without a path, such as https://core.example') destination = args.install_dir.resolve() try: install(BUNDLE, destination, args.host, args.web_port, args.public_url, args.upgrade) diff --git a/deploy/test_install_source.py b/deploy/test_install_source.py index 685d8547d..587f8f657 100644 --- a/deploy/test_install_source.py +++ b/deploy/test_install_source.py @@ -112,6 +112,7 @@ def test_upgrade_checks_the_new_release_before_replacing_it_and_keeps_operator_e override = json.loads((self.destination / 'compose.override.yaml').read_text()) nfs = {'type': 'bind', 'source': '/srv/workspaces', 'target': '/srv/workspaces'} override['services']['core']['volumes'] = [nfs] + override['services']['core']['image'] = 'old' (self.destination / 'compose.override.yaml').write_text(json.dumps(override)) before = self.snapshot() upgraded = 'b' * 40 @@ -134,7 +135,8 @@ def test_upgrade_checks_the_new_release_before_replacing_it_and_keeps_operator_e services = json.loads((self.destination / 'compose.override.yaml').read_text())['services'] self.assertEqual(services['core']['volumes'], [nfs]) self.assertEqual(os.stat(self.destination / 'compose.override.yaml').st_mode & 0o777, 0o600) - self.assertEqual({services[name]['image'] for name in ('core', 'web', 'init', 'database')}, {'sha256:' + '2' * 64}) + self.assertFalse([name for name, service in services.items() if 'image' in service]) + self.assertEqual((self.destination / 'compose.yaml').read_text().count('sha256:' + '2' * 64), 4) self.assertEqual([volume['target'] for volume in services['web']['volumes']], ['/node-payload']) self.assertIn('OAC_REVISION: ' + upgraded, (self.destination / 'compose.yaml').read_text()) self.assertEqual((self.destination / 'oac').read_bytes(), b'upgraded operator') @@ -151,6 +153,19 @@ def test_thin_bundle_rejected_before_creating_installation(self): self.install() self.assertFalse(self.destination.exists()) + def test_docker_children_ignore_the_shells_installation_settings(self): + with patch.dict(os.environ, {'COMPOSE_PROJECT_NAME': 'other', 'OAC_PUBLIC_URL': 'https://other.example', 'PATH_KEEP': '1'}), patch.object(subprocess, 'run') as run: + run.return_value.returncode = 0 + installer.run('failed', 'docker', 'compose', 'up') + env = run.call_args.kwargs['env'] + self.assertEqual((env.get('PATH_KEEP'), 'COMPOSE_PROJECT_NAME' in env, 'OAC_PUBLIC_URL' in env), ('1', False, False)) + + def test_public_url_must_be_what_core_accepts(self): + for value in ('https://core.example', 'http://10.0.0.5:8080', 'http://[::1]:8091'): + self.assertTrue(installer.valid_public_origin(value), value) + for value in ('https://core.example/', 'https://Core.example', 'https://core.example/path', 'https://core.example?', 'https://user@core.example', 'https://core.example:0', 'https://core.example:65536', 'ftp://core.example', 'https://core..example'): + self.assertFalse(installer.valid_public_origin(value), value) + def test_failures_exit_with_one_line_and_the_next_step(self): (self.bundle / 'SHA256SUMS').write_text('0' * 64 + ' manifest.json\n') self.assertEqual(self.main(), 'Distribution manifest checksum mismatch') diff --git a/docs/getting-started/operations.md b/docs/getting-started/operations.md index e8bc7b441..cb553ecff 100644 --- a/docs/getting-started/operations.md +++ b/docs/getting-started/operations.md @@ -136,8 +136,9 @@ For an installation whose data all lives in its Docker volume, with no independe cd ~/.oac/core docker compose stop project=$(sed -n 's/^COMPOSE_PROJECT_NAME=//p' .env) +image=$(docker compose images --quiet database) mkdir -p ~/oac-backup -docker run --rm -v "${project}_data:/data:ro" -v ~/oac-backup:/backup --entrypoint tar postgres:16-alpine czf /backup/data.tar.gz -C /data . +docker run --rm -v "${project}_data:/data:ro" -v ~/oac-backup:/backup --entrypoint tar "$image" czf /backup/data.tar.gz -C /data . cp -a ~/.oac/core ~/oac-backup/installation docker compose start ``` @@ -150,7 +151,8 @@ cd ~/.oac/core project=$(sed -n 's/^COMPOSE_PROJECT_NAME=//p' .env) docker compose pull docker compose up --no-start -docker run --rm -v "${project}_data:/data" -v ~/oac-backup:/backup:ro --entrypoint tar postgres:16-alpine xzf /backup/data.tar.gz -C /data +image=$(docker compose images --quiet database) +docker run --rm -v "${project}_data:/data" -v ~/oac-backup:/backup:ro --entrypoint tar "$image" xzf /backup/data.tar.gz -C /data ./oac apply ``` @@ -247,7 +249,7 @@ A rollback restores the backup taken before the upgrade: stop the services, [res ~/.oac/core/oac uninstall ``` -On Windows, run `& "$HOME/.oac/core/oac.exe" uninstall`. The command shows the Compose project, the data volume, the installation directory and the installation ID, and deletes nothing until you type the directory name or `yes`. It then removes the containers, the network and the data volume, the installation directory and the [installation lock](../configuration.md#installation-directory) (`~/.oac/core.lock`). On Windows the running `oac.exe` cannot delete itself: the command prints the `Remove-Item` that removes the rest. It keeps the Docker images so that a reinstall doesn't download them again, and prints the `docker image rm` command that removes them. Run it again and it reports that nothing is installed. +On Windows, run `& "$HOME/.oac/core/oac.exe" uninstall`. The command shows the Compose project, the data volume, the installation directory and the installation ID, and deletes nothing until you type the directory name or `yes`. It then removes the containers, the network and the data volume, the installation directory and the [installation lock](../configuration.md#installation-directory) (`~/.oac/core.lock`). On Windows the running `oac.exe` cannot delete itself: the command prints the `Remove-Item` that removes the rest, and you run it before you reinstall into the same directory. It keeps the Docker images so that a reinstall doesn't download them again, and prints the `docker image rm` command that removes them. All data goes with it: Projects and API keys, Session history, stored credentials and the Core key. To keep the data, stop the installation with `docker compose stop` instead, or [back it up](#back-up) first. diff --git a/docs/maintainers.md b/docs/maintainers.md index 4f1ac07f1..8ee15c1d4 100644 --- a/docs/maintainers.md +++ b/docs/maintainers.md @@ -142,7 +142,7 @@ Distribution and Runtime archives use `pigz` level 6 with at most four compressi Version releases and manual `build-` drafts publish `ghcr.io/minimax-ai/openagentcore/:`, where `` is `core`, `web`, `runtime`, `ingress` or `postgres`. Core, Web and ingress indexes contain Linux amd64 and arm64 images; Runtime contains Linux amd64. Platform images use `-` tags and are loaded from the release archives. Existing version tags must match the release images and platform set. The publisher verifies every version index before updating `latest` for a stable release; prereleases and drafts leave `latest` unchanged. `postgres` is the official PostgreSQL index pinned in `deploy/compose/database-image.txt`, copied with all its platforms and its digest; the publisher refuses a registry tag that names another digest. To move PostgreSQL, change that file to a new digest of the same major version, which is the one place the template, the offline bundle and the mirror read. SemVer build metadata uses `_` in place of `+` in container tags; version strings are limited to 128 characters. After verifying the images, the publisher uploads the release's `compose.yaml` and checksum list. Compose pins Core, Web, ingress and PostgreSQL by their release index digests, and initialization checks its build revision against the Compose revision. -The combined build/publication job uses `GITHUB_TOKEN` with `packages: write`. On the first publication, GitHub creates each container package as private: a package administrator must change all four packages to **Public** in their package settings before users can pull anonymously. See [GitHub container visibility](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry). Verify an unauthenticated pull after changing visibility. Repository visibility alone does not make a new container package public. +The combined build/publication job uses `GITHUB_TOKEN` with `packages: write`. On the first publication, GitHub creates each container package as private: a package administrator must change all five packages (`core`, `web`, `runtime`, `ingress` and `postgres`) to **Public** in their package settings before users can pull anonymously. See [GitHub container visibility](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry). Verify an unauthenticated pull after changing visibility. Repository visibility alone does not make a new container package public. GHCR and GitHub Releases do not share a transaction. A failed release may leave some matching version tags in GHCR; preserve those images and follow the draft recovery procedure below using the original artifacts. Registry failures other than a missing manifest stop publication. The job summary records digest-pinned references. These images and the rendered Compose files still require the configuration, secrets and routing described in [Configuration](./configuration.md). diff --git a/docs/zh/getting-started/operations.md b/docs/zh/getting-started/operations.md index b4a265576..cb5a1e1ad 100644 --- a/docs/zh/getting-started/operations.md +++ b/docs/zh/getting-started/operations.md @@ -1,7 +1,7 @@ --- title: "运维" source: docs/getting-started/operations.md -source_hash: daba721bddebf8d6544ddef597390307a7b4a28f041978133acc200004812d69 +source_hash: 1f07f9a80da3ad0218949a4856093d716d9d3af2b40426be9e74e3bd0b3ca18b --- 安装运维人员负责 Core 主机、存储和可用性。节点主机运行各自的服务；参阅[节点](nodes.md)。设置见[配置参考](../configuration.md)。 @@ -138,8 +138,9 @@ Core 记录每次公开资源写入所使用的密钥；历史保留策略为 [` cd ~/.oac/core docker compose stop project=$(sed -n 's/^COMPOSE_PROJECT_NAME=//p' .env) +image=$(docker compose images --quiet database) mkdir -p ~/oac-backup -docker run --rm -v "${project}_data:/data:ro" -v ~/oac-backup:/backup --entrypoint tar postgres:16-alpine czf /backup/data.tar.gz -C /data . +docker run --rm -v "${project}_data:/data:ro" -v ~/oac-backup:/backup --entrypoint tar "$image" czf /backup/data.tar.gz -C /data . cp -a ~/.oac/core ~/oac-backup/installation docker compose start ``` @@ -152,7 +153,8 @@ cd ~/.oac/core project=$(sed -n 's/^COMPOSE_PROJECT_NAME=//p' .env) docker compose pull docker compose up --no-start -docker run --rm -v "${project}_data:/data" -v ~/oac-backup:/backup:ro --entrypoint tar postgres:16-alpine xzf /backup/data.tar.gz -C /data +image=$(docker compose images --quiet database) +docker run --rm -v "${project}_data:/data" -v ~/oac-backup:/backup:ro --entrypoint tar "$image" xzf /backup/data.tar.gz -C /data ./oac apply ``` @@ -249,7 +251,7 @@ v0.0.10 及更高版本的安装可以原地升级到更新的发布版本，并 ~/.oac/core/oac uninstall ``` -Windows 上运行 `& "$HOME/.oac/core/oac.exe" uninstall`。命令会展示 Compose 项目、数据卷、安装目录和安装 ID，在你输入目录名或 `yes` 之前不会删除任何内容。随后它删除容器、网络和数据卷、安装目录以及[安装锁](../configuration.md#installation-directory)（`~/.oac/core.lock`）。Windows 上正在运行的 `oac.exe` 无法删除自身：命令会打印删除其余内容的 `Remove-Item`。它保留 Docker 镜像，重新安装时不必再次下载，并打印删除镜像的 `docker image rm` 命令。再次运行会提示没有已安装的内容。 +Windows 上运行 `& "$HOME/.oac/core/oac.exe" uninstall`。命令会展示 Compose 项目、数据卷、安装目录和安装 ID，在你输入目录名或 `yes` 之前不会删除任何内容。随后它删除容器、网络和数据卷、安装目录以及[安装锁](../configuration.md#installation-directory)（`~/.oac/core.lock`）。Windows 上正在运行的 `oac.exe` 无法删除自身：命令会打印删除其余内容的 `Remove-Item`，在同一目录重新安装之前先运行它。它保留 Docker 镜像，重新安装时不必再次下载，并打印删除镜像的 `docker image rm` 命令。 全部数据随之删除：项目和 API 密钥、Session 历史、存储的凭据和 Core 密钥。要保留数据，请用 `docker compose stop` 停止安装，或先[备份](#back-up)。 diff --git a/docs/zh/maintainers.md b/docs/zh/maintainers.md index 397c4566e..7597b1a9d 100644 --- a/docs/zh/maintainers.md +++ b/docs/zh/maintainers.md @@ -1,7 +1,7 @@ --- title: "构建并发布 OpenAgentCore" source: docs/maintainers.md -source_hash: e2ebdce087ab0890f1c28871b09e7301d8efead9a2174d22469e9c8f6e9b2b77 +source_hash: 18bb1549e69536abfe758b043e8621e77899a848efa8613760f7aff728afd63e --- 本指南面向负责构建和发布 OpenAgentCore 的维护者。要安装 Core 和 Web，请使用 [安装指南](getting-started/install.md)。安装器代码遵循的规则见 [部署](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/README.md) 和 [节点安装器](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/deploy/node/README.md)；必需检查见 [CONTRIBUTING](https://github.com/MiniMax-AI/OpenAgentCore/blob/main/CONTRIBUTING.md#required-checks)。 @@ -144,7 +144,7 @@ git push origin v1.2.3 版本发布和手动 `build-` 草稿使用 `ghcr.io/minimax-ai/openagentcore/:`，其中 `` 为 `core`、`web`、`runtime`、`ingress` 或 `postgres`。Core、Web 和 ingress 索引包含 Linux amd64 和 arm64 镜像，Runtime 包含 Linux amd64。各平台镜像使用 `-` 标签，从发行归档加载。已有版本标签必须与发行镜像及平台集合一致。发布器校验全部版本索引后，才为稳定版更新 `latest`；预发布版和草稿保持 `latest` 不变。`postgres` 是固定在 `deploy/compose/database-image.txt` 中的官方 PostgreSQL 索引，连同全部平台和原摘要一并复制；注册表中的标签若指向其他摘要，发布器会拒绝。要升级 PostgreSQL，请把该文件改为同一主版本的新摘要，模板、离线包和镜像复制都读取这一处。容器标签中的 SemVer 构建元数据用 `_` 替换 `+`，版本字符串上限为 128 个字符。镜像校验后，发布器上传该版本的 `compose.yaml` 和校验和清单。Compose 用对应发行版的索引摘要固定 Core、Web、ingress 和 PostgreSQL，初始化时检查其构建版本与 Compose 版本一致。 -合并的构建/发布作业使用具有 `packages: write` 权限的 `GITHUB_TOKEN`。首次发布时，GitHub 会将每个容器软件包创建为私有：软件包管理员必须先在各自的软件包设置中将全部四个软件包改为 **Public**，用户才能匿名拉取。请参阅 [GitHub container visibility](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry)。更改可见性后，请验证未认证拉取。仅更改仓库可见性并不会使新的容器软件包变为公开。 +合并的构建/发布作业使用具有 `packages: write` 权限的 `GITHUB_TOKEN`。首次发布时，GitHub 会将每个容器软件包创建为私有：软件包管理员必须先在各自的软件包设置中将全部五个软件包（`core`、`web`、`runtime`、`ingress` 和 `postgres`）改为 **Public**，用户才能匿名拉取。请参阅 [GitHub container visibility](https://docs.github.com/en/packages/working-with-a-github-packages-registry/working-with-the-container-registry)。更改可见性后，请验证未认证拉取。仅更改仓库可见性并不会使新的容器软件包变为公开。 GHCR 和 GitHub Releases 不共享事务。发布失败后，GHCR 中可能仍会保留一些匹配的版本标签；请保留这些镜像，并使用原始构件按照下文的草稿恢复流程操作。除清单缺失以外，注册表故障都会停止发布。作业摘要会记录按摘要固定的引用。这些镜像和渲染后的 Compose 文件仍需要[配置](configuration.md)中描述的配置、机密和路由。 diff --git a/services/core/cmd/oac/install.go b/services/core/cmd/oac/install.go index 8cc2d91eb..2ebb1b700 100644 --- a/services/core/cmd/oac/install.go +++ b/services/core/cmd/oac/install.go @@ -21,6 +21,8 @@ import ( "strconv" "strings" "time" + + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/publicorigin" ) // Installation is host-independent. Only the launchers select a native binary; @@ -83,7 +85,15 @@ func dockerOutput(ctx context.Context, dir string, args ...string) ([]byte, erro cmd.Env = dockerEnv() var stderr bytes.Buffer cmd.Stderr = &stderr - output, err := cmd.Output() + var output []byte + var err error + if len(args) > 0 && args[0] == "pull" { + // A pull transfers hundreds of megabytes; Docker's own progress shows it is not stuck. + cmd.Stdout = os.Stdout + err = cmd.Run() + } else { + output, err = cmd.Output() + } if err != nil { if reason := strings.TrimSpace(stderr.String()); reason != "" { return nil, errors.New(reason) @@ -179,14 +189,11 @@ func (i installer) install(ctx context.Context, o installOptions) error { } } } - u, err := url.Parse(o.publicURL) - if err != nil || (u.Scheme != "http" && u.Scheme != "https") || u.Hostname() == "" || u.User != nil || u.RawQuery != "" || u.Fragment != "" || (u.Path != "" && u.Path != "/") { - return errors.New("--public-url must be an HTTP(S) origin") + if _, err := publicorigin.New(o.publicURL); err != nil { + return errors.New("--public-url must be a canonical http or https origin without a path, such as https://core.example") } - for _, value := range []string{o.publicURL, o.version} { - if strings.ContainsAny(value, "\r\n'\\") { - return errors.New("installation options contain invalid characters") - } + if strings.ContainsAny(o.version, "\r\n'\\") { + return errors.New("installation options contain invalid characters") } o.dir = filepath.Clean(o.dir) if err := os.MkdirAll(filepath.Dir(o.dir), 0o700); err != nil { @@ -211,7 +218,7 @@ func (i installer) installLocked(ctx context.Context, o installOptions, detected switch { case errors.Is(err, exec.ErrNotFound): return errors.New("Docker is not installed; install Docker Engine 26 or newer with the Compose plugin, or Docker Desktop on macOS and Windows, then rerun the installer") - case err != nil && strings.Contains(err.Error(), "permission denied"): + case err != nil && strings.Contains(err.Error(), "permission denied while trying to connect to"): return errors.New("This account cannot use Docker: add it to the docker group and log in again, or use Docker Desktop or rootless Docker. Do not run the installer with sudo.") case err != nil: return fmt.Errorf("Docker is not running or not reachable; start it and rerun the installer: %w", err) @@ -245,6 +252,9 @@ func (i installer) installLocked(ctx context.Context, o installOptions, detected return err } fresh := len(entries) == 0 + if fresh && o.upgrade { + return fmt.Errorf("No installation in %s; pass the installation's --install-dir.", o.dir) + } installed := buildRevision if !fresh { for _, name := range []string{"compose.yaml", ".env", cliName()} { @@ -390,16 +400,24 @@ func (i installer) installLocked(ctx context.Context, o installOptions, detected return nil } -// portHolderHint names an OpenAgentCore Compose project that publishes the port. Its containers keep running after the installation directory is deleted by hand. +// portHolderHint names an OpenAgentCore Compose project that publishes the port. A project whose directory is gone keeps running; one whose directory exists is a live installation, so the hint names it instead of offering to delete it. func (i installer) portHolderHint(ctx context.Context, port int) string { - names, err := i.docker(ctx, "", "ps", "--filter", "publish="+strconv.Itoa(port), "--format", `{{.Label "com.docker.compose.project"}}`) + rows, err := i.docker(ctx, "", "ps", "--filter", "publish="+strconv.Itoa(port), "--format", `{{.Label "com.docker.compose.project"}} {{.Label "com.docker.compose.project.working_dir"}}`) if err != nil { return "" } - for _, name := range strings.Fields(string(names)) { - if strings.HasPrefix(name, "oac-") { - return fmt.Sprintf("\nThe Compose project %s holds it. If you deleted its installation directory, remove it with: docker compose -p %s down --volumes --remove-orphans (this deletes its data)", name, name) + for _, row := range strings.Split(string(rows), "\n") { + name, dir, _ := strings.Cut(strings.TrimSpace(row), "\t") + if !strings.HasPrefix(name, "oac-") { + continue + } + if dir == "" { + return fmt.Sprintf("\nThe Compose project %s holds it.", name) } + if _, err := os.Stat(dir); err == nil { + return fmt.Sprintf("\nThe installation in %s holds it; upgrade or manage it with --install-dir %s.", dir, dir) + } + return fmt.Sprintf("\nThe Compose project %s holds it, and its installation directory %s no longer exists. Remove it with: docker compose -p %s down --volumes --remove-orphans (this deletes its data)", name, dir, name) } return "" } @@ -443,13 +461,9 @@ func (i installer) differingSettings(o installOptions) []string { // releaseCompose downloads the requested release's compose.yaml, verifies its published checksum and requires it to start this installer's release. func (i installer) releaseCompose(ctx context.Context, version string) ([]byte, error) { - repository := os.Getenv("OAC_REPOSITORY") - if repository == "" { - repository = "MiniMax-AI/OpenAgentCore" - } - base := "https://github.com/" + repository + "/releases/latest/download/" + base := "https://github.com/MiniMax-AI/OpenAgentCore/releases/latest/download/" if version != "latest" { - base = "https://github.com/" + repository + "/releases/download/" + url.PathEscape(version) + "/" + base = "https://github.com/MiniMax-AI/OpenAgentCore/releases/download/" + url.PathEscape(version) + "/" } fmt.Println("Downloading release configuration...") sums, err := i.download(ctx, base+"compose-sha256sums.txt") diff --git a/services/core/cmd/oac/install_test.go b/services/core/cmd/oac/install_test.go index 519137900..2a2311d2a 100644 --- a/services/core/cmd/oac/install_test.go +++ b/services/core/cmd/oac/install_test.go @@ -538,3 +538,16 @@ func TestUnknownCommandPrintsUsageOutsideAnInstallation(t *testing.T) { t.Fatalf("got %v", err) } } + +func TestInstallRejectsWhatCoreRejectsBeforePublishing(t *testing.T) { + for _, value := range []string{"https://core.example/", "https://Core.example", "https://core.example/path"} { + f := newInstallFixture(t) + f.options.publicURL = value + if err := f.run(); err == nil || !strings.Contains(err.Error(), "--public-url must be a canonical") { + t.Fatalf("%s: %v", value, err) + } + if exists(f.options.dir) || exists(f.options.dir+".staging") { + t.Fatalf("%s: files were published", value) + } + } +} diff --git a/services/core/cmd/oac/uninstall.go b/services/core/cmd/oac/uninstall.go index 00d33a8e7..87be1a6d4 100644 --- a/services/core/cmd/oac/uninstall.go +++ b/services/core/cmd/oac/uninstall.go @@ -60,7 +60,18 @@ func uninstall(ctx context.Context, runner Runner, project, dir, keep string, in images, _ := runner.Output(ctx, "config", "--images") fmt.Fprintf(out, "This deletes the OpenAgentCore installation in %s:\n Compose project %s (containers and network)\n Data volume %s_data (database, secrets, Core key, Session history)\n Directory %s\n Installation ID %s\n", dir, project, project, dir, id) fmt.Fprintf(out, "Type %q or yes to delete it: ", filepath.Base(dir)) - answer, _ := bufio.NewReader(in).ReadString('\n') + // Read in the background so Ctrl-C ends the wait at once instead of after Enter. + typed := make(chan string, 1) + go func() { + answer, _ := bufio.NewReader(in).ReadString('\n') + typed <- answer + }() + var answer string + select { + case answer = <-typed: + case <-ctx.Done(): + return ctx.Err() + } if answer = strings.TrimSpace(answer); answer != "yes" && answer != filepath.Base(dir) { return errors.New("Uninstall cancelled; nothing was changed.") } diff --git a/services/core/cmd/oac/uninstall_test.go b/services/core/cmd/oac/uninstall_test.go index 8259c4339..f6c7fbc8c 100644 --- a/services/core/cmd/oac/uninstall_test.go +++ b/services/core/cmd/oac/uninstall_test.go @@ -2,6 +2,8 @@ package main import ( "context" + "errors" + "io" "net" "os" "path/filepath" @@ -84,6 +86,21 @@ func TestUninstallRefusedConfirmationChangesNothing(t *testing.T) { } } +func TestUninstallInterruptedAtThePromptStopsAtOnce(t *testing.T) { + f := newUninstallFixture(t) + ctx, cancel := context.WithCancel(context.Background()) + reader, _ := io.Pipe() // never answers + done := make(chan error, 1) + go func() { done <- uninstall(ctx, f, "oac-abc123", f.dir, "", reader, io.Discard) }() + cancel() + if err := <-done; !errors.Is(err, context.Canceled) { + t.Fatal(err) + } + if !exists(f.dir) { + t.Fatal("an interrupted uninstall removed files") + } +} + func TestUninstallRemovesProjectDirectoryAndLock(t *testing.T) { for _, answer := range []string{"yes\n", "core\n"} { f := newUninstallFixture(t) @@ -102,10 +119,6 @@ func TestUninstallRemovesProjectDirectoryAndLock(t *testing.T) { t.Fatalf("missing %q in %s", want, out) } } - // The second run finds no .env and reports nothing installed. - if _, err := installedProject(f.dir); !os.IsNotExist(err) { - t.Fatal(err) - } } } @@ -133,23 +146,39 @@ func TestUninstallLeavesOnlyTheRunningBinary(t *testing.T) { } } -func TestInstallPortHintNamesAnOrphanedProject(t *testing.T) { - f := newInstallFixture(t) - listener, err := net.Listen("tcp", f.options.host+":0") - if err != nil { - t.Fatal(err) - } - defer listener.Close() - f.options.port = listener.Addr().(*net.TCPAddr).Port - inner := f.docker - f.docker = func(ctx context.Context, dir string, args ...string) ([]byte, error) { - if args[0] == "ps" { - return []byte("oac-abc123\n"), nil +func TestInstallPortHintOffersDeletionOnlyForAMissingDirectory(t *testing.T) { + for _, c := range []struct{ dir, want, not string }{ + {filepath.Join(t.TempDir(), "gone"), "docker compose -p oac-abc123 down --volumes --remove-orphans", ""}, + {t.TempDir(), "--install-dir", "down --volumes"}, + } { + f := newInstallFixture(t) + listener, err := net.Listen("tcp", f.options.host+":0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + f.options.port = listener.Addr().(*net.TCPAddr).Port + inner := f.docker + f.docker = func(ctx context.Context, dir string, args ...string) ([]byte, error) { + if args[0] == "ps" { + return []byte("oac-abc123\t" + c.dir + "\n"), nil + } + return inner(ctx, dir, args...) + } + err = f.run() + if err == nil || !strings.Contains(err.Error(), c.want) || (c.not != "" && strings.Contains(err.Error(), c.not)) { + t.Fatal(err) } - return inner(ctx, dir, args...) } - err = f.run() - if err == nil || !strings.Contains(err.Error(), "docker compose -p oac-abc123 down --volumes --remove-orphans") { +} + +func TestUpgradeWithoutAnInstallationRefuses(t *testing.T) { + f := newInstallFixture(t) + f.options.upgrade = true + if err := f.run(); err == nil || !strings.Contains(err.Error(), "No installation in "+f.options.dir) { t.Fatal(err) } + if exists(f.options.dir) || exists(f.options.dir+".staging") { + t.Fatal("a refused upgrade left files") + } } diff --git a/services/core/cmd/server/managed_setup_test.go b/services/core/cmd/server/managed_setup_test.go index aaaf9cd02..f866174e6 100644 --- a/services/core/cmd/server/managed_setup_test.go +++ b/services/core/cmd/server/managed_setup_test.go @@ -3,6 +3,7 @@ package main import ( "context" "errors" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/publicorigin" "os" "path/filepath" "testing" @@ -25,7 +26,7 @@ import ( ) func TestWebSetupCreatesManagerWithoutLocalProvider(t *testing.T) { - origin, err := deployment.NewPublicOrigin("https://core.example") + origin, err := publicorigin.New("https://core.example") if err != nil { t.Fatal(err) } diff --git a/services/core/internal/deployment/public_url.go b/services/core/internal/deployment/public_url.go index e6d947160..d9bd86ae1 100644 --- a/services/core/internal/deployment/public_url.go +++ b/services/core/internal/deployment/public_url.go @@ -2,72 +2,8 @@ package deployment import ( "context" - "net" - "net/url" - "strconv" - "strings" ) -// PublicOrigin is OAC_PUBLIC_URL, the one origin applications, nodes, -// sandboxes and self-hosted executors use. Every Core address they are given -// is derived from it; none is parsed back into an origin. -type PublicOrigin struct{ origin string } - -// NewPublicOrigin accepts a canonical public origin, never a path or -// credential. It may be http or https: a reverse proxy in front of Web -// terminates TLS when the installation uses it. -func NewPublicOrigin(value string) (PublicOrigin, error) { - u, err := url.Parse(value) - if err != nil || u.Hostname() == "" || u.User != nil || u.Path != "" || u.RawPath != "" || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || u.RawFragment != "" || u.Opaque != "" || u.String() != value || u.Host != strings.ToLower(u.Host) { - return PublicOrigin{}, ErrInvalidInput - } - if strings.ContainsAny(u.Host, "\\% \t\r\n") || strings.HasSuffix(u.Host, ":") { - return PublicOrigin{}, ErrInvalidInput - } - if port := u.Port(); port != "" { - n, err := strconv.Atoi(port) - if err != nil || n < 1 || n > 65535 || strconv.Itoa(n) != port { - return PublicOrigin{}, ErrInvalidInput - } - } - if net.ParseIP(u.Hostname()) == nil { - if len(u.Hostname()) > 253 || strings.ContainsAny(u.Host, "[]") { - return PublicOrigin{}, ErrInvalidInput - } - for _, label := range strings.Split(u.Hostname(), ".") { - if len(label) == 0 || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' { - return PublicOrigin{}, ErrInvalidInput - } - for _, char := range label { - if (char < 'a' || char > 'z') && (char < '0' || char > '9') && char != '-' { - return PublicOrigin{}, ErrInvalidInput - } - } - } - } - if u.Scheme != "https" && u.Scheme != "http" { - return PublicOrigin{}, ErrInvalidInput - } - return PublicOrigin{origin: value}, nil -} - -// String is the origin itself. -func (o PublicOrigin) String() string { return o.origin } - -// API is the base URL of the Agents API. -func (o PublicOrigin) API() string { return o.origin + "/v1" } - -// RuntimeAPI is the base URL sandboxes and nodes call. -func (o PublicOrigin) RuntimeAPI() string { return o.origin + "/api/v1" } - -// DaemonWebSocket is the daemon transport: ws on an http origin, wss on https. -func (o PublicOrigin) DaemonWebSocket() string { - return "ws" + strings.TrimPrefix(o.origin, "http") + "/api/v1/agent-daemon/ws" -} - -// InstallerBase is the prefix of the versioned native installer downloads. -func (o PublicOrigin) InstallerBase() string { return o.origin + "/api/v1/agent-daemon/install/" } - // AddressBindings counts what is bound to an installation address: nodes // connect to the address they enrolled with, hosted sandboxes were started with // the address current at the time, and self-hosted executors were installed diff --git a/services/core/internal/deployment/rules_test.go b/services/core/internal/deployment/rules_test.go index 53c875dcc..cddba64ee 100644 --- a/services/core/internal/deployment/rules_test.go +++ b/services/core/internal/deployment/rules_test.go @@ -12,31 +12,6 @@ import ( "github.com/google/uuid" ) -func TestPublicOrigin(t *testing.T) { - for _, value := range []string{"https://core.example", "https://core.example:8443", "http://localhost:8091", "http://127.0.0.2:8091", "http://[::1]:8091", "https://[2001:db8::1]", "http://core.example", "http://core:8091", "http://10.0.0.5:8080"} { - if origin, err := NewPublicOrigin(value); err != nil || origin.String() != value { - t.Errorf("valid Core URL %q rejected: %v", value, err) - } - } - for value, socket := range map[string]string{"https://core.example": "wss://core.example/api/v1/agent-daemon/ws", "http://[::1]:8091": "ws://[::1]:8091/api/v1/agent-daemon/ws"} { - origin, err := NewPublicOrigin(value) - if err != nil || origin.DaemonWebSocket() != socket || origin.API() != value+"/v1" || origin.RuntimeAPI() != value+"/api/v1" || origin.InstallerBase() != value+"/api/v1/agent-daemon/install/" { - t.Errorf("addresses derived from %q: %+v %v", value, origin, err) - } - } - for _, value := range []string{ - "", "ftp://core.example", "ws://core.example", "https://core.example/", "https://user:secret@core.example", - "https://core.example/path", "https://core.example?", "https://core.example?q=x", "https://core.example#x", "https://core.example#", - "https://CORE.example", "https://core.example:", "https://core.example:0", "https://core.example:65536", "https://core.example:0080", - "https://core.example\\evil", "https://[not-an-ip]", "https://-core.example", "https://core..example", "https://core_example", - "https://core.example.", "https://bücher.example", "https://core.example:0443", - } { - if _, err := NewPublicOrigin(value); !errors.Is(err, ErrInvalidInput) { - t.Errorf("invalid Core URL %q accepted: %v", value, err) - } - } -} - func TestParseID(t *testing.T) { id := uuid.New() if got, err := parseID(strings.ToUpper(id.String())); err != nil || got != id.String() { diff --git a/services/core/internal/processconfig/config.go b/services/core/internal/processconfig/config.go index c8eeb67ad..f87c099ca 100644 --- a/services/core/internal/processconfig/config.go +++ b/services/core/internal/processconfig/config.go @@ -10,6 +10,7 @@ import ( "cmp" "encoding/base64" "encoding/json" + "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/publicorigin" "io" "net/url" "os" @@ -22,7 +23,6 @@ import ( "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/api" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/credentialcrypto" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/databaseurl" - "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/deployment" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/engine" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/execution" "github.com/MiniMax-AI/OpenAgentCore/services/core/internal/oauthrefresh" @@ -44,7 +44,7 @@ type Config struct { // Addr is OAC_ADDR, the listener address. Addr string // PublicOrigin is OAC_PUBLIC_URL. - PublicOrigin deployment.PublicOrigin + PublicOrigin publicorigin.Origin // DatabaseURL is OAC_DATABASE_URL with the password from // OAC_DATABASE_PASSWORD_FILE. DatabaseURL string @@ -110,7 +110,7 @@ func Load() (Config, error) { if value == "" { return Config{}, configError("OAC_PUBLIC_URL is required; applications, nodes and sandboxes reach Core at this origin") } - if c.PublicOrigin, err = deployment.NewPublicOrigin(value); err != nil { + if c.PublicOrigin, err = publicorigin.New(value); err != nil { return Config{}, configError("OAC_PUBLIC_URL must be a canonical http or https origin without path, credentials, query or fragment, such as https://core.example") } if c.DatabaseURL, err = databaseurl.FromEnvironment(); err != nil { diff --git a/services/core/internal/publicorigin/publicorigin.go b/services/core/internal/publicorigin/publicorigin.go new file mode 100644 index 000000000..5d5320b4c --- /dev/null +++ b/services/core/internal/publicorigin/publicorigin.go @@ -0,0 +1,74 @@ +// Package publicorigin owns OAC_PUBLIC_URL's one rule, shared by Core and the +// installer. +package publicorigin + +import ( + "errors" + "net" + "net/url" + "strconv" + "strings" +) + +// ErrInvalid reports a value that is not a canonical public origin. +var ErrInvalid = errors.New("invalid public origin") + +// Origin is OAC_PUBLIC_URL, the one origin applications, nodes, +// sandboxes and self-hosted executors use. Every Core address they are given +// is derived from it; none is parsed back into an origin. +type Origin struct{ origin string } + +// New accepts a canonical public origin, never a path or +// credential. It may be http or https: a reverse proxy in front of Web +// terminates TLS when the installation uses it. +func New(value string) (Origin, error) { + u, err := url.Parse(value) + if err != nil || u.Hostname() == "" || u.User != nil || u.Path != "" || u.RawPath != "" || u.RawQuery != "" || u.ForceQuery || u.Fragment != "" || u.RawFragment != "" || u.Opaque != "" || u.String() != value || u.Host != strings.ToLower(u.Host) { + return Origin{}, ErrInvalid + } + if strings.ContainsAny(u.Host, "\\% \t\r\n") || strings.HasSuffix(u.Host, ":") { + return Origin{}, ErrInvalid + } + if port := u.Port(); port != "" { + n, err := strconv.Atoi(port) + if err != nil || n < 1 || n > 65535 || strconv.Itoa(n) != port { + return Origin{}, ErrInvalid + } + } + if net.ParseIP(u.Hostname()) == nil { + if len(u.Hostname()) > 253 || strings.ContainsAny(u.Host, "[]") { + return Origin{}, ErrInvalid + } + for _, label := range strings.Split(u.Hostname(), ".") { + if len(label) == 0 || len(label) > 63 || label[0] == '-' || label[len(label)-1] == '-' { + return Origin{}, ErrInvalid + } + for _, char := range label { + if (char < 'a' || char > 'z') && (char < '0' || char > '9') && char != '-' { + return Origin{}, ErrInvalid + } + } + } + } + if u.Scheme != "https" && u.Scheme != "http" { + return Origin{}, ErrInvalid + } + return Origin{origin: value}, nil +} + +// String is the origin itself. +func (o Origin) String() string { return o.origin } + +// API is the base URL of the Agents API. +func (o Origin) API() string { return o.origin + "/v1" } + +// RuntimeAPI is the base URL sandboxes and nodes call. +func (o Origin) RuntimeAPI() string { return o.origin + "/api/v1" } + +// DaemonWebSocket is the daemon transport: ws on an http origin, wss on https. +func (o Origin) DaemonWebSocket() string { + return "ws" + strings.TrimPrefix(o.origin, "http") + "/api/v1/agent-daemon/ws" +} + +// InstallerBase is the prefix of the versioned native installer downloads. +func (o Origin) InstallerBase() string { return o.origin + "/api/v1/agent-daemon/install/" } diff --git a/services/core/internal/publicorigin/publicorigin_test.go b/services/core/internal/publicorigin/publicorigin_test.go new file mode 100644 index 000000000..2127a975a --- /dev/null +++ b/services/core/internal/publicorigin/publicorigin_test.go @@ -0,0 +1,31 @@ +package publicorigin + +import ( + "errors" + "testing" +) + +func TestNew(t *testing.T) { + for _, value := range []string{"https://core.example", "https://core.example:8443", "http://localhost:8091", "http://127.0.0.2:8091", "http://[::1]:8091", "https://[2001:db8::1]", "http://core.example", "http://core:8091", "http://10.0.0.5:8080"} { + if origin, err := New(value); err != nil || origin.String() != value { + t.Errorf("valid Core URL %q rejected: %v", value, err) + } + } + for value, socket := range map[string]string{"https://core.example": "wss://core.example/api/v1/agent-daemon/ws", "http://[::1]:8091": "ws://[::1]:8091/api/v1/agent-daemon/ws"} { + origin, err := New(value) + if err != nil || origin.DaemonWebSocket() != socket || origin.API() != value+"/v1" || origin.RuntimeAPI() != value+"/api/v1" || origin.InstallerBase() != value+"/api/v1/agent-daemon/install/" { + t.Errorf("addresses derived from %q: %+v %v", value, origin, err) + } + } + for _, value := range []string{ + "", "ftp://core.example", "ws://core.example", "https://core.example/", "https://user:secret@core.example", + "https://core.example/path", "https://core.example?", "https://core.example?q=x", "https://core.example#x", "https://core.example#", + "https://CORE.example", "https://core.example:", "https://core.example:0", "https://core.example:65536", "https://core.example:0080", + "https://core.example\\evil", "https://[not-an-ip]", "https://-core.example", "https://core..example", "https://core_example", + "https://core.example.", "https://bücher.example", "https://core.example:0443", + } { + if _, err := New(value); !errors.Is(err, ErrInvalid) { + t.Errorf("invalid Core URL %q accepted: %v", value, err) + } + } +}