diff --git a/apps/web/e2e/nodes.spec.ts b/apps/web/e2e/nodes.spec.ts index 9c2a4a08f..df9a1230b 100644 --- a/apps/web/e2e/nodes.spec.ts +++ b/apps/web/e2e/nodes.spec.ts @@ -19,7 +19,7 @@ test("adds a node: host requirements, a root/sudo command, a countdown, the same await expect(add.getByText("What the sandbox backend needs on the host; the installer checks it and names anything missing")).toBeVisible(); await expect(add.getByText("SELinux is not enforcing")).toBeVisible(); await expect(add.getByText("In sudo mode, one Core per host: a host already running a sudo-mode node for another Core is refused.")).toBeVisible(); - await expect(add.getByText("CPUs and memory for at least one sandbox: 2 CPU · 4 GiB; about 2 GB of disk for the Runtime image")).toBeVisible(); + await expect(add.getByText("CPUs and memory for at least one sandbox: 2 CPU · 4 GiB; about 3 GB of free disk while the Runtime image installs")).toBeVisible(); await expect(add.getByText("Reaches https://core.example.com, as do its sandboxes")).toBeVisible(); // Node installation only offers a system service; there is no user-mode alternative. await expect(add.getByText("No sudo on this host?")).toHaveCount(0); diff --git a/apps/web/src/features/sandbox/enrollment-command.test.ts b/apps/web/src/features/sandbox/enrollment-command.test.ts index d0d1a2405..4f669c51c 100644 --- a/apps/web/src/features/sandbox/enrollment-command.test.ts +++ b/apps/web/src/features/sandbox/enrollment-command.test.ts @@ -1,7 +1,7 @@ import { describe, expect, it } from "vitest"; import { spawnSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { chmodSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs"; +import { chmodSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; import { nodeInstallCommand, nodeLogCommand, nodeUninstallCommand } from "./enrollment-command"; @@ -12,7 +12,7 @@ describe("sandbox connection and enrollment", () => { expect(nodeLogCommand("a b")).toBe("sudo journalctl -u 'oac-node-a b.service'"); }); // "as root": a root shell runs the sudo command without sudo. - it.each(["success", "download failure", "checksum mismatch", "installer failure", "as root", "no color", "uppercase proxy", "lowercase proxy", "empty proxy", "root proxy", "unsupported platform", "old python", "uninstall", "killed download"])("executes safely, passes the token only on stdin and cleans private downloads after %s", (scenario) => { + it.each(["success", "download failure", "checksum mismatch", "installer failure", "as root", "no color", "uppercase proxy", "lowercase proxy", "empty proxy", "root proxy", "unsupported platform", "old python", "linked directory", "uninstall", "killed download"])("executes safely, passes the token only on stdin and cleans private downloads after %s", (scenario) => { const parent = join(homedir(), ".oac", "tests"); mkdirSync(parent, { recursive: true }); const root = mkdtempSync(join(parent, "node-command-")); @@ -63,7 +63,15 @@ process.exit(process.env.SCENARIO==='installer failure'?7:0);`); if (scenario === "lowercase proxy") Object.assign(proxyEnv, { http_proxy: proxy, https_proxy: proxy, no_proxy: "core.example,localhost", HTTP_PROXY: "http://wrong.invalid" }); if (scenario === "empty proxy") proxyEnv.http_proxy = ""; const options = { env: { ...process.env, ...proxyEnv, HOME: root, NO_COLOR: scenario === "no color" ? "" : undefined, PATH: `${bin}:${process.env.PATH}`, TMPDIR: temporary, SCENARIO: scenario, FIXTURE: fixture, REPORT: report, SUDO_REPORT: sudoReport, PRINTF_REPORT: printfReport }, encoding: "utf8" as const }; + if (scenario === "linked directory") { mkdirSync(join(root, ".oac")); symlinkSync(temporary, join(root, ".oac/node-bootstrap")); } let result = spawnSync("sh", ["-c", command], options); + if (scenario === "linked directory") { + expect(result.status).toBe(1); + expect(result.stderr).toContain(`${join(root, ".oac/node-bootstrap")} is a symbolic link`); + expect(result.stdout).not.toContain("Downloading"); + expect(readdirSync(temporary)).toEqual([]); + return; + } if (scenario === "killed download") { expect(result.signal === "SIGKILL" || result.status === 137).toBe(true); expect(readdirSync(join(root, ".oac/node-bootstrap"))).toContain("download.partial"); diff --git a/apps/web/src/features/sandbox/enrollment-command.ts b/apps/web/src/features/sandbox/enrollment-command.ts index a81e7046b..e9e0b2eec 100644 --- a/apps/web/src/features/sandbox/enrollment-command.ts +++ b/apps/web/src/features/sandbox/enrollment-command.ts @@ -15,9 +15,12 @@ for tool in curl sha256sum python3 flock; do command -v "$tool" >/dev/null || { python3 -c 'import sys; sys.exit(sys.version_info < (3, 9))' || { printf 'Python 3.9 or newer is required.\\n' >&2; exit 1; } [ "$(id -u)" -eq 0 ] || command -v sudo >/dev/null || { printf 'sudo is required; ask the host administrator to run this command.\\n' >&2; exit 1; } d="$HOME/.oac/node-bootstrap" -[ ! -L "$d" ] && mkdir -p "$d" && [ -d "$d" ] && [ -O "$d" ] && chmod 700 "$d" || { printf 'Cannot use the private node bootstrap directory.\\n' >&2; exit 1; } +[ ! -L "$d" ] || { printf '%s is a symbolic link; remove it and rerun the command.\\n' "$d" >&2; exit 1; } +mkdir -p "$d" || { printf 'Cannot create %s. A root-owned ~/.oac usually means this command ran under sudo; run it as your own user.\\n' "$d" >&2; exit 1; } +{ [ -d "$d" ] && [ -O "$d" ]; } || { printf '%s must be a directory you own. If root owns it, remove it with sudo and run the command as your own user.\\n' "$d" >&2; exit 1; } +chmod 700 "$d" || exit installer="$d"/${quote(scriptDigest + ".pyz")} -for f in "$d/download.lock" "$d/download.partial" "$installer"; do [ ! -L "$f" ] && { [ ! -e "$f" ] || { [ -f "$f" ] && [ -O "$f" ]; }; } || { printf 'Invalid node bootstrap file.\\n' >&2; exit 1; }; done +for f in "$d/download.lock" "$d/download.partial" "$installer"; do [ ! -L "$f" ] && { [ ! -e "$f" ] || { [ -f "$f" ] && [ -O "$f" ]; }; } || { printf '%s must be a regular file you own, not a link.\\n' "$f" >&2; exit 1; }; done exec 9>>"$d/download.lock" flock -n 9 || { printf 'Another node download is running; wait and retry.\\n' >&2; exit 1; } rm -f "$d/download.partial" || exit @@ -26,7 +29,7 @@ s=; [ "$(id -u)" -eq 0 ] || s=sudo export http_proxy="\${http_proxy-\${HTTP_PROXY-}}" https_proxy="\${https_proxy-\${HTTPS_PROXY-}}" no_proxy="\${no_proxy-\${NO_PROXY-}}" export HTTP_PROXY="$http_proxy" HTTPS_PROXY="$https_proxy" NO_PROXY="$no_proxy" printf '\\n==> Downloading node installer...\\n' && -curl -fs --connect-timeout 15 --max-time 60 --retry 2 --retry-connrefused --retry-delay 1 --max-filesize 1048576 ${quote(sourceUrl + "/node-install/node-install.pyz")} -o "$d/download.partial" || { c=$?; printf 'Cannot download node installer; check the console URL, TLS and proxy settings.\\n' >&2; exit "$c"; } +curl -fsS --connect-timeout 15 --max-time 60 --retry 2 --retry-connrefused --retry-delay 1 --max-filesize 1048576 ${quote(sourceUrl + "/node-install/node-install.pyz")} -o "$d/download.partial" || { c=$?; printf 'Cannot download node installer; check the console URL, TLS and proxy settings.\\n' >&2; exit "$c"; } printf '==> Verifying node installer...\\n' && printf '%s %s\\n' ${quote(scriptDigest)} "$d/download.partial" | sha256sum -c --status || { printf 'Node installer checksum mismatch; generate a fresh command in Web and retry.\\n' >&2; exit 1; } mv "$d/download.partial" "$installer" || exit diff --git a/apps/web/src/features/sandbox/node-enrollment.ts b/apps/web/src/features/sandbox/node-enrollment.ts index b68c450ae..93328f795 100644 --- a/apps/web/src/features/sandbox/node-enrollment.ts +++ b/apps/web/src/features/sandbox/node-enrollment.ts @@ -28,7 +28,7 @@ export interface HostPrerequisite { * and install nothing; * - `host_capacity`: the host's CPUs and memory hold one sandbox of the * deployment's size (else the node reports capacity_insufficient); the Runtime - * image needs about 2 GB of disk; + * image needs about 3 GB of free disk while it installs (`require_disk`); * - network: node files and artifacts only from the console at the public URL * (`fetch` and `metadata`, which never use a release URL), Core's /api/v1 at the * same URL (node_spec.py, the `register` call), and sandboxes reach Core as well @@ -41,7 +41,7 @@ export function hostRequirements(sized: boolean): HostPrerequisite[] { { label: "SELinux is not enforcing" }, { label: "In sudo mode, one Core per host: a host already running a sudo-mode node for another Core is refused." }, { label: "What the sandbox backend needs on the host; the installer checks it and names anything missing" }, - { label: sized ? "CPUs and memory for at least one sandbox: {{size}}; about 2 GB of disk for the Runtime image" : "CPUs and memory for at least one sandbox; about 2 GB of disk for the Runtime image" }, + { label: sized ? "CPUs and memory for at least one sandbox: {{size}}; about 3 GB of free disk while the Runtime image installs" : "CPUs and memory for at least one sandbox; about 3 GB of free disk while the Runtime image installs" }, { label: "Reaches {{core}}, as do its sandboxes" }, ]; } diff --git a/apps/web/src/i18n/locales/zh-CN/sandbox.ts b/apps/web/src/i18n/locales/zh-CN/sandbox.ts index 836699d15..4e61b2b21 100644 --- a/apps/web/src/i18n/locales/zh-CN/sandbox.ts +++ b/apps/web/src/i18n/locales/zh-CN/sandbox.ts @@ -156,8 +156,8 @@ export const sandbox = { "{{name}} still points at the old Core address {{address}}.": "{{name}} 仍指向旧的 Core 地址 {{address}}。", "If this node's old Core address no longer responds, first remove it on the Nodes page, then add --force to the uninstall command.": "如果这个节点的旧 Core 地址已无法访问，请先在“节点”页移除它，再在卸载命令后加上 --force。", "Uninstall command with --force": "带 --force 的卸载命令", - "CPUs and memory for at least one sandbox: {{size}}; about 2 GB of disk for the Runtime image": "CPU 和内存至少够一个沙箱：{{size}}；Runtime 镜像约需 2 GB 磁盘", - "CPUs and memory for at least one sandbox; about 2 GB of disk for the Runtime image": "CPU 和内存至少够一个沙箱；Runtime 镜像约需 2 GB 磁盘", + "CPUs and memory for at least one sandbox: {{size}}; about 3 GB of free disk while the Runtime image installs": "CPU 和内存至少够一个沙箱：{{size}}；安装 Runtime 镜像期间约需 3 GB 空闲磁盘", + "CPUs and memory for at least one sandbox; about 3 GB of free disk while the Runtime image installs": "CPU 和内存至少够一个沙箱；安装 Runtime 镜像期间约需 3 GB 空闲磁盘", "Reaches {{core}}, as do its sandboxes": "能访问 {{core}}，它的沙箱也要能访问", "The command creates the oac-node service user and a system service. It installs no software; if something is missing it stops and says what to install.": "命令会创建 oac-node 服务用户和一个系统服务。它不安装任何软件；缺少什么时会停下并说明要装什么。", "Set a public address other machines can reach before adding nodes.": "添加节点前，请先设置其他机器能访问的公开地址。", diff --git a/deploy/node/README.md b/deploy/node/README.md index a1804d37e..18254f1cf 100644 --- a/deploy/node/README.md +++ b/deploy/node/README.md @@ -29,6 +29,10 @@ The node installer runs as root and prepares the host for one node per installat The generated command uses `~/.oac/node-bootstrap` in the invoking account's home. Under `download.lock`, it discards an unfinished `download.partial`, downloads and verifies the bootstrap, then publishes `.pyz`. The lock is released before invoking sudo; verified files stay available to running installers. Node file writes share `distribution.temporary_file`: under the caller's installation lock, remove the previous temporary file, write privately, and publish atomically. The same lifecycle covers payloads, extraction, helpers, settings and enrollment tokens. Registration tokens are removed even when a rerun finds the node already registered. +Core records a node when its registration completes, and the installer marks that with `registered.json`. Until then, an earlier attempt's configuration, identity and cached files that no longer match are disposable: the next command replaces them and keeps the files that match. A registered node's files are never replaced; a rerun that finds a different node program stops and names the remove, uninstall and add steps. + +Before downloading, the installer refuses when the node account's disk lacks the space the missing files and the Runtime import need. It deletes the Runtime archive and its unpacked copy once the image is imported. + Configuration files are published only after their complete contents are written; a retry preserves an existing matching file and refuses conflicting contents. Generation leases are durable locks, not disposable temporary files: interruption between creating a lease and recording its inode identity requires operator inspection. A retry refuses that unrecorded lease instead of replacing an inode that a Runtime helper could still hold. The distribution manifest is the one download contract for nodes: flat versioned file names, and the compressed and unpacked size and SHA-256 of the Runtime. diff --git a/deploy/node/distribution.py b/deploy/node/distribution.py index 1aec341c4..ef59978db 100644 --- a/deploy/node/distribution.py +++ b/deploy/node/distribution.py @@ -44,30 +44,30 @@ def docker_command(arguments, timeout=30): raise DistributionError('Cannot inspect or load the distribution image; check Docker access and disk space') from None +def docker_image(manifest, name, docker=('docker',)): + """The local ID of the manifest's image when Docker already holds it, else None.""" + expected = image_identities(manifest, name) + for identity in dict.fromkeys(expected): + result = docker_command(list(docker) + ['image', 'inspect', identity, '--format', + '{{.Id}} {{.Os}}/{{.Architecture}}']) + if result.returncode: + continue + fields = result.stdout.strip().split() + if len(fields) != 2 or fields[0] not in expected or fields[1] != 'linux/amd64': + raise DistributionError('Docker image identity or platform differs from the distribution: ' + name) + return fields[0] + return None + + def ensure_docker_image(manifest, name, archive, docker=('docker',)): """Resolve a proven local ID; obtain a verified archive only on a cache miss.""" - expected = image_identities(manifest, name) - docker = list(docker) - - def inspect(): - for identity in dict.fromkeys(expected): - result = docker_command(docker + ['image', 'inspect', identity, '--format', - '{{.Id}} {{.Os}}/{{.Architecture}}']) - if result.returncode: - continue - fields = result.stdout.strip().split() - if len(fields) != 2 or fields[0] not in expected or fields[1] != 'linux/amd64': - raise DistributionError('Docker image identity or platform differs from the distribution: ' + name) - return fields[0] - return None - - identity = inspect() + identity = docker_image(manifest, name, docker) if identity is not None: return identity - result = docker_command(docker + ['load', '--input', str(archive())], timeout=1800) + result = docker_command(list(docker) + ['load', '--input', str(archive())], timeout=1800) if result.returncode: raise DistributionError('Cannot load the distribution image; check Docker access and free disk space: ' + name) - identity = inspect() + identity = docker_image(manifest, name, docker) if identity is None: raise DistributionError('Cannot verify the loaded distribution image: ' + name) return identity @@ -139,25 +139,22 @@ def matches(path, entry): return path.stat().st_size == entry['size'] and digest(path) == entry['sha256'] -def obtain_artifact(manifest, logical_path, destination, offline_root=None): +def obtain_artifact(manifest, logical_path, destination, replace=False): + """Download a verified artifact unless the destination already holds it. + + A destination with other bytes is refused, or replaced when the caller owns + no use of it (replace).""" entry = artifact(manifest, logical_path) target = checked_path(destination) if target.exists(): - if not matches(target, entry): + if matches(target, entry): + return target + if not replace: raise ArtifactError('Cached artifact differs; preserve the installation and inspect: ' + logical_path) - return target - source = None - if offline_root is not None: - candidate = Path(offline_root) / 'artifacts' / entry['filename'] - if candidate.exists(): - if candidate.is_symlink() or not candidate.is_file(): - raise DistributionError('Offline artifact must be a regular file: ' + logical_path) - source = candidate - if source is not None: - return copy_artifact(source, target, entry, logical_path) + target.unlink() base = manifest.get('artifact_base_url', '') if not isinstance(base, str) or not base or urlsplit(base).query: - raise ArtifactError('No downloadable artifact source; use the matching offline bundle') + raise ArtifactError('No downloadable artifact source: ' + logical_path) url = safe_url(base.rstrip('/') + '/' + entry['filename']) with temporary_file(target) as partial: for attempt in range(3): @@ -192,22 +189,6 @@ def temporary_file(target): partial.unlink(missing_ok=True) -def copy_artifact(source, target, entry, logical_path): - with temporary_file(target) as temporary: - with os.fdopen(os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), 'wb') as output, source.open('rb') as stream: - count = 0 - for block in iter(lambda: stream.read(1024 * 1024), b''): - count += len(block) - if count > entry['size']: - raise ArtifactError('Artifact exceeds published size: ' + logical_path) - output.write(block) - if count != entry['size'] or digest(temporary) != entry['sha256']: - raise ArtifactError('Artifact checksum mismatch: ' + logical_path) - os.chmod(temporary, 0o700 if logical_path.startswith('native/') else 0o600) - os.replace(temporary, target) - return target - - # Stop downloads that bring fewer bytes than this in a window. SLOW_SECONDS, SLOW_BYTES = 60, 64 * 1024 @@ -247,7 +228,8 @@ def download_artifact(url, partial, entry, logical_path): raise http.client.IncompleteRead(b'', size - count) -def runtime_archive(manifest, cache_root, offline_root=None): +def runtime_archive(manifest, cache_root, replace=False): + """The verified, unpacked Runtime archive; replace as in obtain_artifact.""" entry = artifact(manifest, 'images/runtime.tar.gz') expanded = {'sha256': entry.get('unpacked_sha256'), 'size': entry.get('unpacked_size')} if (not re.fullmatch(r'[0-9a-f]{64}', str(expanded['sha256'])) @@ -256,10 +238,12 @@ def runtime_archive(manifest, cache_root, offline_root=None): root = Path(cache_root) target = checked_path(root / 'images/runtime.tar') if target.exists(): - if not matches(target, expanded): + if matches(target, expanded): + return target + if not replace: raise ArtifactError('Cached Runtime archive differs; preserve state and inspect it') - return target - archive = obtain_artifact(manifest, 'images/runtime.tar.gz', root / 'images/runtime.tar.gz', offline_root) + target.unlink() + archive = obtain_artifact(manifest, 'images/runtime.tar.gz', root / 'images/runtime.tar.gz', replace) with temporary_file(target) as temporary: try: with os.fdopen(os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), 'wb') as output, gzip.open(archive, 'rb') as stream: diff --git a/deploy/node/install_display.py b/deploy/node/install_display.py index 5b003e9a6..46c07714c 100644 --- a/deploy/node/install_display.py +++ b/deploy/node/install_display.py @@ -52,8 +52,8 @@ def heading(message): print("\n" + color(message, "1")) -def error(message): - print(color("Installation failed: ", "31", sys.stderr) + message, file=sys.stderr, flush=True) +def error(message, label="Installation failed: "): + print((color(label, "31", sys.stderr) if label else "") + message, file=sys.stderr, flush=True) def paragraph(message): diff --git a/deploy/node/node_generations.py b/deploy/node/node_generations.py index d056421f0..584ceab92 100644 --- a/deploy/node/node_generations.py +++ b/deploy/node/node_generations.py @@ -35,8 +35,6 @@ def atomic_json(path, value): def helper_archive(args, installer): """Read the trusted bootstrap before dropping access to its private directory.""" source = Path(sys.argv[0]) - if getattr(args, "bundle", None) is not None: - source = args.bundle / "node-install.pyz" if source.suffix == ".pyz" and source.is_file() and not source.is_symlink(): return source.read_bytes() archive = io.BytesIO() @@ -394,7 +392,6 @@ def prepare(args, installer): if not finalized or value is None or not image_available(value, installer): settings = installer.private_json(root / "preparation.json") args.source_url = installer.origin(settings["source_url"]) - args.bundle = None manifest, sums = installer.metadata(args.source_url, prefix="releases/" + runtime["source_commit"] + "/") try: installer.node_spec.verify_release(args.configuration, manifest) @@ -462,7 +459,12 @@ def collection_lease(root, generation, installer, identity=None, initialize=Fals created = False if initialize and not record.exists() and not record.is_symlink(): # O_EXCL prevents adopting an old unrecorded inode after interruption. - descriptor = os.open(path, flags | os.O_CREAT | os.O_EXCL, 0o600) + try: + descriptor = os.open(path, flags | os.O_CREAT | os.O_EXCL, 0o600) + except FileExistsError: + raise installer.InstallError( + "An earlier run stopped while creating " + str(path) + ". Check that no process of the oac-node user still " + "uses this installation (`pgrep -u oac-node`), delete that file with sudo, then rerun the command.") from None created = True else: descriptor = os.open(path, flags) diff --git a/deploy/node/node_install.py b/deploy/node/node_install.py index 5cda1e2fc..6777a33dc 100644 --- a/deploy/node/node_install.py +++ b/deploy/node/node_install.py @@ -67,6 +67,9 @@ class RuntimeDownloadError(InstallError): CHILD_DOCKER_CONFIG = Path("/run/oac-node-docker") LOCK_WAIT_SECONDS = 600 NOTHING_CHANGED = " Nothing was changed." +DISK_FULL = "The disk is full. Free space under " + str(SERVICE_HOME) + ", then rerun the command." +# Prefixes failures of an installation; uninstall and generation helpers print theirs bare. +FAILED = "Installation failed: " def origin(value): @@ -98,6 +101,15 @@ def checked(arguments, failure, explain=None, **kwargs): "prerequisites; keep its state and rerun the command to recover.") # The node program's rejection line: Core's status and fixed error code, nothing else. REJECTION = re.compile(rb"node enrollment rejected \(HTTP (\d{3})(?: ([a-z_]{1,64}))?\)") +# The node program's failed local check before it contacts Core; only its fixed class code is shown. +READINESS_CHECK = re.compile(rb"local provider readiness check failed \(([a-z_]{1,64})\)") +READINESS_FAILURES = { + "provider_unavailable": "The sandbox provider does not answer on this host, for example because Docker is stopped.", + "host_unsupported": "This host lacks a capability the sandbox provider needs, such as Docker CPU and memory limits or read-write access to /dev/kvm.", + "capacity_insufficient": "This host is too small for one sandbox.", + "runtime_image_unavailable": "The sandbox provider does not have the Runtime image.", + "artifacts_unavailable": "A sandbox provider file is missing or fails its checksum.", +} class AddressChanged(InstallError): @@ -108,21 +120,23 @@ def registration_failure(stderr): """A fixed message for Core's answer to registration; never the node program's own text.""" matches = list(REJECTION.finditer(stderr or b"")) if not matches: + failed = READINESS_CHECK.search(stderr or b"") + if failed and failed.group(1).decode() in READINESS_FAILURES: + return InstallError(READINESS_FAILURES[failed.group(1).decode()] + " Fix it, then rerun the command; the token was not used.") return InstallError(REGISTRATION_UNCONFIRMED) status, code = matches[-1].group(1).decode(), (matches[-1].group(2) or b"").decode() if code == "sandbox_node_address_mismatch": return AddressChanged(node_spec.PUBLIC_URL_CHANGED + " The token was not used; downloaded files are kept.") if status == "401": - return InstallError("The enrollment command expired or was already used. Generate a new command on the Nodes " - "page and run it on this host; downloaded files are kept.") + return InstallError(node_spec.ENROLLMENT_EXPIRED + " Downloaded files are kept.") return InstallError(REGISTRATION_UNCONFIRMED + " Core answered HTTP " + status + (" " + code if code else "") + ".") def discard_unregistered(root): - """Remove the files that name the old address, so a new command can register this host. + """Remove what an earlier attempt decided, so a new command can register this host. Only for a node Core never recorded: downloads, the image and the service file stay.""" - for name in ("installation.json", "provider.json"): + for name in ("installation.json", "provider.json", "runtime-artifacts.json", "preparation.json"): if existing_file(root / name): (root / name).unlink() if (root / "state/node").is_dir() and not (root / "state/node").is_symlink(): @@ -159,8 +173,12 @@ def fetch(source, name): try: with open_request(source + "/node-install/" + name) as response: raw = response.read(1024 * 1024 + 1) + # A body cut short by the connection reads as a normal end; the declared length tells. + declared = getattr(response, "headers", {}).get("Content-Length", "") if len(raw) > 1024 * 1024: raise RuntimeDownloadError("Node bootstrap metadata is too large: " + name) + if declared.isdigit() and int(declared) != len(raw): + raise http.client.IncompleteRead(raw, int(declared) - len(raw)) return io.BytesIO(raw) except (urllib.error.URLError, TimeoutError, ConnectionError, http.client.IncompleteRead) as error: if not transient(error) or attempt == 2: @@ -169,15 +187,8 @@ def fetch(source, name): time.sleep(attempt + 1) -def metadata(source, bundle=None, prefix=""): - if bundle is not None: - def read(name): - path = bundle / name - if path.is_symlink() or not path.is_file() or not path.resolve().is_relative_to(bundle): - raise InstallError("Invalid local distribution metadata") - return path.open("rb") - else: - read = lambda name: fetch(source, prefix + name) +def metadata(source, prefix=""): + read = lambda name: fetch(source, prefix + name) with read("SHA256SUMS") as response: raw = response.read(1024 * 1024 + 1) if len(raw) > 1024 * 1024: @@ -206,7 +217,7 @@ def read(name): for name in dict.fromkeys(item["path"] for items in provider_assets.CATALOG.values() for item in items if item["role"] != "policy"): distribution.artifact(manifest, name) # Metadata stays on the console; artifact requests may redirect to its pinned release. - manifest["artifact_base_url"] = source + "/node-install/releases/" + manifest["source_commit"] + "/artifacts" if source else "" + manifest["artifact_base_url"] = source + "/node-install/releases/" + manifest["source_commit"] + "/artifacts" return manifest, sums @@ -232,7 +243,8 @@ def write_once(path, value): with distribution.temporary_file(path) as temporary: if existing_file(path): if path.read_text() != value: - raise InstallError("Existing node configuration differs; preserve its state and use the upgrade guide") + raise InstallError("Existing node configuration differs from Core's. This node is registered, so its state is kept; " + "remove the node on the Nodes page, uninstall it on this host, then add the host again with Add node.") return descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600) with os.fdopen(descriptor, "w") as stream: @@ -254,13 +266,15 @@ def file_digest(path): return result.hexdigest() -def download(source, name, root, expected, prefix=""): +def download(source, name, root, expected, prefix="", replace=False): target = root / name safe_directory(target.parent) if existing_file(target): - if file_digest(target) != expected: + if file_digest(target) == expected: + return + if not replace: raise RuntimeDownloadError("Installed node payload differs; refusing to overwrite it") - return + target.unlink() with distribution.temporary_file(target) as temporary: with os.fdopen(os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), "wb") as output, fetch(source, prefix + name) as response: for block in iter(lambda: response.read(1024 * 1024), b""): @@ -347,11 +361,33 @@ def provider_config(root, args, runtime_image): return result +def drop_runtime_archives(root): + """The imported image is the cache; the archives only hold disk.""" + for name in ("images/runtime.tar", "images/runtime.tar.gz"): + (root / name).unlink(missing_ok=True) + + +def micro_env(root, args): + return dict(os.environ, MSB_BACKEND="local", MSB_HOME=str(getattr(args, "runtime_home", micro_home(args.installation_id))), + MSB_PATH=str(root / MICRO[1]), MSB_LIBKRUNFW_PATH=str(root / MICRO[2])) + + +def micro_image_ready(root, args, manifest): + inspect = [str(root / MICRO[1]), "image", "inspect", manifest["runtime_ref"], "--format", "json"] + try: + value = json.loads(checked(inspect, "Runtime image is not installed", env=micro_env(root, args))) + return (value.get("digest") == manifest["runtime_ref"].split("@", 1)[1] + and value.get("architecture") == "amd64" and value.get("os") == "linux") + except (InstallError, ValueError, AttributeError): + return False + + def prepare_runtime(root, args, manifest): if args.provider == "docker": docker = ["docker", "--host", "unix:///var/run/docker.sock"] image = distribution.ensure_docker_image( - manifest, "runtime", lambda: distribution.runtime_archive(manifest, root, getattr(args, "bundle", None)), docker) + manifest, "runtime", lambda: distribution.runtime_archive(manifest, root, getattr(args, "disposable", False)), docker) + drop_runtime_archives(root) network = "oac-node-" + args.installation_id networks = checked(docker + ["network", "ls", "--format", "{{.Name}}"], "Cannot inspect Docker networks").splitlines() if network not in networks: @@ -363,22 +399,14 @@ def prepare_runtime(root, args, manifest): output = result.stdout.decode() if "not found" in output or (result.returncode and "statically linked" not in output and "not a dynamic executable" not in output): raise InstallError("Install the microsandbox host shared-library prerequisites") - env = dict(os.environ, MSB_BACKEND="local", MSB_HOME=str(getattr(args, "runtime_home", micro_home(args.installation_id))), - MSB_PATH=str(root / MICRO[1]), MSB_LIBKRUNFW_PATH=str(root / MICRO[2])) - inspect = [str(root / MICRO[1]), "image", "inspect", manifest["runtime_ref"], "--format", "json"] - def matches(): - try: - value = json.loads(checked(inspect, "Runtime image is not installed", env=env)) - return (value.get("digest") == manifest["runtime_ref"].split("@", 1)[1] - and value.get("architecture") == "amd64" and value.get("os") == "linux") - except (InstallError, ValueError, AttributeError): - return False - if not matches(): - archive = distribution.runtime_archive(manifest, root, getattr(args, "bundle", None)) + if not micro_image_ready(root, args, manifest): + archive = distribution.runtime_archive(manifest, root, getattr(args, "disposable", False)) checked([str(root / MICRO[1]), "image", "load", "--input", str(archive), "--tag", manifest["runtime_ref"], "--quiet"], - "Cannot import the microsandbox runtime image; check free disk space and host libraries", timeout=1800, env=env) - if not matches(): + "Cannot import the microsandbox runtime image; check free disk space and host libraries", timeout=1800, + env=micro_env(root, args)) + if not micro_image_ready(root, args, manifest): raise InstallError("Imported microsandbox runtime image identity or platform differs") + drop_runtime_archives(root) def unit_name(installation_id): @@ -389,7 +417,17 @@ def configure_node(root, args, token): """Read the Core specification and check the host under the installation lock.""" identity_file = root / "state/node/identity.json" retained = json.loads(identity_file.read_text()) if existing_file(identity_file) else None - args.configuration = node_spec.fetch(args, token, retained, open_request, allow_enrollment=not (root / "registered.json").exists()) + registered = (root / "registered.json").exists() + try: + args.configuration = node_spec.fetch(args, token, retained, open_request, allow_enrollment=not registered) + except node_spec.UnrecordedIdentity: + retained = None + args.configuration = node_spec.fetch(args, token, None, open_request, allow_enrollment=True) + # Core records a node only when its registration completes. Until then the earlier attempt's + # files bind nobody, whatever configuration or release it was made for. + args.disposable = not registered and retained is None + if args.disposable: + discard_unregistered(root) args.provider = args.configuration["provider"] preflight(args.provider) if args.provider != "microsandbox" and getattr(args, "checkpoint_root", None) is not None: @@ -417,25 +455,74 @@ def install_lock(root): try: fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB) except BlockingIOError: - raise InstallError("Another node installation is running") from None + raise InstallError("Another node installation is running, or this node is preparing a new sandbox configuration; " + "retry once the Nodes page no longer shows Preparing target.") from None yield +NEWER_NODE_SOFTWARE = ("This host's node program differs from the one this console serves, usually because Core was upgraded " + "after the node was added. Nothing was changed. To move the node to this release, choose Remove node on " + "the Nodes page, uninstall it on this host, then add the host again with Add node.") + + +def refuse_newer_node_software(root, manifest, names): + """A registered node keeps its own program; a rerun cannot swap it for another release's.""" + for name in names: + target = root / name + if existing_file(target) and not distribution.matches(target, distribution.artifact(manifest, name)): + raise InstallError(NEWER_NODE_SOFTWARE) + + +GIB = 1024 ** 3 + + +def require_disk(root, args, program_manifest, manifest, names): + """Refuse before downloading when the node account's disk cannot hold what is still missing. + + The Runtime archive and its unpacked copy exist only until the image is imported; a + microsandbox store then keeps about one more unpacked image under the same disk.""" + needed = 0 + for name in names: + if name != "runtime/seccomp.json": + entry = distribution.artifact(program_manifest if name in provider_assets.artifacts(args.provider, ("node",)) else manifest, name) + target = root / name + if not (target.is_file() and distribution.matches(target, entry)): + needed += entry["size"] + if args.provider == "docker": + imported = distribution.docker_image(manifest, "runtime", DOCKER) is not None + else: + imported = (root / MICRO[1]).is_file() and micro_image_ready(root, args, manifest) + if not imported: + archive = distribution.artifact(manifest, "images/runtime.tar.gz") + if not (root / "images/runtime.tar.gz").is_file(): + needed += archive["size"] + if not (root / "images/runtime.tar").is_file(): + needed += archive["unpacked_size"] + if args.provider == "microsandbox": + needed += archive["unpacked_size"] + free = shutil.disk_usage(root).free + if free < needed: + raise InstallError("Not enough disk space under %s: this installation needs about %.1f GB free and %.1f GB is available. " + "Free space, then rerun the command." % (Path.home(), needed / GIB, free / GIB)) + + def register_node(root, args, token, helper_archive=None, *, secret_path): """Download and verify the payload, prepare the Runtime and register; not the service.""" install_display.step("Downloading and verifying node files") - program_manifest, program_sums = metadata(args.source_url, getattr(args, "bundle", None)) + program_manifest, program_sums = metadata(args.source_url) selected = args.configuration["specification"]["runtime"] manifest, sums = program_manifest, program_sums if selected["source_commit"] != program_manifest["source_commit"]: - if getattr(args, "bundle", None) is not None: - raise InstallError("This bundle does not contain Core's selected Runtime; install through the console --source-url that retains its release") manifest, sums = metadata(args.source_url, prefix="releases/" + selected["source_commit"] + "/") node_spec.verify_release(args.configuration, manifest) runtime_prefix = "releases/" + manifest["source_commit"] + "/" names = provider_assets.artifacts(args.provider, ("node", "runtime", "policy")) if "runtime/seccomp.json" not in sums: raise InstallError("The distribution is missing required node checksums") + node_names = provider_assets.artifacts(args.provider, ("node",)) + if (root / "registered.json").exists(): + refuse_newer_node_software(root, program_manifest, node_names) + require_disk(root, args, program_manifest, manifest, names) state = {"installation_id": args.installation_id, "provider": args.provider, "core_url": args.core_url, "source_commit": manifest["source_commit"], "generation": args.configuration["generation"], "specification_digest": args.configuration["specification_digest"]} @@ -443,19 +530,12 @@ def register_node(root, args, token, helper_archive=None, *, secret_path): node_generations.record_root_runtime(root, args, manifest, sums, sys.modules[__name__]) for name in names: if name == "runtime/seccomp.json": - if getattr(args, "bundle", None) is None: - download(args.source_url, name, root, sums[name], prefix=runtime_prefix) - else: - source = args.bundle / name - if source.is_symlink() or file_digest(source) != sums[name]: - raise InstallError("Local node payload checksum mismatch") - safe_directory((root / name).parent) - write_once(root / name, source.read_text()) + download(args.source_url, name, root, sums[name], prefix=runtime_prefix, replace=args.disposable) else: target = root / name safe_directory(target.parent) existing_file(target) - distribution.obtain_artifact(program_manifest if name in provider_assets.artifacts(args.provider, ("node",)) else manifest, name, target, getattr(args, "bundle", None)) + distribution.obtain_artifact(program_manifest if name in node_names else manifest, name, target, args.disposable) os.chmod(target, 0o700) node_generations.install_helper(root, args, sys.modules[__name__], helper_archive) safe_directory(root / "state/node") @@ -692,10 +772,13 @@ def service_child(account, function, arguments, parent, mask, read_ends, output_ function(*arguments) code = 0 except (InstallError, node_spec.SpecificationError, distribution.DistributionError) as error: - install_display.error(str(error)) + install_display.error(str(error), FAILED) except Exception as error: # noqa: BLE001 - the child must always report and exit - print("The step running as " + SERVICE_USER + " failed unexpectedly (" + type(error).__name__ - + "). Inspect the host, then rerun the command.", file=sys.stderr) + if isinstance(error, OSError) and error.errno == errno.ENOSPC: + install_display.error(DISK_FULL, FAILED) + else: + print("The step running as " + SERVICE_USER + " failed unexpectedly (" + type(error).__name__ + + "). Inspect the host, then rerun the command.", file=sys.stderr) finally: # A closed output pipe must not skip the exit below. for stream in (sys.stdout, sys.stderr): @@ -743,7 +826,8 @@ def host_lock(): def host_checks(): if sys.version_info < (3, 9) or platform.system() != "Linux" or platform.machine() not in ("x86_64", "amd64"): - raise InstallError("Run with Python 3.9+ on Linux amd64." + NOTHING_CHANGED) + raise InstallError("Python 3.9+ on Linux amd64 is required; this command ran %s (Python %d.%d, %s %s)." + % (sys.executable, *sys.version_info[:2], platform.system(), platform.machine()) + NOTHING_CHANGED) if not SYSTEMD_RUNNING.is_dir(): raise InstallError("systemd is not the init system here, so this host cannot run the node service." + NOTHING_CHANGED) for tool in ("systemctl", "useradd", "usermod", "userdel", "gpasswd"): @@ -903,6 +987,11 @@ def device_group(provider, device, gid): return name +def installation_containers(installation_id): + return checked(list(DOCKER) + ["ps", "--all", "--filter", "label=io.oac.installation=" + installation_id, + "--format", "{{.Names}}"], "Cannot inspect Docker containers").split() + + def other_node(args, provider): """Refuse a second sudo-mode Core on this host, or a second node for this installation. @@ -925,7 +1014,8 @@ def other_node(args, provider): + ". Preserve it and arrange cleanup separately before installing a system node." + NOTHING_CHANGED) if provider == "docker": networks = checked(list(DOCKER) + ["network", "ls", "--format", "{{.Name}}"], "Cannot inspect Docker networks").splitlines() - if "oac-node-" + args.installation_id in networks: + # A network that no container uses is a leftover (Docker was down during an uninstall); the install reuses it. + if "oac-node-" + args.installation_id in networks and installation_containers(args.installation_id): raise InstallError("Another node for this installation already uses this Docker engine (network oac-node-" + args.installation_id + "). Remove it on the Nodes page and uninstall it first." + NOTHING_CHANGED) @@ -1049,7 +1139,7 @@ def install_system(args, token): root_file(unit, system_unit(root, provider)) install_display.step("Starting the node service") checked(["systemctl", "daemon-reload"], "Cannot reload systemd") - checked(["systemctl", "enable", "--now", unit.name], "Cannot start the node service; retained identity is unchanged") + checked(["systemctl", "enable", "--now", unit.name], "Cannot start the node service; its identity is kept. Inspect sudo journalctl -u " + unit.name) checked(["systemctl", "is-active", "--quiet", unit.name], "Node service is unavailable; inspect sudo journalctl -u " + unit.name) install_display.step("Waiting for Core connection and provider readiness") run_as(account, wait_ready, root, args) @@ -1126,8 +1216,7 @@ def release_docker_network(installation_id): try: if network not in checked(list(DOCKER) + ["network", "ls", "--format", "{{.Name}}"], "Docker unavailable").splitlines(): return - remaining = checked(list(DOCKER) + ["ps", "--all", "--filter", "label=io.oac.installation=" + installation_id, - "--format", "{{.Names}}"], "Docker unavailable").split() + remaining = installation_containers(installation_id) except InstallError: print("Docker is unavailable, so network " + network + " was not removed.") return @@ -1149,7 +1238,7 @@ def remove_node_files(root, installation_id): shutil.rmtree(root) # The service user can write provider.json; print only what an image ID can be. if isinstance(image, str) and re.fullmatch(r"sha256:[0-9a-f]{64}", image): - print("Kept the Runtime image " + image + "; remove it with `docker image rm " + image + "` if no other node uses it.") + print("Kept the Runtime image " + image + "; remove it with `sudo docker image rm " + image + "` if no other node uses it.") if runtime_home.exists(): sudo_mode = str(Path.home()) == str(SERVICE_HOME) remove = ("sudo -u " + SERVICE_USER + " rm -rf " if sudo_mode else "rm -rf ") + str(runtime_home) @@ -1166,15 +1255,18 @@ def uninstall_system(args): record = node_record(args.installation_id) unit = SYSTEM_UNITS / unit_name(args.installation_id) # Only root-owned files decide whether a node is installed; the service home is not read here. + absent = "No node for installation " + args.installation_id + " has a system installation on this host. Nothing was changed." if record is None and not unit.exists() and not (SYSTEM_RECORDS / "account.json").exists(): - print("No node for installation " + args.installation_id + " has a system installation on this host. Nothing was changed.") + print(absent) return with host_lock(): record = node_record(args.installation_id) account, account_record = service_account(), read_root_json(SYSTEM_RECORDS / "account.json") root = SERVICE_HOME / ".oac/nodes" / args.installation_id if record is None and not unit.exists(): - print("No node for installation " + args.installation_id + " was installed with sudo on this host.") + # A rerun that only finishes the account; release_account says what it did. + if node_records(): + print(absent) else: if not ours(account, (account_record or {}).get("uid")): raise InstallError("The " + SERVICE_USER + " account is missing or differs from the one this installer " @@ -1291,9 +1383,7 @@ def read_token(args): def main(argv=None): parser = argparse.ArgumentParser(description=__doc__) - source = parser.add_mutually_exclusive_group() - source.add_argument("--source-url", type=origin) - source.add_argument("--bundle", type=Path) + parser.add_argument("--source-url", type=origin) parser.add_argument("--core-url", type=origin) parser.add_argument("--provider", choices=("docker", "microsandbox"), help="Optional assertion; Core owns provider selection") parser.add_argument("--installation-id", required=True) @@ -1302,18 +1392,22 @@ def main(argv=None): parser.add_argument("--generation-action", choices=("prepare", "collect"), help=argparse.SUPPRESS) parser.add_argument("--generation", type=int, help=argparse.SUPPRESS) parser.add_argument("--specification-digest", help=argparse.SUPPRESS) - parser.add_argument("--update", action="store_true", help=argparse.SUPPRESS) parser.add_argument("--uninstall", action="store_true", help="Remove this host's node after it was removed on the Nodes page") parser.add_argument("--force", action="store_true", help="With --uninstall: skip the Core check, for a Core that no longer exists") parser.add_argument("--no-color", action="store_true", help="Disable terminal colors") args = parser.parse_args(argv) if args.no_color: os.environ["NO_COLOR"] = "1" - if str(uuid.UUID(args.installation_id)) != args.installation_id: - raise InstallError("Installation ID must be a canonical UUID") - if args.update: - raise InstallError("Node version updates are not supported; preserve the existing state and reinstall " - "separately. Nothing was changed.") + global FAILED + if args.uninstall or args.generation_action: + FAILED = "" + try: + valid = str(uuid.UUID(args.installation_id)) == args.installation_id + except ValueError: + valid = False + if not valid: + raise InstallError("--installation-id must be the installation's UUID, as in the command from Add node or in the " + "file names under /etc/oac-node. Nothing was changed.") if args.generation_action: if args.generation is None or not 1 <= args.generation <= 9223372036854775807 or not re.fullmatch(r"[0-9a-f]{64}", args.specification_digest or ""): parser.error("Invalid generation authorization") @@ -1326,16 +1420,14 @@ def main(argv=None): if os.geteuid() != 0: raise InstallError("Node installation and removal require root. Run this command with sudo.") if args.uninstall: - if args.source_url or args.bundle or args.core_url or args.provider or args.enrollment_token_stdin or args.checkpoint_root: + if args.source_url or args.core_url or args.provider or args.enrollment_token_stdin or args.checkpoint_root: parser.error("--uninstall takes only --installation-id and --force") uninstall_system(args) return if args.force: parser.error("--force applies only to --uninstall") - if not (args.source_url or args.bundle) or not args.core_url: - parser.error("--source-url (or --bundle) and --core-url are required") - if args.bundle is not None and (not args.bundle.is_absolute() or args.bundle.resolve() != args.bundle): - raise InstallError("Local bundle must be an absolute directory without symlinks") + if not args.source_url or not args.core_url: + parser.error("--source-url and --core-url are required") token = read_token(args) if len(token) > 4096 or any(c.isspace() for c in token): raise InstallError("A valid one-time enrollment credential is required") @@ -1353,5 +1445,11 @@ def main(argv=None): print(INTERRUPTED, file=sys.stderr) sys.exit(130) except (InstallError, node_spec.SpecificationError, distribution.DistributionError, OSError, ValueError, KeyError, subprocess.SubprocessError) as error: - install_display.error(str(error) if isinstance(error, (InstallError, node_spec.SpecificationError, distribution.DistributionError)) else "check host prerequisites and retained private files") + if isinstance(error, (InstallError, node_spec.SpecificationError, distribution.DistributionError)): + message = str(error) + elif isinstance(error, OSError) and error.errno == errno.ENOSPC: + message = DISK_FULL + else: + message = "check host prerequisites and retained private files" + install_display.error(message, FAILED) sys.exit(1) diff --git a/deploy/node/node_spec.py b/deploy/node/node_spec.py index 678f5bc26..0a3425986 100644 --- a/deploy/node/node_spec.py +++ b/deploy/node/node_spec.py @@ -12,8 +12,14 @@ class SpecificationError(Exception): pass +class UnrecordedIdentity(SpecificationError): + """Core rejects the retained identity of a node that never registered, so it never recorded it.""" + + PUBLIC_URL_CHANGED = ("Core's public URL changed after this command was generated. Generate a new command on the " "Nodes page and run it on this host.") +ENROLLMENT_EXPIRED = ("The enrollment command expired or was already used. Generate a new command on the Nodes page " + "and run it on this host.") def release(manifest): @@ -128,9 +134,7 @@ def fetch(args, token, retained, open_request, allow_enrollment=False, generatio raise # Identity is persisted before enrollment. A failed first registration # may therefore have no durable credential on Core yet. - request = urllib.request.Request(args.core_url + "/api/v1/sandbox-node/configuration", - headers={"Authorization": "Bearer " + token}) - response = open_request(request) + raise UnrecordedIdentity("Core has no record of the retained node identity") from None with response: raw = response.read(16385) if len(raw) > 16384: @@ -145,7 +149,9 @@ def fetch(args, token, retained, open_request, allow_enrollment=False, generatio raise SpecificationError("Core node configuration was not found (HTTP 404); route /api/v1 on the Core origin directly to Core, not to Web") from None if error.code == 409: raise SpecificationError("Core refused node configuration (HTTP 409): the deployment is resetting or conflicts with this node's retained specification; inspect the deployment before retrying") from None - raise SpecificationError("Core rejected the node configuration read (HTTP " + str(error.code) + "); verify the retained or enrollment credential") from None + if error.code == 401: + raise SpecificationError(ENROLLMENT_EXPIRED) from None + raise SpecificationError("Core rejected the node configuration read (HTTP " + str(error.code) + "); check the Core URL and the reverse proxy in front of it, then rerun the command") from None except (urllib.error.URLError, TimeoutError, ConnectionError, http.client.IncompleteRead): raise SpecificationError("Cannot reach Core node configuration; verify the Core origin and that the reverse proxy routes /api/v1 to Core") from None except (ValueError, AttributeError): @@ -155,7 +161,8 @@ def fetch(args, token, retained, open_request, allow_enrollment=False, generatio if (identity.get("provider") != data["provider"] or not allow_selection_change and (identity.get("specification_digest") != data["specification_digest"] or identity.get("deployment_generation") != data["generation"])): - raise SpecificationError("Retained node specification differs from Core; preserve its state and follow the deployment change procedure") + raise SpecificationError("This node is registered and prepares the changed sandbox configuration itself; the Nodes page " + "shows its progress. Nothing was changed.") if generation is not None and data["generation"] != generation: raise SpecificationError("Core returned a different generation") return data diff --git a/deploy/node/test_distribution.py b/deploy/node/test_distribution.py index bee168ab9..a0fbf096b 100644 --- a/deploy/node/test_distribution.py +++ b/deploy/node/test_distribution.py @@ -131,21 +131,28 @@ def test_a_stalled_download_stops_and_removes_its_part(self): self.assertEqual(len(self.requests), 1) # Not retried as a network error. self.assertEqual(list(self.root.iterdir()), []) - def test_offline_and_runtime_expansion_are_verified(self): + def test_runtime_expansion_is_verified_and_a_differing_cache_is_refused_or_replaced(self): raw = b'synthetic tar contents' * 1000 self.data = gzip.compress(raw) entry = self.entry('images/runtime.tar.gz', self.data) entry.update(unpacked_size=len(raw), unpacked_sha256=hashlib.sha256(raw).hexdigest()) - offline = self.root / 'offline' - (offline / 'artifacts').mkdir(parents=True) - (offline / 'artifacts' / entry['filename']).write_bytes(self.data) - output = distribution.runtime_archive(self.manifest, self.root / 'cache', offline) + output = distribution.runtime_archive(self.manifest, self.root / 'cache') self.assertEqual(output.read_bytes(), raw) - self.assertEqual(self.requests, []) self.assertEqual(distribution.runtime_archive(self.manifest, self.root / 'cache'), output) + self.assertEqual(len(self.requests), 1) output.write_bytes(b'bad cache') with self.assertRaisesRegex(distribution.DistributionError, 'differs'): distribution.runtime_archive(self.manifest, self.root / 'cache') + self.assertEqual(distribution.runtime_archive(self.manifest, self.root / 'cache', replace=True).read_bytes(), raw) + + def test_a_differing_cached_artifact_is_refused_unless_the_caller_replaces_it(self): + target = self.root / 'node' + target.write_bytes(b'other release') + with self.assertRaisesRegex(distribution.ArtifactError, 'Cached artifact differs'): + distribution.obtain_artifact(self.manifest, 'native/bin/node', target) + self.assertEqual(target.read_bytes(), b'other release') + distribution.obtain_artifact(self.manifest, 'native/bin/node', target, replace=True) + self.assertEqual(target.read_bytes(), self.data) def test_retry_after_sigkill_discards_unfinished_artifact(self): target = self.root / 'node' diff --git a/deploy/node/test_generation_review_regressions.py b/deploy/node/test_generation_review_regressions.py index f160c2386..82eb11515 100644 --- a/deploy/node/test_generation_review_regressions.py +++ b/deploy/node/test_generation_review_regressions.py @@ -116,7 +116,7 @@ def test_interrupted_lease_initialization_is_not_reclaimed(self): with self.assertRaises(OSError): with node_generations.collection_lease(case.root, 3, installer, identity, initialize=True): self.fail('initialization not durable') - with self.assertRaises(FileExistsError): + with self.assertRaisesRegex(installer.InstallError, 'earlier run stopped while creating.*delete that file'): with node_generations.collection_lease(case.root, 3, installer, identity, initialize=True): self.fail('interrupted inode adopted') (case.directory / '1.lease-identity').unlink() @@ -273,13 +273,3 @@ def available(value,installer): final=installer.private_json(case.root/'state/node/generations/2.json') self.assertNotEqual(final['native']['helper_path'],original['native']['helper_path']) self.assertIn(1,node_generations.retained_configs(case.root,installer)) - - def test_operator_update_refuses_before_download_or_update(self): - result = subprocess.run( - [sys.executable, 'node_install.py', '--installation-id', '94be54a1-138c-4f30-bc87-b13686272dbe', - '--update', '--source-url', 'https://core.invalid', '--core-url', 'https://core.invalid'], - capture_output=True, text=True, timeout=10, cwd=Path(__file__).parent, - ) - self.assertEqual(result.returncode, 1, result.stderr) - self.assertIn('updates are not supported', result.stderr) - self.assertIn('reinstall', result.stderr) diff --git a/deploy/node/test_node_helper_transfer.py b/deploy/node/test_node_helper_transfer.py index 14516c94b..f0a7a92aa 100644 --- a/deploy/node/test_node_helper_transfer.py +++ b/deploy/node/test_node_helper_transfer.py @@ -21,8 +21,8 @@ def test_captured_archive_survives_removal_of_private_source(self): with tempfile.TemporaryDirectory() as directory: root = Path(directory) source = root / "node-install.pyz" - source.write_bytes(node_generations.helper_archive(SimpleNamespace(bundle=None), installer)) - args = SimpleNamespace(bundle=None, source_url="https://core.example", core_url="https://core.example") + source.write_bytes(node_generations.helper_archive(SimpleNamespace(), installer)) + args = SimpleNamespace(source_url="https://core.example", core_url="https://core.example") with mock.patch.object(sys, "argv", [str(source)]): captured = node_generations.helper_archive(args, installer) source.unlink() @@ -45,9 +45,9 @@ def test_real_service_uid_cannot_read_private_download_but_can_install_snapshot( uid = gid = 65534 os.chown(root, uid, gid) source = download / "node-install.pyz" - source.write_bytes(node_generations.helper_archive(SimpleNamespace(bundle=None), installer)) + source.write_bytes(node_generations.helper_archive(SimpleNamespace(), installer)) source.chmod(0o600) - args = SimpleNamespace(bundle=None, source_url="https://core.example", core_url="https://core.example") + args = SimpleNamespace(source_url="https://core.example", core_url="https://core.example") with mock.patch.object(sys, "argv", [str(source)]): captured = node_generations.helper_archive(args, installer) pid = os.fork() diff --git a/deploy/node/test_node_install.py b/deploy/node/test_node_install.py index aada90152..0d52ec773 100644 --- a/deploy/node/test_node_install.py +++ b/deploy/node/test_node_install.py @@ -2,6 +2,7 @@ import argparse import contextlib import copy +import errno import fcntl import hashlib import gzip @@ -11,6 +12,7 @@ from pathlib import Path import signal import socket +import shutil import stat import subprocess import sys @@ -24,6 +26,8 @@ import node_install as installer import node_spec +REAL_FETCH = installer.fetch + def ended(pid, wait=5): @@ -67,6 +71,8 @@ def setUp(self): self.fail_service = False self.fail_registration = False self.register_stderr = None + self.cpus = 3 + self.core_knows_node = True for patch in (mock.patch.object(installer.Path, "home", return_value=self.home), mock.patch.object(installer, "preflight"), mock.patch.object(installer, "wait_ready"), @@ -81,7 +87,10 @@ def setUp(self): self.addCleanup(patch.stop) def configuration_response(self, request, **kwargs): - resources = {"cpus": 3, "memory_mib": 6144} + # Core answers 401 to the credential of a node it never recorded. + if not self.core_knows_node and request.headers.get("Authorization") == "Bearer " + "a" * 64: + raise urllib.error.HTTPError(request.full_url, 401, "", {}, None) + resources = {"cpus": self.cpus, "memory_mib": 6144} if self.args.provider == "microsandbox": resources.update(root_disk_mib=10240, environment_disk_mib=12288) spec = {"resources": resources, "runtime": node_spec.release(self.manifest)} @@ -211,18 +220,6 @@ def test_current_program_pairs_with_core_selected_retained_docker_runtime(self): def test_current_program_pairs_with_core_selected_retained_micro_runtime(self): self.install_current_program_with_retained_runtime("microsandbox") - def test_bundle_without_selected_runtime_refuses_before_payload_or_registration(self): - program = copy.deepcopy(self.manifest) - program["source_commit"] = "f" * 40 - self.args.bundle = self.home / "bundle" - self.args.bundle.mkdir() - with mock.patch.object(installer, "metadata", return_value=(program, {})): - with self.assertRaisesRegex(installer.InstallError, "does not contain Core's selected Runtime"): - self.install() - self.assertFalse((self.root / "installation.json").exists()) - self.assertFalse((self.root / installer.provider_assets.artifacts("docker", ("node",))[0]).exists()) - self.assertFalse(any("register" in command or "load" in command or "enable" in command for command, _ in self.calls)) - def test_missing_retained_runtime_never_falls_back_to_current_runtime(self): program = copy.deepcopy(self.manifest) program["source_commit"] = "f" * 40 @@ -494,7 +491,7 @@ def test_microsandbox_imports_image_and_allows_only_explicit_private_core_endpoi def test_repeat_preserves_registration_and_recovers_service_start_failure(self): system = self.sudo_host() self.fail_service = True - with self.assertRaisesRegex(installer.InstallError, "Cannot start"): + with self.assertRaisesRegex(installer.InstallError, "Cannot start.*sudo journalctl -u oac-node-"): installer.install_system(self.args, "synthetic-once-token") saved = (self.root / "registered.json").read_bytes() self.calls.clear() @@ -626,12 +623,12 @@ def test_different_commit_provider_or_core_cannot_overwrite_retained_installatio for field, value in (("provider", "microsandbox"), ("core_url", "https://other.example")): before = getattr(self.args, field) setattr(self.args, field, value) - with self.assertRaisesRegex((installer.InstallError, node_spec.SpecificationError), "differs"): + with self.assertRaisesRegex((installer.InstallError, node_spec.SpecificationError), "differs|prepares the changed"): self.install() setattr(self.args, field, before) self.manifest["source_commit"] = "f" * 40 self.refresh_manifest() - with self.assertRaisesRegex((installer.InstallError, node_spec.SpecificationError), "differs"): + with self.assertRaisesRegex((installer.InstallError, node_spec.SpecificationError), "differs|prepares the changed"): self.install() self.assertEqual((self.root / "provider.json").read_bytes(), original) @@ -643,14 +640,14 @@ def test_corrupt_manifest_and_payload_fail_before_provider_use(self): self.payloads[installer.provider_assets.artifacts("docker", ("node",))[0]] += b"corrupt" with self.assertRaisesRegex(installer.distribution.DistributionError, "published size|checksum"): self.install() - self.assertFalse(self.calls) + self.assertFalse([call for call, _ in self.calls if "inspect" not in call]) self.assertFalse((self.root / installer.provider_assets.artifacts("docker", ("node",))[0]).exists()) def test_installed_payload_and_config_are_not_overwritten(self): self.install() target = self.root / installer.provider_assets.artifacts("docker", ("node",))[0] target.write_bytes(b"existing-different-payload") - with self.assertRaisesRegex(installer.distribution.DistributionError, "Cached artifact differs"): + with self.assertRaisesRegex(installer.InstallError, "Remove node.*Add node"): self.install() self.assertEqual(target.read_bytes(), b"existing-different-payload") @@ -963,6 +960,12 @@ def test_uninstall_prints_only_an_image_id_from_the_service_home(self): with mock.patch.object(installer.sys, "stdout", output): installer.remove_node_files(root, self.args.installation_id) self.assertNotIn("Finish", output.getvalue()) + root.mkdir() + (root / "provider.json").write_text(json.dumps({"native": {"image": "sha256:" + "b" * 64}})) + output = io.StringIO() + with mock.patch.object(installer.sys, "stdout", output): + installer.remove_node_files(root, self.args.installation_id) + self.assertIn("`sudo docker image rm sha256:", output.getvalue()) def test_service_user_step_gets_its_own_session_keyring(self): libc = installer.ctypes.CDLL(None, use_errno=True) @@ -1060,24 +1063,6 @@ def test_token_comes_on_standard_input_only(self): installer.main(arguments) self.assertEqual(install.call_args.args[1], "synthetic-once-token") - def test_offline_bundle_uses_same_bootstrap_and_verified_artifacts(self): - bundle = self.home / "bundle" - bundle.mkdir() - for name in ("manifest.json", "SHA256SUMS", "runtime/seccomp.json"): - target = bundle / name - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(self.payloads[name]) - for name, entry in self.manifest["artifacts"].items(): - target = bundle / "artifacts" / entry["filename"] - target.parent.mkdir(exist_ok=True) - target.write_bytes(self.payloads[name]) - self.args.bundle = bundle - self.args.source_url = None - with mock.patch.object(installer, "fetch", side_effect=AssertionError("Unexpected metadata download")), \ - mock.patch.object(installer.distribution.urllib.request, "build_opener", side_effect=AssertionError("Unexpected artifact download")): - self.install() - self.assertEqual(json.loads((self.root / "provider.json").read_text())["specification"], self.args.configuration["specification"]) - def test_origin_rejects_other_schemes_credentials_paths_and_redirects(self): for value in ("ftp://core.example", "https://user@core.example", "https://@core.example", "https://core.example/v1", "https://core.example?", "https://core.example#", "https://core.example\\path", "https://core.example:bad", ""): with self.subTest(value=value), self.assertRaises(argparse.ArgumentTypeError): @@ -1088,6 +1073,113 @@ def test_origin_rejects_other_schemes_credentials_paths_and_redirects(self): installer.NoRedirect().redirect_request(None, None, 302, "", {}, "https://other.example") + def test_an_install_that_never_registered_continues_after_the_configuration_and_release_changed(self): + self.fail_registration = True + with self.assertRaises(installer.InstallError): + self.install() + self.fail_registration = False + self.core_knows_node = False + program = installer.provider_assets.artifacts("docker", ("node",))[0] + self.payloads[program] = b"program of the upgraded Core" + self.cpus = 4 + self.refresh_manifest() + self.install() + self.assertTrue((self.root / "registered.json").exists()) + self.assertEqual((self.root / program).read_bytes(), self.payloads[program]) + self.assertEqual(json.loads((self.root / "provider.json").read_text())["specification"]["resources"]["cpus"], 4) + + def test_a_registered_node_keeps_its_state_when_the_configuration_changes(self): + self.install() + saved = (self.root / "provider.json").read_bytes() + self.cpus = 4 + with self.assertRaisesRegex(node_spec.SpecificationError, "registered and prepares the changed sandbox configuration"): + self.install() + self.assertEqual((self.root / "provider.json").read_bytes(), saved) + + def test_a_registered_node_given_a_newer_release_says_to_remove_and_add_it_again(self): + self.install() + program = self.root / installer.provider_assets.artifacts("docker", ("node",))[0] + saved = program.read_bytes() + self.payloads[program.relative_to(self.root).as_posix()] = b"program of the upgraded Core" + self.refresh_manifest() + with self.assertRaisesRegex(installer.InstallError, "Remove node on the Nodes page.*Add node"): + self.install() + self.assertEqual(program.read_bytes(), saved) + + def test_too_little_disk_is_refused_before_anything_downloads(self): + with mock.patch.object(installer.shutil, "disk_usage", return_value=SimpleNamespace(free=0)): + with self.assertRaisesRegex(installer.InstallError, "Not enough disk space under .*[Ff]ree space, then rerun"): + self.install() + self.assertFalse((self.root / "installation.json").exists()) + self.assertFalse((self.root / installer.provider_assets.artifacts("docker", ("node",))[0]).exists()) + + def test_a_full_disk_in_the_service_step_has_its_own_message(self): + def write(): + raise OSError(errno.ENOSPC, "No space left on device") + errors = io.StringIO() + with self.assertRaises(installer.ChildFailed): + self.service_step(write, errors=errors) + self.assertIn("The disk is full. Free space under", errors.getvalue()) + self.assertNotIn("OSError", errors.getvalue()) + + def test_the_runtime_archives_are_deleted_once_the_image_is_imported(self): + for provider in ("docker", "microsandbox"): + with self.subTest(provider=provider): + self.args.provider = provider + self.install() + self.assertEqual(list(self.root.glob("images/*")), []) + shutil.rmtree(self.root) + + def test_a_failed_local_provider_check_names_its_class_not_the_nodes_text(self): + message = str(installer.registration_failure(b"x: local provider readiness check failed (host_unsupported): /home/secret")) + self.assertIn("/dev/kvm", message) + self.assertIn("the token was not used", message) + self.assertNotIn("secret", message) + + def test_a_leftover_docker_network_without_containers_does_not_block_adding_the_node(self): + network = "oac-node-" + self.args.installation_id + containers = [""] + def docker(arguments, failure, **kwargs): + if "ps" in arguments: + return containers[0] + return network if "network" in arguments else "" + with mock.patch.object(installer, "checked", side_effect=docker), mock.patch.object(installer, "node_records", return_value=[]): + installer.other_node(self.args, "docker") + containers[0] = "oac-sandbox-1" + with self.assertRaisesRegex(installer.InstallError, "already uses this Docker engine"): + installer.other_node(self.args, "docker") + + def test_a_metadata_body_cut_short_is_retried(self): + class Cut(io.BytesIO): + headers = {"Content-Length": "10"} + bodies = [Cut(b"123"), Response(b"1234567890")] + bodies[1].headers = {"Content-Length": "10"} + with mock.patch.object(installer, "open_request", side_effect=bodies), mock.patch.object(installer.time, "sleep"): + self.assertEqual(REAL_FETCH("https://console.example", "SHA256SUMS").read(), b"1234567890") + + def test_a_busy_install_lock_mentions_a_configuration_in_preparation(self): + self.root.mkdir(parents=True) + with installer.install_lock(self.root): + with self.assertRaisesRegex(installer.InstallError, "preparing a new sandbox configuration"): + with installer.install_lock(self.root): + pass + + def test_a_python_too_old_for_the_installer_is_named(self): + with mock.patch.object(installer.platform, "system", return_value="Darwin"): + with self.assertRaisesRegex(installer.InstallError, "ran " + installer.sys.executable.replace("\\", "\\\\")): + installer.host_checks() + + def test_an_installation_id_that_is_not_a_uuid_says_where_to_find_it(self): + with self.assertRaisesRegex(installer.InstallError, "--installation-id must be the installation's UUID"): + installer.main(["--installation-id", "", "--uninstall"]) + + def test_uninstall_and_helper_errors_have_no_installation_prefix(self): + errors = io.StringIO() + with mock.patch.object(installer.sys, "stderr", errors): + installer.install_display.error("Core still lists this node.", "") + self.assertEqual(errors.getvalue(), "Core still lists this node.\n") + + class NodePrerequisiteTests(unittest.TestCase): def test_preflight_rejects_missing_kvm_before_downloads(self): with mock.patch.object(installer.platform, "system", return_value="Linux"), \ @@ -1145,11 +1237,5 @@ def test_foreign_contents_and_symlink_are_not_adopted(self): installer.prepare_checkpoint_root(root) -class UnsupportedNodeUpdateTests(unittest.TestCase): - def test_update_refuses_without_host_operations(self): - with self.assertRaisesRegex(installer.InstallError, "not supported;.*reinstall"): - installer.main(["--installation-id", "00000000-0000-0000-0000-000000000001", "--update"]) - - if __name__ == "__main__": unittest.main() diff --git a/deploy/node/test_node_spec.py b/deploy/node/test_node_spec.py index ef6c7458f..3e3846c59 100644 --- a/deploy/node/test_node_spec.py +++ b/deploy/node/test_node_spec.py @@ -44,17 +44,17 @@ def test_repeat_uses_retained_credentials_not_a_replacement_token(self): self.assertEqual(headers["Authorization"], "Bearer " + self.retained["credential"]) self.assertEqual(headers["X-oac-node-id"], self.retained["identity"]["node_id"]) - def test_partial_registration_can_use_enrollment_after_unauthenticated_identity(self): + def test_a_retained_identity_core_never_recorded_is_reported_so_it_can_be_discarded(self): rejected = urllib.error.HTTPError("https://core.example", 401, "private details", {}, None) - opener = mock.Mock(side_effect=[rejected, self.response()]) - node_spec.fetch(self.args, "once", self.retained, opener, allow_enrollment=True) - self.assertEqual(opener.call_count, 2) - self.assertEqual(dict(opener.call_args.args[0].header_items()), {"Authorization": "Bearer once"}) + opener = mock.Mock(side_effect=rejected) + with self.assertRaises(node_spec.UnrecordedIdentity): + node_spec.fetch(self.args, "once", self.retained, opener, allow_enrollment=True) + self.assertEqual(opener.call_count, 1) + # A registered node, or a rerun without a token, has nothing to fall back to. for allowed, token in ((False, "once"), (True, "")): - opener = mock.Mock(side_effect=rejected) - with self.assertRaises(node_spec.SpecificationError): - node_spec.fetch(self.args, token, self.retained, opener, allow_enrollment=allowed) - self.assertEqual(opener.call_count, 1) + with self.assertRaises(node_spec.SpecificationError) as raised: + node_spec.fetch(self.args, token, self.retained, mock.Mock(side_effect=rejected), allow_enrollment=allowed) + self.assertNotIsInstance(raised.exception, node_spec.UnrecordedIdentity) def test_changed_public_url_says_to_generate_a_new_command(self): self.data["core_url"] = "https://core-new.example" @@ -64,7 +64,8 @@ def test_changed_public_url_says_to_generate_a_new_command(self): def test_failures_name_their_cause(self): for failure, message in ((urllib.error.HTTPError("https://core.example", 404, "", {}, None), "route /api/v1"), (urllib.error.HTTPError("https://core.example", 409, "", {}, None), "resetting"), - (urllib.error.HTTPError("https://core.example", 401, "", {}, None), "credential"), + (urllib.error.HTTPError("https://core.example", 401, "", {}, None), "Generate a new command"), + (urllib.error.HTTPError("https://core.example", 502, "", {}, None), "reverse proxy in front of it"), (urllib.error.URLError("refused"), "reverse proxy routes /api/v1")): with self.assertRaisesRegex(node_spec.SpecificationError, message): node_spec.fetch(self.args, "once", None, mock.Mock(side_effect=failure)) @@ -74,7 +75,7 @@ def test_changed_generation_or_specification_rejects_retained_node(self): original = copy.deepcopy(self.data) change() self.data["specification_digest"] = node_spec.digest("docker", self.data["specification"]) - with self.assertRaisesRegex(node_spec.SpecificationError, "Retained node specification differs"): + with self.assertRaisesRegex(node_spec.SpecificationError, "prepares the changed sandbox configuration itself"): node_spec.fetch(self.args, "once", self.retained, mock.Mock(return_value=self.response())) self.data = original self.spec = self.data["specification"] diff --git a/docs/getting-started/nodes.md b/docs/getting-started/nodes.md index ac2834646..b49697b8f 100644 --- a/docs/getting-started/nodes.md +++ b/docs/getting-started/nodes.md @@ -47,7 +47,7 @@ The installer shows each phase as it runs and, once Core confirms the node, a su - SELinux not enforcing. The installer does not support hosts with enforcing SELinux. - Docker: rootful Docker Engine running, its socket `/var/run/docker.sock` owned by the `docker` group with mode `0660`, enforcing CPU and memory limits (cgroup v2). - microsandbox: `/dev/kvm` in the `kvm` group (hardware or nested virtualization), and the libraries microsandbox links (glibc). -- CPUs and memory for at least one sandbox of the installation's size, and about 2 GB of disk for the Runtime image. +- CPUs and memory for at least one sandbox of the installation's size, and about 3 GB of free disk while the Runtime image installs: its download and unpacked copy sit under `/var/lib/oac-node` until the import finishes, and the image takes about 1.5 GB in Docker's storage or the microsandbox store. The installer checks the space before it downloads and deletes the archives after the import, so the node keeps the image and under 0.2 GB of programs. - Access to the console and Core at the public URL; sandboxes reach Core too. ### Download through a proxy @@ -84,11 +84,12 @@ Root only prepares the account, the group and the unit; everything else, the Doc ## Rerun, expiry and slow links -- Rerunning the same command is safe. Once the node is registered, a rerun uses the node's own credential, changes nothing that already matches and needs no token. It refuses, without changing anything, when the node's retained identity belongs to another Core address or installation, and with `Retained node specification differs from Core` once the sandbox configuration has changed since the node registered. A node added with this command prepares new configurations itself; see [Change the sandbox configuration](#change-the-sandbox-configuration). -- A command that already expired, or was used on another host, fails at once with `Core rejected the node configuration read (HTTP 401)`: generate a new command and run it within 10 minutes. +- Rerunning the same command is safe. Once the node is registered, a rerun uses the node's own credential, changes nothing that already matches and needs no token. It changes nothing and says why when the node's retained identity belongs to another Core address or installation, when the sandbox configuration has changed since the node registered (the node prepares it itself; see [Change the sandbox configuration](#change-the-sandbox-configuration)), or when Core was upgraded and serves a newer node program: choose **Remove node** on **Nodes**, uninstall the node on the host, then add it again with **Add node**. +- A run that fails before the node registers leaves nothing that Core knows. The next command, even after a configuration change or a Core upgrade, replaces the earlier attempt's configuration and any downloaded file that no longer matches, keeps the files that match and continues. +- A command that already expired, or was used on another host, fails at once with `The enrollment command expired or was already used`: generate a new command and run it within 10 minutes. - If the command expires during a slow download, registration fails with `The enrollment command expired or was already used`. The downloaded files are kept: generate a new command in Web and run it. - Failed or interrupted downloads discard their temporary files. A rerun downloads missing files from the beginning and reuses verified complete files. A transfer that brings less than 64 KiB in a minute stops; check the network and retry. -- The Docker Runtime image is about 500 MB. On a slow link, load it first: copy the release's `oac--linux-amd64-runtime.tar.gz` asset to the host and run `sudo docker load -i` on it. The installer then finds the exact image and skips the download. +- The Docker Runtime image is about 400 MB. On a slow link, load it first: copy the release's `oac--linux-amd64-runtime.tar.gz` asset to the host and run `sudo docker load -i` on it. The installer then finds the exact image and skips the download. - Interrupting the installer, or closing its terminal, stops it; run the command again to continue. ## Logs @@ -199,7 +200,12 @@ A registered node that can't connect to Core says why in its [log](#logs): `sand | Message | Fix | | --- | --- | -| `Core rejected the node configuration read (HTTP 401)` before anything downloads, or `The enrollment command expired or was already used` | Generate a new command in Web and run it within 10 minutes | +| `The enrollment command expired or was already used` | Generate a new command in Web and run it within 10 minutes | +| `Core rejected the node configuration read (HTTP …)` | Check the Core URL and the reverse proxy in front of it, then rerun | +| `Not enough disk space under …` or `The disk is full` | Free the space it names, then rerun; nothing was downloaded | +| This host's node program differs from the one this console serves | Core was upgraded after the node was added. Choose **Remove node**, uninstall it on the host, then add it again with **Add node** | +| Another node installation is running, or this node is preparing a new sandbox configuration | Retry once **Nodes** no longer shows **Preparing target** | +| A sandbox provider check failed before registering, such as `This host lacks a capability …` | Fix what it names (see [Readiness codes](#readiness-codes)) and rerun; the token was not used | | Core's public URL changed after this command was generated | Generate a new command in Web and run it | | This host's node uses `
`, but this command uses `
` | The node was added under an older public URL. Remove it in Web, uninstall it, then add it again | | Docker Engine is not installed, or Docker is not running | Install Docker Engine, or `sudo systemctl enable --now docker`, then rerun | diff --git a/docs/zh/getting-started/nodes.md b/docs/zh/getting-started/nodes.md index aa8d9a02c..3e98f4309 100644 --- a/docs/zh/getting-started/nodes.md +++ b/docs/zh/getting-started/nodes.md @@ -1,7 +1,7 @@ --- title: "添加和管理节点" source: docs/getting-started/nodes.md -source_hash: 8d0888dcd75b63daeb6bcd1207bcd006606f3f843ce786e5427a6c092b1b3b55 +source_hash: da770b5195c28cc9ffa6e2d374113adfb2b30a3526710386ff7c49ce8ac62a94 --- 节点是一台 Linux 主机，在沙箱提供商为 microsandbox（推荐）或 Docker 时，为 Core 托管 Session 运行沙箱。Core 将新 Session 分配给有空余容量的节点；节点创建沙箱，沙箱回连 Core。E2B 不需要节点。应用为自己的 Session 连接的机器是[自托管执行器](self-hosted.md)，而不是节点。 @@ -49,7 +49,7 @@ Microsandbox 的检查点执行兼容性包含主机内核和 CPU profile。因 - SELinux 不处于 enforcing 模式。安装程序不支持 enforcing SELinux 主机。 - Docker：正在运行的 rootful Docker Engine，其 `/var/run/docker.sock` 套接字属于 `docker` 组，权限为 `0660`，并强制执行 CPU 和内存限制（cgroup v2）。 - microsandbox：`/dev/kvm` 属于 `kvm` 组（硬件或嵌套虚拟化），并具有 microsandbox 链接的库（glibc）。 -- CPU 和内存至少足以运行一个所配置规格的沙箱，以及约 2 GB 的 Runtime 镜像磁盘空间。 +- CPU 和内存至少足以运行一个所配置规格的沙箱；安装 Runtime 镜像期间约需 3 GB 空闲磁盘：下载的归档及其解包副本在导入完成前位于 `/var/lib/oac-node`，镜像本身在 Docker 存储或 microsandbox 存储中约占 1.5 GB。安装程序在下载前检查空间，并在导入后删除归档，因此节点最终只保留镜像和不足 0.2 GB 的程序。 - 可通过公开 URL 访问控制台和 Core；沙箱也能访问 Core。 ### 通过代理下载 {#download-through-a-proxy} @@ -86,11 +86,12 @@ root 只准备账号、组和服务单元；其他操作（包括 Docker 网络 ## 重新运行、过期与慢速网络 {#rerun-expiry-and-slow-links} -- 重新执行同一命令是安全的。节点注册后，重新运行使用节点自身凭据，不改变已匹配的内容，也无需令牌。保留的节点身份属于其他 Core 地址或安装时，程序拒绝且不修改任何内容；节点注册后沙箱配置发生变化时，程序同样不做修改，并以 `Retained node specification differs from Core` 拒绝。通过此命令添加的节点会自行准备新配置；参阅[修改沙箱配置](#change-the-sandbox-configuration)。 -- 已过期或已在其他主机使用的命令立即失败，显示 `Core rejected the node configuration read (HTTP 401)`：生成新命令并在 10 分钟内执行。 +- 重新执行同一命令是安全的。节点注册后，重新运行使用节点自身凭据，不改变已匹配的内容，也无需令牌。以下情况程序不做任何修改并说明原因：保留的节点身份属于其他 Core 地址或安装；节点注册后沙箱配置发生变化（节点会自行准备，参阅[修改沙箱配置](#change-the-sandbox-configuration)）；或 Core 已升级、提供了更新的节点程序，此时在 **节点** 选择 **移除节点**，在主机上卸载该节点，再通过 **添加节点** 重新添加。 +- 节点注册前失败的运行不会在 Core 留下任何记录。下一条命令即使在配置变更或 Core 升级之后执行，也会替换上次尝试的配置和不再匹配的已下载文件，保留匹配的文件并继续。 +- 已过期或已在其他主机使用的命令立即失败，显示 `The enrollment command expired or was already used`：生成新命令并在 10 分钟内执行。 - 慢速下载期间命令过期时，注册失败并显示 `The enrollment command expired or was already used`。下载文件保留：在 Web 生成新命令后执行。 - 失败或中断时删除下载残片；重新执行时从头下载缺失文件，复用已校验的完整文件。每分钟下载不足 64 KiB 时会停止，请检查网络后重试。 -- Docker Runtime 镜像约 500 MB。网络较慢时可提前加载：将发行资产 `oac--linux-amd64-runtime.tar.gz` 复制到主机并运行 `sudo docker load -i`。安装程序发现精确匹配的镜像后跳过下载。 +- Docker Runtime 镜像约 400 MB。网络较慢时可提前加载：将发行资产 `oac--linux-amd64-runtime.tar.gz` 复制到主机并运行 `sudo docker load -i`。安装程序发现精确匹配的镜像后跳过下载。 - 中断安装程序或关闭终端会停止程序；重新执行命令继续。 ## 日志 {#logs} @@ -201,7 +202,12 @@ root 只准备账号、组和服务单元；其他操作（包括 Docker 网络 | 消息 | 解决方法 | | --- | --- | -| 下载前出现 `Core rejected the node configuration read (HTTP 401)`，或 `The enrollment command expired or was already used` | 在 Web 生成新命令并在 10 分钟内执行 | +| `The enrollment command expired or was already used` | 在 Web 生成新命令并在 10 分钟内执行 | +| `Core rejected the node configuration read (HTTP …)` | 检查 Core URL 及其前面的反向代理后重试 | +| `Not enough disk space under …` 或 `The disk is full` | 按提示释放空间后重试；尚未下载任何内容 | +| This host's node program differs from the one this console serves | 节点添加后 Core 已升级。选择 **移除节点**，在主机上卸载，再通过 **添加节点** 重新添加 | +| Another node installation is running, or this node is preparing a new sandbox configuration | 待 **节点** 不再显示 **正在准备目标配置** 后重试 | +| 注册前的沙箱提供商检查失败，例如 `This host lacks a capability …` | 修复提示的问题（参阅[就绪状态码](#readiness-codes)）后重试；令牌未被使用 | | Core's public URL changed after this command was generated | 在 Web 生成并执行新命令 | | This host's node uses `
`, but this command uses `
` | 节点在旧公开 URL 下添加。在 Web 移除、卸载，然后重新添加 | | Docker Engine is not installed, or Docker is not running | 安装 Docker Engine，或运行 `sudo systemctl enable --now docker` 后重试 |