Recover node installs that never registered

apps/web/e2e/nodes.spec.ts

@@ -19,7 +19,7 @@ test("adds a node: host requirements, a root/sudo command, a countdown, the same
   await expect(add.getByText("What the sandbox backend needs on the host; the installer checks it and names anything missing")).toBeVisible();
   await expect(add.getByText("SELinux is not enforcing")).toBeVisible();
   await expect(add.getByText("In sudo mode, one Core per host: a host already running a sudo-mode node for another Core is refused.")).toBeVisible();
-  await expect(add.getByText("CPUs and memory for at least one sandbox: 2 CPU · 4 GiB; about 2 GB of disk for the Runtime image")).toBeVisible();
+  await expect(add.getByText("CPUs and memory for at least one sandbox: 2 CPU · 4 GiB; about 3 GB of free disk while the Runtime image installs")).toBeVisible();
   await expect(add.getByText("Reaches https://core.example.com, as do its sandboxes")).toBeVisible();
   // Node installation only offers a system service; there is no user-mode alternative.
   await expect(add.getByText("No sudo on this host?")).toHaveCount(0);

apps/web/src/features/sandbox/enrollment-command.test.ts

@@ -1,7 +1,7 @@
 import { describe, expect, it } from "vitest";
 import { spawnSync } from "node:child_process";
 import { createHash } from "node:crypto";
-import { chmodSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from "node:fs";
+import { chmodSync, mkdirSync, mkdtempSync, readFileSync, readdirSync, rmSync, symlinkSync, writeFileSync } from "node:fs";
 import { homedir } from "node:os";
 import { join } from "node:path";
 import { nodeInstallCommand, nodeLogCommand, nodeUninstallCommand } from "./enrollment-command";
@@ -12,7 +12,7 @@ describe("sandbox connection and enrollment", () => {
     expect(nodeLogCommand("a b")).toBe("sudo journalctl -u 'oac-node-a b.service'");
   });
   // "as root": a root shell runs the sudo command without sudo.
-  it.each(["success", "download failure", "checksum mismatch", "installer failure", "as root", "no color", "uppercase proxy", "lowercase proxy", "empty proxy", "root proxy", "unsupported platform", "old python", "uninstall", "killed download"])("executes safely, passes the token only on stdin and cleans private downloads after %s", (scenario) => {
+  it.each(["success", "download failure", "checksum mismatch", "installer failure", "as root", "no color", "uppercase proxy", "lowercase proxy", "empty proxy", "root proxy", "unsupported platform", "old python", "linked directory", "uninstall", "killed download"])("executes safely, passes the token only on stdin and cleans private downloads after %s", (scenario) => {
     const parent = join(homedir(), ".oac", "tests");
     mkdirSync(parent, { recursive: true });
     const root = mkdtempSync(join(parent, "node-command-"));
@@ -63,7 +63,15 @@ process.exit(process.env.SCENARIO==='installer failure'?7:0);`);
       if (scenario === "lowercase proxy") Object.assign(proxyEnv, { http_proxy: proxy, https_proxy: proxy, no_proxy: "core.example,localhost", HTTP_PROXY: "http://wrong.invalid" });
       if (scenario === "empty proxy") proxyEnv.http_proxy = "";
       const options = { env: { ...process.env, ...proxyEnv, HOME: root, NO_COLOR: scenario === "no color" ? "" : undefined, PATH: `${bin}:${process.env.PATH}`, TMPDIR: temporary, SCENARIO: scenario, FIXTURE: fixture, REPORT: report, SUDO_REPORT: sudoReport, PRINTF_REPORT: printfReport }, encoding: "utf8" as const };
+      if (scenario === "linked directory") { mkdirSync(join(root, ".oac")); symlinkSync(temporary, join(root, ".oac/node-bootstrap")); }
       let result = spawnSync("sh", ["-c", command], options);
+      if (scenario === "linked directory") {
+        expect(result.status).toBe(1);
+        expect(result.stderr).toContain(`${join(root, ".oac/node-bootstrap")} is a symbolic link`);
+        expect(result.stdout).not.toContain("Downloading");
+        expect(readdirSync(temporary)).toEqual([]);
+        return;
+      }
       if (scenario === "killed download") {
         expect(result.signal === "SIGKILL" || result.status === 137).toBe(true);
         expect(readdirSync(join(root, ".oac/node-bootstrap"))).toContain("download.partial");

apps/web/src/features/sandbox/enrollment-command.ts

@@ -15,9 +15,12 @@ for tool in curl sha256sum python3 flock; do command -v "$tool" >/dev/null || {
 python3 -c 'import sys; sys.exit(sys.version_info < (3, 9))' || { printf 'Python 3.9 or newer is required.\\n' >&2; exit 1; }
 [ "$(id -u)" -eq 0 ] || command -v sudo >/dev/null || { printf 'sudo is required; ask the host administrator to run this command.\\n' >&2; exit 1; }
 d="$HOME/.oac/node-bootstrap"
-[ ! -L "$d" ] && mkdir -p "$d" && [ -d "$d" ] && [ -O "$d" ] && chmod 700 "$d" || { printf 'Cannot use the private node bootstrap directory.\\n' >&2; exit 1; }
+[ ! -L "$d" ] || { printf '%s is a symbolic link; remove it and rerun the command.\\n' "$d" >&2; exit 1; }
+mkdir -p "$d" || { printf 'Cannot create %s. A root-owned ~/.oac usually means this command ran under sudo; run it as your own user.\\n' "$d" >&2; exit 1; }
+{ [ -d "$d" ] && [ -O "$d" ]; } || { printf '%s must be a directory you own. If root owns it, remove it with sudo and run the command as your own user.\\n' "$d" >&2; exit 1; }
+chmod 700 "$d" || exit
 installer="$d"/${quote(scriptDigest + ".pyz")}
-for f in "$d/download.lock" "$d/download.partial" "$installer"; do [ ! -L "$f" ] && { [ ! -e "$f" ] || { [ -f "$f" ] && [ -O "$f" ]; }; } || { printf 'Invalid node bootstrap file.\\n' >&2; exit 1; }; done
+for f in "$d/download.lock" "$d/download.partial" "$installer"; do [ ! -L "$f" ] && { [ ! -e "$f" ] || { [ -f "$f" ] && [ -O "$f" ]; }; } || { printf '%s must be a regular file you own, not a link.\\n' "$f" >&2; exit 1; }; done
 exec 9>>"$d/download.lock"
 flock -n 9 || { printf 'Another node download is running; wait and retry.\\n' >&2; exit 1; }
 rm -f "$d/download.partial" || exit
@@ -26,7 +29,7 @@ s=; [ "$(id -u)" -eq 0 ] || s=sudo
 export http_proxy="\${http_proxy-\${HTTP_PROXY-}}" https_proxy="\${https_proxy-\${HTTPS_PROXY-}}" no_proxy="\${no_proxy-\${NO_PROXY-}}"
 export HTTP_PROXY="$http_proxy" HTTPS_PROXY="$https_proxy" NO_PROXY="$no_proxy"
 printf '\\n==> Downloading node installer...\\n' &&
-curl -fs --connect-timeout 15 --max-time 60 --retry 2 --retry-connrefused --retry-delay 1 --max-filesize 1048576 ${quote(sourceUrl + "/node-install/node-install.pyz")} -o "$d/download.partial" || { c=$?; printf 'Cannot download node installer; check the console URL, TLS and proxy settings.\\n' >&2; exit "$c"; }
+curl -fsS --connect-timeout 15 --max-time 60 --retry 2 --retry-connrefused --retry-delay 1 --max-filesize 1048576 ${quote(sourceUrl + "/node-install/node-install.pyz")} -o "$d/download.partial" || { c=$?; printf 'Cannot download node installer; check the console URL, TLS and proxy settings.\\n' >&2; exit "$c"; }
 printf '==> Verifying node installer...\\n' &&
 printf '%s  %s\\n' ${quote(scriptDigest)} "$d/download.partial" | sha256sum -c --status || { printf 'Node installer checksum mismatch; generate a fresh command in Web and retry.\\n' >&2; exit 1; }
 mv "$d/download.partial" "$installer" || exit

apps/web/src/features/sandbox/node-enrollment.ts

@@ -28,7 +28,7 @@ export interface HostPrerequisite {
  *   and install nothing;
  * - `host_capacity`: the host's CPUs and memory hold one sandbox of the
  *   deployment's size (else the node reports capacity_insufficient); the Runtime
- *   image needs about 2 GB of disk;
+ *   image needs about 3 GB of free disk while it installs (`require_disk`);
  * - network: node files and artifacts only from the console at the public URL
  *   (`fetch` and `metadata`, which never use a release URL), Core's /api/v1 at the
  *   same URL (node_spec.py, the `register` call), and sandboxes reach Core as well
@@ -41,7 +41,7 @@ export function hostRequirements(sized: boolean): HostPrerequisite[] {
     { label: "SELinux is not enforcing" },
     { label: "In sudo mode, one Core per host: a host already running a sudo-mode node for another Core is refused." },
     { label: "What the sandbox backend needs on the host; the installer checks it and names anything missing" },
-    { label: sized ? "CPUs and memory for at least one sandbox: {{size}}; about 2 GB of disk for the Runtime image" : "CPUs and memory for at least one sandbox; about 2 GB of disk for the Runtime image" },
+    { label: sized ? "CPUs and memory for at least one sandbox: {{size}}; about 3 GB of free disk while the Runtime image installs" : "CPUs and memory for at least one sandbox; about 3 GB of free disk while the Runtime image installs" },
     { label: "Reaches {{core}}, as do its sandboxes" },
   ];
 }

apps/web/src/i18n/locales/zh-CN/sandbox.ts

@@ -156,8 +156,8 @@ export const sandbox = {
   "{{name}} still points at the old Core address {{address}}.": "{{name}} 仍指向旧的 Core 地址 {{address}}。",
   "If this node's old Core address no longer responds, first remove it on the Nodes page, then add --force to the uninstall command.": "如果这个节点的旧 Core 地址已无法访问，请先在“节点”页移除它，再在卸载命令后加上 --force。",
   "Uninstall command with --force": "带 --force 的卸载命令",
-  "CPUs and memory for at least one sandbox: {{size}}; about 2 GB of disk for the Runtime image": "CPU 和内存至少够一个沙箱：{{size}}；Runtime 镜像约需 2 GB 磁盘",
-  "CPUs and memory for at least one sandbox; about 2 GB of disk for the Runtime image": "CPU 和内存至少够一个沙箱；Runtime 镜像约需 2 GB 磁盘",
+  "CPUs and memory for at least one sandbox: {{size}}; about 3 GB of free disk while the Runtime image installs": "CPU 和内存至少够一个沙箱：{{size}}；安装 Runtime 镜像期间约需 3 GB 空闲磁盘",
+  "CPUs and memory for at least one sandbox; about 3 GB of free disk while the Runtime image installs": "CPU 和内存至少够一个沙箱；安装 Runtime 镜像期间约需 3 GB 空闲磁盘",
   "Reaches {{core}}, as do its sandboxes": "能访问 {{core}}，它的沙箱也要能访问",
   "The command creates the oac-node service user and a system service. It installs no software; if something is missing it stops and says what to install.": "命令会创建 oac-node 服务用户和一个系统服务。它不安装任何软件；缺少什么时会停下并说明要装什么。",
   "Set a public address other machines can reach before adding nodes.": "添加节点前，请先设置其他机器能访问的公开地址。",

deploy/node/README.md

@@ -29,6 +29,10 @@ The node installer runs as root and prepares the host for one node per installat
 
 The generated command uses `~/.oac/node-bootstrap` in the invoking account's home. Under `download.lock`, it discards an unfinished `download.partial`, downloads and verifies the bootstrap, then publishes `<sha256>.pyz`. The lock is released before invoking sudo; verified files stay available to running installers. Node file writes share `distribution.temporary_file`: under the caller's installation lock, remove the previous temporary file, write privately, and publish atomically. The same lifecycle covers payloads, extraction, helpers, settings and enrollment tokens. Registration tokens are removed even when a rerun finds the node already registered.
 
+Core records a node when its registration completes, and the installer marks that with `registered.json`. Until then, an earlier attempt's configuration, identity and cached files that no longer match are disposable: the next command replaces them and keeps the files that match. A registered node's files are never replaced; a rerun that finds a different node program stops and names the remove, uninstall and add steps.
+
+Before downloading, the installer refuses when the node account's disk lacks the space the missing files and the Runtime import need. It deletes the Runtime archive and its unpacked copy once the image is imported.
+
 Configuration files are published only after their complete contents are written; a retry preserves an existing matching file and refuses conflicting contents. Generation leases are durable locks, not disposable temporary files: interruption between creating a lease and recording its inode identity requires operator inspection. A retry refuses that unrecorded lease instead of replacing an inode that a Runtime helper could still hold.
 
 The distribution manifest is the one download contract for nodes: flat versioned file names, and the compressed and unpacked size and SHA-256 of the Runtime.

deploy/node/distribution.py

@@ -44,30 +44,30 @@ def docker_command(arguments, timeout=30):
         raise DistributionError('Cannot inspect or load the distribution image; check Docker access and disk space') from None
 
 
+def docker_image(manifest, name, docker=('docker',)):
+    """The local ID of the manifest's image when Docker already holds it, else None."""
+    expected = image_identities(manifest, name)
+    for identity in dict.fromkeys(expected):
+        result = docker_command(list(docker) + ['image', 'inspect', identity, '--format',
+                                                '{{.Id}} {{.Os}}/{{.Architecture}}'])
+        if result.returncode:
+            continue
+        fields = result.stdout.strip().split()
+        if len(fields) != 2 or fields[0] not in expected or fields[1] != 'linux/amd64':
+            raise DistributionError('Docker image identity or platform differs from the distribution: ' + name)
+        return fields[0]
+    return None
+
+
 def ensure_docker_image(manifest, name, archive, docker=('docker',)):
     """Resolve a proven local ID; obtain a verified archive only on a cache miss."""
-    expected = image_identities(manifest, name)
-    docker = list(docker)
-
-    def inspect():
-        for identity in dict.fromkeys(expected):
-            result = docker_command(docker + ['image', 'inspect', identity, '--format',
-                                             '{{.Id}} {{.Os}}/{{.Architecture}}'])
-            if result.returncode:
-                continue
-            fields = result.stdout.strip().split()
-            if len(fields) != 2 or fields[0] not in expected or fields[1] != 'linux/amd64':
-                raise DistributionError('Docker image identity or platform differs from the distribution: ' + name)
-            return fields[0]
-        return None
-
-    identity = inspect()
+    identity = docker_image(manifest, name, docker)
     if identity is not None:
         return identity
-    result = docker_command(docker + ['load', '--input', str(archive())], timeout=1800)
+    result = docker_command(list(docker) + ['load', '--input', str(archive())], timeout=1800)
     if result.returncode:
         raise DistributionError('Cannot load the distribution image; check Docker access and free disk space: ' + name)
-    identity = inspect()
+    identity = docker_image(manifest, name, docker)
     if identity is None:
         raise DistributionError('Cannot verify the loaded distribution image: ' + name)
     return identity
@@ -139,25 +139,22 @@ def matches(path, entry):
     return path.stat().st_size == entry['size'] and digest(path) == entry['sha256']
 
 
-def obtain_artifact(manifest, logical_path, destination, offline_root=None):
+def obtain_artifact(manifest, logical_path, destination, replace=False):
+    """Download a verified artifact unless the destination already holds it.
+
+    A destination with other bytes is refused, or replaced when the caller owns
+    no use of it (replace)."""
     entry = artifact(manifest, logical_path)
     target = checked_path(destination)
     if target.exists():
-        if not matches(target, entry):
+        if matches(target, entry):
+            return target
+        if not replace:
             raise ArtifactError('Cached artifact differs; preserve the installation and inspect: ' + logical_path)
-        return target
-    source = None
-    if offline_root is not None:
-        candidate = Path(offline_root) / 'artifacts' / entry['filename']
-        if candidate.exists():
-            if candidate.is_symlink() or not candidate.is_file():
-                raise DistributionError('Offline artifact must be a regular file: ' + logical_path)
-            source = candidate
-    if source is not None:
-        return copy_artifact(source, target, entry, logical_path)
+        target.unlink()
     base = manifest.get('artifact_base_url', '')
     if not isinstance(base, str) or not base or urlsplit(base).query:
-        raise ArtifactError('No downloadable artifact source; use the matching offline bundle')
+        raise ArtifactError('No downloadable artifact source: ' + logical_path)
     url = safe_url(base.rstrip('/') + '/' + entry['filename'])
     with temporary_file(target) as partial:
         for attempt in range(3):
@@ -192,22 +189,6 @@ def temporary_file(target):
         partial.unlink(missing_ok=True)
 
 
-def copy_artifact(source, target, entry, logical_path):
-    with temporary_file(target) as temporary:
-        with os.fdopen(os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), 'wb') as output, source.open('rb') as stream:
-            count = 0
-            for block in iter(lambda: stream.read(1024 * 1024), b''):
-                count += len(block)
-                if count > entry['size']:
-                    raise ArtifactError('Artifact exceeds published size: ' + logical_path)
-                output.write(block)
-        if count != entry['size'] or digest(temporary) != entry['sha256']:
-            raise ArtifactError('Artifact checksum mismatch: ' + logical_path)
-        os.chmod(temporary, 0o700 if logical_path.startswith('native/') else 0o600)
-        os.replace(temporary, target)
-        return target
-
-
 # Stop downloads that bring fewer bytes than this in a window.
 SLOW_SECONDS, SLOW_BYTES = 60, 64 * 1024
 
@@ -247,7 +228,8 @@ def download_artifact(url, partial, entry, logical_path):
             raise http.client.IncompleteRead(b'', size - count)
 
 
-def runtime_archive(manifest, cache_root, offline_root=None):
+def runtime_archive(manifest, cache_root, replace=False):
+    """The verified, unpacked Runtime archive; replace as in obtain_artifact."""
     entry = artifact(manifest, 'images/runtime.tar.gz')
     expanded = {'sha256': entry.get('unpacked_sha256'), 'size': entry.get('unpacked_size')}
     if (not re.fullmatch(r'[0-9a-f]{64}', str(expanded['sha256']))
@@ -256,10 +238,12 @@ def runtime_archive(manifest, cache_root, offline_root=None):
     root = Path(cache_root)
     target = checked_path(root / 'images/runtime.tar')
     if target.exists():
-        if not matches(target, expanded):
+        if matches(target, expanded):
+            return target
+        if not replace:
             raise ArtifactError('Cached Runtime archive differs; preserve state and inspect it')
-        return target
-    archive = obtain_artifact(manifest, 'images/runtime.tar.gz', root / 'images/runtime.tar.gz', offline_root)
+        target.unlink()
+    archive = obtain_artifact(manifest, 'images/runtime.tar.gz', root / 'images/runtime.tar.gz', replace)
     with temporary_file(target) as temporary:
         try:
             with os.fdopen(os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600), 'wb') as output, gzip.open(archive, 'rb') as stream:

deploy/node/install_display.py

@@ -52,8 +52,8 @@ def heading(message):
     print("\n" + color(message, "1"))
 
 
-def error(message):
-    print(color("Installation failed: ", "31", sys.stderr) + message, file=sys.stderr, flush=True)
+def error(message, label="Installation failed: "):
+    print((color(label, "31", sys.stderr) if label else "") + message, file=sys.stderr, flush=True)
 
 
 def paragraph(message):