diff --git a/apps/web/e2e/fixture-console.mjs b/apps/web/e2e/fixture-console.mjs index 5cd5b3f44..e662bc01a 100644 --- a/apps/web/e2e/fixture-console.mjs +++ b/apps/web/e2e/fixture-console.mjs @@ -436,14 +436,14 @@ async function sandboxRoute(request, response, path, url) { if (path === "/nodes") return send(response, 200, { data: state.nodes }); if (path === "/enrollment-tokens" && request.method === "POST") { if (state.deployment.reset) return error(response, 409, "A sandbox reset is in progress.", "sandbox_reset_in_progress"); - // As Core: a one-time token valid for ten minutes, whose limits and enrollment ID the node it enrolls takes; + // As Core: a one-time token valid for one hour, whose limits and enrollment ID the node it enrolls takes; // only microsandbox keeps a retained limit above the active one. const input = await body(request); const maxActive = input.max_active ?? 1; const serial = state.nextId++; const enrollmentId = `00000000-0000-4000-8000-${String(serial).padStart(12, "0")}`; state.enrollment = { max_active: maxActive, max_retained: state.deployment?.provider === "microsandbox" ? input.max_retained ?? maxActive : maxActive, enrollment_id: enrollmentId }; - return send(response, 201, { token: `enroll_fixture_${serial}`, expires_at: new Date(Date.now() + 10 * 60_000).toISOString(), enrollment_id: enrollmentId }); + return send(response, 201, { token: `enroll_fixture_${serial}`, expires_at: new Date(Date.now() + 60 * 60_000).toISOString(), enrollment_id: enrollmentId }); } let m; if ((m = path.match(/^\/nodes\/([^/]+)\/allocations$/))) return send(response, 200, { data: state.allocations.filter((entry) => entry.node_id === m[1]) }); diff --git a/apps/web/e2e/nodes.spec.ts b/apps/web/e2e/nodes.spec.ts index df9a1230b..cb1c8f3e5 100644 --- a/apps/web/e2e/nodes.spec.ts +++ b/apps/web/e2e/nodes.spec.ts @@ -6,11 +6,11 @@ test.afterEach(async ({ request }) => expectManagementBoundary(request)); test("adds a node: host requirements, a root/sudo command, a countdown, the same command after closing, a new one after expiry, then its own node's registration", async ({ page, request }) => { await page.clock.install(); - // Core counts a token's ten minutes on its own clock; the page's clock stands in for it, so fast-forwarding expires a command. + // Core counts a token's hour on its own clock; the page's clock stands in for it, so fast-forwarding expires a command. await page.route("**/core/v1/sandbox/enrollment-tokens", async (route) => { const response = await route.fetch(); const now = await page.evaluate(() => Date.now()); - await route.fulfill({ response, json: { ...await response.json(), expires_at: new Date(now + 10 * 60_000).toISOString() } }); + await route.fulfill({ response, json: { ...await response.json(), expires_at: new Date(now + 60 * 60_000).toISOString() } }); }); await openConsole(page, request, "nodes"); await page.getByRole("button", { name: "Add node" }).click(); @@ -39,7 +39,7 @@ test("adds a node: host requirements, a root/sudo command, a countdown, the same await expect(field).toHaveValue(/curl [^\n]* 'https:\/\/core\.example\.com\/node-install\/node-install\.pyz' /); await expect(field).toHaveValue(/ --source-url 'https:\/\/core\.example\.com' --core-url 'https:\/\/core\.example\.com' /); await expect(add.getByText("If the command is interrupted or the download stalls, run it again: unfinished downloads restart, and verified files are reused.")).toBeVisible(); - await expect(add.getByRole("timer")).toHaveText(/^Expires in (10:00|9:\d\d)$/); + await expect(add.getByRole("timer")).toHaveText(/^Expires in (1:00:00|59:\d\d)$/); const progress = add.getByRole("status", { name: "Registration progress" }); await expect(progress).toHaveText(/Waiting for registration.*Connect.*Docker check/); @@ -55,7 +55,7 @@ test("adds a node: host requirements, a root/sudo command, a countdown, the same await expect(progress).toHaveText(/Waiting for registration.*Connect.*Docker check/); // Once expired, a new command is issued only on request, for the same limits. - await page.clock.fastForward("10:30"); + await page.clock.fastForward("01:00:30"); await expect(progress).toContainText("Command expired"); await expect(field).toBeHidden(); const reissued = tokenRequest(); @@ -69,7 +69,7 @@ test("adds a node: host requirements, a root/sudo command, a countdown, the same await add.getByRole("button", { name: "Close dialog" }).click(); await expect(add).toBeHidden(); await setNode(request, { id: "node-new", name: "edge-04" }); - await page.clock.fastForward("10:30"); + await page.clock.fastForward("01:00:30"); // Every render from the reopening on is watched, so even a brief "Command expired" would count. await page.evaluate(() => { const watch = window as unknown as { expiredShown: boolean; expiredWatch: MutationObserver }; diff --git a/deploy/node/README.md b/deploy/node/README.md index 79c5edac3..883810251 100644 --- a/deploy/node/README.md +++ b/deploy/node/README.md @@ -40,7 +40,7 @@ The distribution manifest is the one download contract for nodes: flat versioned - A Compose installation keeps only the node metadata from its release archive; Core's image never acquires execution-only payloads. - A node obtains bootstrap metadata from the console that generated its command. Web serves artifacts it has locally and redirects missing declared execution artifacts to the versioned HTTPS release base in the verified manifest. Web never downloads or caches those bytes. - Only artifact requests may follow HTTPS redirects, and only without credentials or cookies. Metadata and enrollment requests stay on the configured console. The console publishes only fixed non-secret files and declared artifact names. -- Download into private temporary files, verify size and SHA-256 before an atomic rename, clear leftover temporary downloads and extracted files before retrying, and reuse only verified cache entries or exact image identities. Never select a release other than the pinned one. +- Download into private temporary files, verify size and SHA-256 before an atomic rename, resume a leftover partial artifact download with an HTTP range (a server that cannot resume, or bytes that fail verification, restart the file) and clear leftover extracted files before retrying, and reuse only verified cache entries or exact image identities. Never select a release other than the pinned one. - Release downloads are anonymous. Never add repository credentials to installed node or Runtime configuration. - Manual builds use the `build-` release tag and tag builds the `v*` tag. The manifest's download base must match the release tag; artifact file names and source provenance keep the full source SHA. diff --git a/deploy/node/distribution.py b/deploy/node/distribution.py index ef59978db..9d23504e2 100644 --- a/deploy/node/distribution.py +++ b/deploy/node/distribution.py @@ -114,8 +114,9 @@ def redirect_request(self, request, fp, code, msg, headers, newurl): safe_url(newurl) if urlsplit(newurl).scheme != 'https' or request.get_method() not in ('GET', 'HEAD'): raise ArtifactError('Artifact redirects require HTTPS') - # Artifact redirects carry no credentials, cookies or custom headers. - return urllib.request.Request(newurl, method=request.get_method(), unverifiable=True) + # Artifact redirects carry no credentials, cookies or custom headers; only a resume range follows. + headers = {'Range': request.get_header('Range')} if request.has_header('Range') else {} + return urllib.request.Request(newurl, headers=headers, method=request.get_method(), unverifiable=True) class NoRedirect(urllib.request.HTTPRedirectHandler): @@ -156,29 +157,57 @@ def obtain_artifact(manifest, logical_path, destination, replace=False): if not isinstance(base, str) or not base or urlsplit(base).query: raise ArtifactError('No downloadable artifact source: ' + logical_path) url = safe_url(base.rstrip('/') + '/' + entry['filename']) - with temporary_file(target) as partial: - for attempt in range(3): - try: - download_artifact(url, partial, entry, logical_path) - break - except urllib.error.HTTPError as error: - if error.code not in (408, 429, 500, 502, 503, 504) or attempt == 2: - raise ArtifactError(f'Artifact download failed (HTTP {error.code}): {logical_path}. Check the console and retry.') from None - except (urllib.error.URLError, socket.timeout, ConnectionError, http.client.HTTPException): - if attempt == 2: - raise ArtifactError('Artifact transfer interrupted: ' + logical_path + '. Check network access and rerun.') from None - time.sleep(attempt + 1) - if digest(partial) != entry['sha256']: + partial = partial_path(target) + failures = 0 + while True: + had = resumable(partial, entry['size']) + try: + if had < entry['size']: + download_artifact(url, partial, entry, logical_path, had) + except urllib.error.HTTPError as error: + failures += 1 + if error.code not in (408, 429, 500, 502, 503, 504) or failures == 3: + raise ArtifactError(f'Artifact download failed (HTTP {error.code}): {logical_path}. Check the console and retry.') from None + time.sleep(failures) + continue + except (urllib.error.URLError, socket.timeout, ConnectionError, http.client.HTTPException): + failures += 1 + if failures == 3: + raise ArtifactError('Artifact transfer interrupted: ' + logical_path + '. Check network access and rerun; the download continues where it stopped.') from None + time.sleep(failures) + continue + if digest(partial) == entry['sha256']: + break + partial.unlink() + if not had: raise ArtifactError('Artifact checksum mismatch: ' + logical_path) - os.chmod(partial, 0o700 if logical_path.startswith('native/') else 0o600) - os.replace(partial, target) - return target + print(f'The earlier partial download of {logical_path} does not match this release; downloading it again.', flush=True) + os.chmod(partial, 0o700 if logical_path.startswith('native/') else 0o600) + os.replace(partial, target) + return target + + +def partial_path(target): + return target.with_name('.' + target.name + '.partial') + + +def resumable(partial, size): + """The bytes an earlier interrupted download of this file left, 0 when none is usable.""" + if partial.is_symlink() or (partial.exists() and + (not partial.is_file() or partial.stat().st_uid != os.getuid())): + raise DistributionError('Temporary file must be an owned regular file: ' + str(partial)) + if not partial.exists(): + return 0 + if partial.stat().st_size > size: + partial.unlink() + return 0 + return partial.stat().st_size @contextmanager def temporary_file(target): """Clear unfinished work under the caller's installation lock.""" - partial = target.with_name('.' + target.name + '.partial') + partial = partial_path(target) if partial.is_symlink() or (partial.exists() and (not partial.is_file() or partial.stat().st_uid != os.getuid())): raise DistributionError('Temporary file must be an owned regular file: ' + str(target)) @@ -193,33 +222,51 @@ def temporary_file(target): SLOW_SECONDS, SLOW_BYTES = 60, 64 * 1024 -def download_artifact(url, partial, entry, logical_path): - """Download a complete artifact into a disposable file.""" +def download_artifact(url, partial, entry, logical_path, have=0): + """Download an artifact into its partial file, continuing after the first `have` bytes when the server allows it.""" size = entry['size'] opener = urllib.request.build_opener(ArtifactRedirect()) - with opener.open(url, timeout=60) as stream: - flags = os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW | os.O_TRUNC + mib = 1024 * 1024 + request = urllib.request.Request(url, headers={'Range': f'bytes={have}-'} if have else {}) + try: + stream = opener.open(request, timeout=60) + except urllib.error.HTTPError as error: + if error.code != 416 or not have: + raise + # The server holds fewer bytes than the partial file: start over. + have = 0 + stream = opener.open(url, timeout=60) + with stream: + if have and (stream.status != 206 or not (stream.headers.get('Content-Range') or '').startswith(f'bytes {have}-')): + have = 0 + if have: + print(f'Resuming {logical_path} at {have / mib:.1f} MiB of {size / mib:.1f} MiB', flush=True) + flags = os.O_WRONLY | os.O_CREAT | os.O_NOFOLLOW with os.fdopen(os.open(partial, flags, 0o600), 'wb') as output: - count, reported = 0, 0 + output.truncate(have) + output.seek(have) + count = have + reported = count * 10 // size window, window_count = time.monotonic(), 0 while True: - block = stream.read1(1024 * 1024) + block = stream.read1(mib) if not block: break count += len(block) if count > size: + partial.unlink() raise ArtifactError('Artifact exceeds published size: ' + logical_path) output.write(block) window_count += len(block) if time.monotonic() - window >= SLOW_SECONDS: if window_count < SLOW_BYTES: raise ArtifactError(f'Artifact download stalled (under {SLOW_BYTES // 1024} KiB in {SLOW_SECONDS} s): ' - f'{logical_path}. Check the network, then rerun the command.') + f'{logical_path}. Check the network, then rerun the command; the download continues where it stopped.') window, window_count = time.monotonic(), 0 - if size >= 50 * 1024 * 1024 and count * 10 // size > reported: + if count * 10 // size > reported: reported = count * 10 // size try: - print(f'Downloading {logical_path}: {reported * 10}% of {size // (1024 * 1024)} MiB', flush=True) + print(f'Downloading {logical_path}: {reported * 10}% of {size / mib:.1f} MiB', flush=True) except BrokenPipeError: # Nobody reads the output any more (the installer's terminal is gone); # stop instead of retrying it as a network error. diff --git a/deploy/node/node_install.py b/deploy/node/node_install.py index 790d555a1..0b38305a2 100644 --- a/deploy/node/node_install.py +++ b/deploy/node/node_install.py @@ -487,15 +487,16 @@ def require_disk(root, args, program_manifest, manifest, names): entry = distribution.artifact(program_manifest if name in provider_assets.artifacts(args.provider, ("node",)) else manifest, name) target = root / name if not (target.is_file() and distribution.matches(target, entry)): - needed += entry["size"] + needed += entry["size"] - distribution.resumable(distribution.partial_path(target), entry["size"]) if args.provider == "docker": imported = distribution.docker_image(manifest, "runtime", DOCKER) is not None else: imported = (root / MICRO[1]).is_file() and micro_image_ready(root, args, manifest) if not imported: archive = distribution.artifact(manifest, "images/runtime.tar.gz") - if not (root / "images/runtime.tar.gz").is_file(): - needed += archive["size"] + packed = root / "images/runtime.tar.gz" + if not packed.is_file(): + needed += archive["size"] - distribution.resumable(distribution.partial_path(packed), archive["size"]) if not (root / "images/runtime.tar").is_file(): needed += archive["unpacked_size"] if args.provider == "microsandbox": diff --git a/deploy/node/test_distribution.py b/deploy/node/test_distribution.py index a0fbf096b..193f9d751 100644 --- a/deploy/node/test_distribution.py +++ b/deploy/node/test_distribution.py @@ -29,6 +29,7 @@ def setUp(self): self.ranges = [] self.if_ranges = [] self.status = 200 + self.resumable = True self.redirect_target = 'https://elsewhere.example' self.interrupted_once = False test = self @@ -48,12 +49,16 @@ def do_GET(self): self.end_headers() self.wfile.write(test.data[start:] if start else test.data[:len(test.data) // 2]) return - self.send_response(test.status) + start = int(self.headers['Range'][6:-1]) if test.resumable and test.status == 200 and self.headers.get('Range') else 0 + self.send_response(206 if start else test.status) if test.status in (302, 307): self.send_header('Location', test.redirect_target + self.path) + if start: + self.send_header('Content-Range', f'bytes {start}-{len(test.data) - 1}/{len(test.data)}') + self.send_header('Content-Length', str(len(test.data) - start)) self.end_headers() if test.status == 200: - self.wfile.write(test.data) + self.wfile.write(test.data[start:]) def log_message(self, *args): pass self.server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), Handler) @@ -110,26 +115,54 @@ def test_truncated_transfer_can_be_rerun(self): self.data = self.data[:20] with patch.object(distribution.time, 'sleep'), self.assertRaisesRegex(distribution.DistributionError, 'interrupted'): distribution.obtain_artifact(self.manifest, 'native/bin/node', self.root / 'node') - self.assertEqual(list(self.root.iterdir()), []) + self.assertEqual([path.name for path in self.root.iterdir()], ['.node.partial']) self.data = b'prebuilt artifact' * 1000 distribution.obtain_artifact(self.manifest, 'native/bin/node', self.root / 'node') self.assertEqual([path.name for path in self.root.iterdir()], ['node']) + self.assertEqual((self.root / 'node').read_bytes(), self.data) - def test_interrupted_download_restarts_with_a_clean_file(self): + def test_interrupted_download_resumes_where_it_stopped(self): self.interrupted_once = True with patch.object(distribution.time, 'sleep'): distribution.obtain_artifact(self.manifest, 'native/bin/node', self.root / 'node') self.assertEqual((self.root / 'node').read_bytes(), self.data) - self.assertEqual(self.ranges, [None, None]) - self.assertEqual(self.if_ranges, [None, None]) + self.assertEqual(self.ranges, [None, f'bytes={len(self.data) // 2}-']) + self.assertEqual([path.name for path in self.root.iterdir()], ['node']) + + def test_a_partial_file_from_an_earlier_run_resumes_and_reports_it(self): + target = self.root / 'node' + target.with_name('.node.partial').write_bytes(self.data[:5000]) + output = io.StringIO() + with patch('sys.stdout', output): + distribution.obtain_artifact(self.manifest, 'native/bin/node', target) + self.assertEqual(target.read_bytes(), self.data) + self.assertEqual(self.ranges, ['bytes=5000-']) + self.assertIn('Resuming native/bin/node at 0.0 MiB of 0.0 MiB', output.getvalue()) + self.assertIn('Downloading native/bin/node: 100%', output.getvalue()) + self.assertEqual([path.name for path in self.root.iterdir()], ['node']) + + def test_a_server_that_ignores_the_range_gets_a_full_download(self): + self.resumable = False + target = self.root / 'node' + target.with_name('.node.partial').write_bytes(self.data[:5000]) + distribution.obtain_artifact(self.manifest, 'native/bin/node', target) + self.assertEqual(target.read_bytes(), self.data) + self.assertEqual(self.ranges, ['bytes=5000-']) self.assertEqual([path.name for path in self.root.iterdir()], ['node']) - def test_a_stalled_download_stops_and_removes_its_part(self): + def test_a_partial_file_longer_than_the_artifact_is_dropped(self): + target = self.root / 'node' + target.with_name('.node.partial').write_bytes(self.data + b'more') + distribution.obtain_artifact(self.manifest, 'native/bin/node', target) + self.assertEqual(target.read_bytes(), self.data) + self.assertEqual(self.ranges, [None]) + + def test_a_stalled_download_stops_and_keeps_its_part(self): with patch.object(distribution, 'SLOW_SECONDS', 0), \ self.assertRaisesRegex(distribution.DistributionError, 'stalled.*rerun the command'): distribution.obtain_artifact(self.manifest, 'native/bin/node', self.root / 'node') self.assertEqual(len(self.requests), 1) # Not retried as a network error. - self.assertEqual(list(self.root.iterdir()), []) + self.assertEqual([path.name for path in self.root.iterdir()], ['.node.partial']) def test_runtime_expansion_is_verified_and_a_differing_cache_is_refused_or_replaced(self): raw = b'synthetic tar contents' * 1000 @@ -171,7 +204,8 @@ def test_retry_after_sigkill_discards_unfinished_artifact(self): distribution.obtain_artifact(self.manifest, 'native/bin/node', target) self.assertEqual(target.read_bytes(), self.data) self.assertFalse(partial.exists()) - self.assertEqual(self.ranges, [None]) + # The unrelated bytes fail the checksum, so the file is fetched again from the start. + self.assertEqual(self.ranges, ['bytes=16-', None]) def test_expansion_discards_stale_temporary_and_preserves_verified_archive(self): raw = b'complete runtime image' * 1000 @@ -216,7 +250,7 @@ def test_artifact_downgrades_and_metadata_redirects_are_refused(self): self.assertEqual(list(self.root.iterdir()), []) - def test_https_redirect_discards_stale_partial_and_verifies_on_the_node(self): + def test_https_redirect_resumes_a_partial_and_verifies_on_the_node(self): certificate, key = self.root / 'cert.pem', self.root / 'key.pem' subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-keyout', str(key), '-out', str(certificate), '-days', '1', @@ -253,7 +287,7 @@ def log_message(self, *args): distribution.obtain_artifact(self.manifest, 'native/bin/node', target) self.assertEqual(target.read_bytes(), self.data) self.assertEqual(len(received), 1) - self.assertNotIn('Range', received[0]) + self.assertEqual(received[0]['Range'], 'bytes=7-') self.assertNotIn('If-Range', received[0]) self.assertFalse(target.with_name('.node.partial').exists()) finally: @@ -261,7 +295,7 @@ def log_message(self, *args): release.server_close() thread.join() - def test_artifact_redirect_strips_credentials_and_custom_headers(self): + def test_artifact_redirect_keeps_only_the_resume_range(self): request = distribution.urllib.request.Request('https://console.example/artifact', headers={ 'Range': 'bytes=123-', 'If-Range': 'etag', 'Authorization': 'Bearer secret', 'Cookie': 'session=secret', 'X-Core-Key': 'secret'}) @@ -270,7 +304,7 @@ def test_artifact_redirect_strips_credentials_and_custom_headers(self): forwarded = redirect.redirect_request(request, None, 307, '', {}, target) self.assertEqual(forwarded.full_url, target) self.assertEqual(dict((k.lower(), v) for k, v in forwarded.header_items()), - {}) + {'range': 'bytes=123-'}) for invalid in ('http://release.example/file', 'file:///tmp/file', 'https://user:secret@release.example/file'): with self.subTest(invalid=invalid), self.assertRaises(distribution.ArtifactError): redirect.redirect_request(request, None, 302, '', {}, invalid) diff --git a/docs/getting-started/nodes.md b/docs/getting-started/nodes.md index 248db3916..a6961bfa1 100644 --- a/docs/getting-started/nodes.md +++ b/docs/getting-started/nodes.md @@ -24,7 +24,7 @@ The Core host joins like any other host: to run sandboxes on it, add it as a nod 1. In Web, open **Nodes** and choose **Add node**. 2. Set **Sandboxes at once** and, with microsandbox, **Retained sandboxes**: the node's [capacity](../configuration.md#node-capacity). You can change them later with **Edit node**. -3. Choose **Generate command** and copy the command. It registers one node, once, and only if it runs within 10 minutes; Web counts down and offers **Generate new command** when it expires. +3. Choose **Generate command** and copy the command. It registers one node, once, and only if it runs within an hour; Web counts down and offers **Generate new command** when it expires. 4. Run it on the host. Web follows the node from registered to connected to ready. The command downloads the node installer from your console, checks its SHA-256 and runs it with a one-time enrollment token. The installer downloads the node files and checks each against the release manifest, imports the Runtime image, registers the node, starts its service and waits until Core reports the node connected and ready. It never installs software, and it stops with a one-line hint before changing anything when a prerequisite is missing. @@ -80,16 +80,16 @@ Root only prepares the account, the group and the unit; everything else, the Doc **Node software.** A node keeps the host program of the console that added it; an upgrade of Core doesn't replace it. A release that changes the node protocol lists the actions it requires; see [protocol changes](./operations.md#protocol-changes). -**The token.** It is single-use, expires after 10 minutes and only registers the node. The installer takes it only on standard input and refuses it in the environment, where `sudo VAR=… python3` would record it in sudo's log. A sudoers policy with `log_input` records standard input, and so the token. +**The token.** It is single-use, expires after an hour and only registers the node. The installer takes it only on standard input and refuses it in the environment, where `sudo VAR=… python3` would record it in sudo's log. A sudoers policy with `log_input` records standard input, and so the token. ## Rerun, expiry and slow links - Rerunning the same command is safe. Once the node is registered, a rerun uses the node's own credential, changes nothing that already matches and needs no token. It changes nothing and says why when the node's retained identity belongs to another Core address or installation, when the sandbox configuration has changed since the node registered (the node prepares it itself; see [Change the sandbox configuration](#change-the-sandbox-configuration)), or when Core was upgraded and serves a newer node program: choose **Remove node** on **Nodes**, uninstall the node on the host, then add it again with **Add node**. - A run that fails before the node registers leaves nothing that Core knows. The next command, even after a configuration change or a Core upgrade, replaces the earlier attempt's configuration and any downloaded file that no longer matches, keeps the files that match and continues. -- A command that already expired, or was used on another host, fails at once with `The enrollment command expired or was already used`: generate a new command and run it within 10 minutes. -- If the command expires during a slow download, registration fails with `The enrollment command expired or was already used`. The downloaded files are kept: generate a new command in Web and run it. -- Failed or interrupted downloads discard their temporary files. A rerun downloads missing files from the beginning and reuses verified complete files. A transfer that brings less than 64 KiB in a minute stops; check the network and retry. -- The Docker Runtime image is about 400 MB. On a slow link, load it first: copy the release's `oac--linux-amd64-runtime.tar.gz` asset to the host and run `sudo docker load -i` on it. The installer then finds the exact image and skips the download. +- A command that already expired, or was used on another host, fails at once with `The enrollment command expired or was already used`: generate a new command and run it within an hour. +- If the command expires during a slow download, registration fails with `The enrollment command expired or was already used`. The downloaded files are kept: generate a new command in Web and run it, and the download continues where it stopped. +- Every file shows its progress. Failed or interrupted downloads keep what arrived, and a rerun resumes each partial file where it stopped (`Resuming …`), reuses verified complete files and downloads the rest. If the server cannot resume, the file starts again from the beginning. A transfer that brings less than 64 KiB in a minute stops; check the network and rerun the command. +- The Docker Runtime image is about 400 MB and the microsandbox files about 500 MB. On a slow link with Docker, load the image first: copy the release's `oac--linux-amd64-runtime.tar.gz` asset to the host and run `sudo docker load -i` on it. The installer then finds the exact image and skips the download. - Interrupting the installer, or closing its terminal, stops it; run the command again to continue. ## Logs @@ -200,7 +200,7 @@ A registered node that can't connect to Core says why in its [log](#logs): `sand | Message | Fix | | --- | --- | -| `The enrollment command expired or was already used` | Generate a new command in Web and run it within 10 minutes | +| `The enrollment command expired or was already used` | Generate a new command in Web and run it within an hour | | `Core rejected the node configuration read (HTTP …)` | Check the Core URL and the reverse proxy in front of it, then rerun | | `Not enough disk space under …` or `The disk is full` | Free the space it names, then rerun; nothing was downloaded | | This host's node program differs from the one this console serves | Core was upgraded after the node was added. Choose **Remove node**, uninstall it on the host, then add it again with **Add node** | diff --git a/docs/zh/getting-started/nodes.md b/docs/zh/getting-started/nodes.md index 0b91f2383..5d20a185f 100644 --- a/docs/zh/getting-started/nodes.md +++ b/docs/zh/getting-started/nodes.md @@ -1,7 +1,7 @@ --- title: "添加和管理节点" source: docs/getting-started/nodes.md -source_hash: 68864862ec5696e17779a10923f326b5c2b9ba8d729cc1e30dad62c6415ee85a +source_hash: 8398432d9a47fa5588fca9d0ffdea96db5407c3ef7b259c892865e5d58da9417 --- 节点是一台 Linux 主机,在沙箱提供商为 microsandbox(推荐)或 Docker 时,为 Core 托管 Session 运行沙箱。Core 将新 Session 分配给有空余容量的节点;节点创建沙箱,沙箱回连 Core。E2B 不需要节点。应用为自己的 Session 连接的机器是[自托管执行器](self-hosted.md),而不是节点。 @@ -26,7 +26,7 @@ Microsandbox 的检查点执行兼容性包含主机内核和 CPU profile。因 1. 在 Web 打开 **节点**,选择 **添加节点**。 2. 设置 **同时运行的沙箱**,使用 microsandbox 时还设置 **保留的沙箱**,即节点[容量](../configuration.md#node-capacity)。之后可以通过 **编辑节点** 修改。 -3. 选择 **生成命令** 并复制命令。命令仅注册一个节点、只能使用一次,且必须在 10 分钟内执行;Web 显示倒计时,过期后提供 **生成新命令**。 +3. 选择 **生成命令** 并复制命令。命令仅注册一个节点、只能使用一次,且必须在 1 小时内执行;Web 显示倒计时,过期后提供 **生成新命令**。 4. 在主机上执行。Web 跟踪节点从注册、连接到就绪的过程。 命令从你的控制台下载节点安装程序、检查 SHA-256,并使用一次性注册令牌运行它。安装程序下载节点文件并逐一对照发行清单验证、导入 Runtime 镜像、注册节点、启动服务,并等待 Core 报告节点已连接且就绪。程序不安装软件;前置条件缺失时,会在修改任何内容之前停止并输出一行提示。 @@ -82,16 +82,16 @@ root 只准备账号、组和服务单元;其他操作(包括 Docker 网络 **节点程序。** 节点保留添加它的控制台所提供的主机程序;升级 Core 不会替换它。更改节点协议的版本会列出所需操作;参阅[协议变更](operations.md#protocol-changes)。 -**令牌。** 令牌仅用于注册节点,一次性使用,10 分钟后过期。安装程序只从标准输入接收它,拒绝环境变量中的令牌,因为 `sudo VAR=… python3` 会在 sudo 日志中记录它。启用了 `log_input` 的 sudoers 策略会记录标准输入,从而记录令牌。 +**令牌。** 令牌仅用于注册节点,一次性使用,1 小时后过期。安装程序只从标准输入接收它,拒绝环境变量中的令牌,因为 `sudo VAR=… python3` 会在 sudo 日志中记录它。启用了 `log_input` 的 sudoers 策略会记录标准输入,从而记录令牌。 ## 重新运行、过期与慢速网络 {#rerun-expiry-and-slow-links} - 重新执行同一命令是安全的。节点注册后,重新运行使用节点自身凭据,不改变已匹配的内容,也无需令牌。以下情况程序不做任何修改并说明原因:保留的节点身份属于其他 Core 地址或安装;节点注册后沙箱配置发生变化(节点会自行准备,参阅[修改沙箱配置](#change-the-sandbox-configuration));或 Core 已升级、提供了更新的节点程序,此时在 **节点** 选择 **移除节点**,在主机上卸载该节点,再通过 **添加节点** 重新添加。 - 节点注册前失败的运行不会在 Core 留下任何记录。下一条命令即使在配置变更或 Core 升级之后执行,也会替换上次尝试的配置和不再匹配的已下载文件,保留匹配的文件并继续。 -- 已过期或已在其他主机使用的命令立即失败,显示 `The enrollment command expired or was already used`:生成新命令并在 10 分钟内执行。 -- 慢速下载期间命令过期时,注册失败并显示 `The enrollment command expired or was already used`。下载文件保留:在 Web 生成新命令后执行。 -- 失败或中断时删除下载残片;重新执行时从头下载缺失文件,复用已校验的完整文件。每分钟下载不足 64 KiB 时会停止,请检查网络后重试。 -- Docker Runtime 镜像约 400 MB。网络较慢时可提前加载:将发行资产 `oac--linux-amd64-runtime.tar.gz` 复制到主机并运行 `sudo docker load -i`。安装程序发现精确匹配的镜像后跳过下载。 +- 已过期或已在其他主机使用的命令立即失败,显示 `The enrollment command expired or was already used`:生成新命令并在 1 小时内执行。 +- 慢速下载期间命令过期时,注册失败并显示 `The enrollment command expired or was already used`。下载文件保留:在 Web 生成新命令后执行,下载从中断处继续。 +- 每个文件都会显示进度。失败或中断时保留已收到的部分;重新执行时,每个未完成的文件从中断处继续(`Resuming …`),复用已校验的完整文件,其余照常下载。服务器不支持续传时,该文件从头开始。每分钟下载不足 64 KiB 时会停止,请检查网络后重新执行命令。 +- Docker Runtime 镜像约 400 MB,microsandbox 文件约 500 MB。使用 Docker 且网络较慢时,可先加载镜像:将发行资产 `oac--linux-amd64-runtime.tar.gz` 复制到主机并运行 `sudo docker load -i`。安装程序发现精确匹配的镜像后跳过下载。 - 中断安装程序或关闭终端会停止程序;重新执行命令继续。 ## 日志 {#logs} @@ -202,7 +202,7 @@ root 只准备账号、组和服务单元;其他操作(包括 Docker 网络 | 消息 | 解决方法 | | --- | --- | -| `The enrollment command expired or was already used` | 在 Web 生成新命令并在 10 分钟内执行 | +| `The enrollment command expired or was already used` | 在 Web 生成新命令并在 1 小时内执行 | | `Core rejected the node configuration read (HTTP …)` | 检查 Core URL 及其前面的反向代理后重试 | | `Not enough disk space under …` 或 `The disk is full` | 按提示释放空间后重试;尚未下载任何内容 | | This host's node program differs from the one this console serves | 节点添加后 Core 已升级。选择 **移除节点**,在主机上卸载,再通过 **添加节点** 重新添加 | diff --git a/services/core/internal/db/queries/runtime_nodes.sql b/services/core/internal/db/queries/runtime_nodes.sql index c7d4b0274..e90b407d2 100644 --- a/services/core/internal/db/queries/runtime_nodes.sql +++ b/services/core/internal/db/queries/runtime_nodes.sql @@ -46,7 +46,7 @@ SELECT id FROM runtime_nodes WHERE id=$1 FOR UPDATE; UPDATE runtime_nodes SET connection_id=NULL FROM runtime_deployment d WHERE runtime_nodes.id=$1 AND connection_id=$2 AND connected_epoch=d.owner_epoch AND d.owner_epoch=sqlc.arg(owner_epoch); -- name: CreateRuntimeEnrollment :exec -INSERT INTO runtime_node_enrollments(id,token_sha256,installation_id,expires_at,max_active,max_retained) VALUES($1,$2,$3,clock_timestamp()+interval '10 minutes',$4,$5); +INSERT INTO runtime_node_enrollments(id,token_sha256,installation_id,expires_at,max_active,max_retained) VALUES($1,$2,$3,clock_timestamp()+interval '1 hour',$4,$5); -- name: ConsumeRuntimeEnrollment :execrows UPDATE runtime_node_enrollments SET consumed_at=clock_timestamp(),node_id=$2 diff --git a/services/core/internal/db/sqlc/runtime_nodes.sql.go b/services/core/internal/db/sqlc/runtime_nodes.sql.go index c57ca09fd..c07025aaa 100644 --- a/services/core/internal/db/sqlc/runtime_nodes.sql.go +++ b/services/core/internal/db/sqlc/runtime_nodes.sql.go @@ -51,7 +51,7 @@ func (q *Queries) ConsumeRuntimeEnrollment(ctx context.Context, arg ConsumeRunti } const createRuntimeEnrollment = `-- name: CreateRuntimeEnrollment :exec -INSERT INTO runtime_node_enrollments(id,token_sha256,installation_id,expires_at,max_active,max_retained) VALUES($1,$2,$3,clock_timestamp()+interval '10 minutes',$4,$5) +INSERT INTO runtime_node_enrollments(id,token_sha256,installation_id,expires_at,max_active,max_retained) VALUES($1,$2,$3,clock_timestamp()+interval '1 hour',$4,$5) ` type CreateRuntimeEnrollmentParams struct { diff --git a/services/core/internal/deployment/storage.go b/services/core/internal/deployment/storage.go index f08eea4e7..d910cc2a2 100644 --- a/services/core/internal/deployment/storage.go +++ b/services/core/internal/deployment/storage.go @@ -307,7 +307,7 @@ type NodeTx interface { InsertNode(node NewNode) (StoredNode, error) UpdateNode(id string, limits NodeLimits) error RemoveNode(id string) error - // CreateEnrollment stores a token that expires in ten minutes and returns + // CreateEnrollment stores a token that expires in one hour and returns // its expiry. CreateEnrollment(enrollment NewEnrollment) (time.Time, error) // ConsumeEnrollment marks an unexpired, unconsumed token of the current