From e41815669cdbff236a6b0636925febd3d9fc1cce Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 10:14:38 +0000 Subject: [PATCH 1/3] wip: roll back started plugins when a boot fails after Phase 2 Claude-Session: https://claude.ai/code/session_01ADzJtzYTLUfgrRZHxkagkX Co-authored-by: Claude --- .../src/kernel.failed-boot-rollback.test.ts | 331 ++++++++++++++++++ packages/core/src/kernel.ts | 27 ++ 2 files changed, 358 insertions(+) create mode 100644 packages/core/src/kernel.failed-boot-rollback.test.ts diff --git a/packages/core/src/kernel.failed-boot-rollback.test.ts b/packages/core/src/kernel.failed-boot-rollback.test.ts new file mode 100644 index 00000000000..b1d7997d8ef --- /dev/null +++ b/packages/core/src/kernel.failed-boot-rollback.test.ts @@ -0,0 +1,331 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +import { describe, it, expect } from 'vitest'; +import { ObjectKernel } from './kernel'; +import type { ObjectKernelConfig } from './kernel'; +import type { Plugin } from './types'; + +/** + * A boot that fails AFTER every plugin's `start()` has run — the readiness + * check, or a `kernel:ready` / `kernel:bootstrapped` / `kernel:listening` + * handler that throws — destroys the plugins that started, in reverse start + * order, through the same rollback a failing `start()` runs. + * + * Before this file only the Phase 2 branch rolled back. A later failure fell + * into the outer catch, which marks the kernel `stopped` for good; `shutdown()` + * on a stopped kernel is a no-op, so no `destroy()` ever ran and a driver + * connected in `start()` stayed connected. The composition that matters is the + * one `DefaultDatasourcePlugin` is: one plugin connects in `start()` and + * disconnects in `destroy()`, and a LATER plugin refuses the boot. + * + * The pins below split on purpose, so an ablation of the new rollback call + * (delete it, rebuild, rerun) reads as a diagnosis and not as a wall of red: + * the "tears down" pins go red; the "still the one contract" pins — the + * Phase 2 control, `shutdown()` after the failure, the original rejection, a + * disabled rollback — stay green, because each is a statement about what the + * failed boot does NOT do twice, or at all. + */ + +type BootHook = 'kernel:ready' | 'kernel:bootstrapped' | 'kernel:listening'; + +/** + * Where the boot is refused. Three are handlers a plugin registers; the fourth + * is the kernel's own readiness check, which runs after the last `start()` and + * before the first hook and throws when the required `data` service is absent + * (which `skipSystemValidation: false` makes it judge). + */ +interface RefusalSite { + readonly site: string; + readonly hook?: BootHook; + readonly config: ObjectKernelConfig; +} + +const HOOK_SITES: readonly RefusalSite[] = [ + { site: 'kernel:ready', hook: 'kernel:ready', config: {} }, + { site: 'kernel:bootstrapped', hook: 'kernel:bootstrapped', config: {} }, + { site: 'kernel:listening', hook: 'kernel:listening', config: {} }, +]; +const READINESS_SITE: RefusalSite = { + site: 'the system requirement check', + config: { skipSystemValidation: false }, +}; +const ALL_SITES: readonly RefusalSite[] = [...HOOK_SITES, READINESS_SITE]; + +function newKernel(extra: ObjectKernelConfig = {}): ObjectKernel { + return new ObjectKernel({ + logger: { level: 'silent' }, + gracefulShutdown: false, + skipSystemValidation: true, + ...extra, + }); +} + +/** The fake datasource driver: connected in `start()`, released in `destroy()`. */ +interface FakeDriver { + connected: boolean; +} + +/** + * Connects in `start()`, disconnects in `destroy()` — the shape of + * `DefaultDatasourcePlugin`. Registered first, so it starts first and is + * destroyed last. + */ +function driverHost(log: string[], driver: FakeDriver, onDestroy?: () => Promise): Plugin { + return { + name: 'test.driver-host', + version: '1.0.0', + init: async () => {}, + start: async () => { + log.push('connect'); + driver.connected = true; + }, + destroy: async () => { + log.push('disconnect'); + driver.connected = false; + await onDestroy?.(); + }, + }; +} + +/** + * The plugin that refuses the boot. It starts AFTER the driver host, so it is + * a started plugin too — its own `destroy()` runs in the rollback, before the + * driver's. `hook` is where it throws; without one it only records, and the + * boot is refused by something else. + */ +function refuser( + log: string[], + hook: BootHook | undefined, + boom: Error, + onHook?: () => Promise, + destroy?: () => Promise, +): Plugin { + return { + name: 'test.refuser', + version: '1.0.0', + dependencies: ['test.driver-host'], + init: async (ctx) => { + if (!hook) return; + ctx.hook(hook, async () => { + log.push(`refuser:${hook}`); + await onHook?.(); + throw boom; + }); + }, + start: async () => { + log.push('refuser:start'); + }, + destroy: + destroy + ?? (async () => { + log.push('refuser:destroy'); + }), + }; +} + +/** What happened before the refusal: connect, the refuser's start, and its hook if it has one. */ +function untilRefusal(site: RefusalSite): string[] { + return ['connect', 'refuser:start', ...(site.hook ? [`refuser:${site.hook}`] : [])]; +} + +/** A booted-and-refused composition: the kernel, what it logged, the driver. */ +async function refusedBoot(site: RefusalSite, boom = new Error(`refused at ${site.site}`)) { + const log: string[] = []; + const driver: FakeDriver = { connected: false }; + const kernel = newKernel(site.config); + await kernel.use(driverHost(log, driver)); + await kernel.use(refuser(log, site.hook, boom)); + + const rejection: unknown = await kernel.bootstrap().then( + () => undefined, + (e: unknown) => e, + ); + return { kernel, log, driver, boom, rejection }; +} + +describe('ObjectKernel — a boot that fails after every plugin started rolls them back (#23644)', () => { + describe('tears down', () => { + it.each(ALL_SITES)( + 'a throw at $site destroys the plugins that started, in reverse start order, and disconnects the driver', + async (site) => { + const { log, driver, rejection } = await refusedBoot(site); + + expect(rejection).toBeInstanceOf(Error); + // Connected in start(), released in destroy(): the pin the card is about. + expect(driver.connected).toBe(false); + // Each started plugin exactly once, the later-started one first. An exact + // sequence, so a plugin destroyed twice reads as red, not as "was destroyed". + expect(log).toEqual([...untilRefusal(site), 'refuser:destroy', 'disconnect']); + }, + ); + + it('a destroy() that throws during the rollback neither replaces the boot error nor stops the plugins started before it', async () => { + const log: string[] = []; + const driver: FakeDriver = { connected: false }; + const boom = new Error('refused at kernel:ready'); + const kernel = newKernel(); + await kernel.use(driverHost(log, driver)); + await kernel.use( + refuser(log, 'kernel:ready', boom, undefined, async () => { + log.push('refuser:destroy'); + throw new Error('refuser destroy failed'); + }), + ); + + const rejection = await kernel.bootstrap().then( + () => undefined, + (e: unknown) => e, + ); + + // The failed destroy is logged by the rollback, not surfaced: the caller still sees the boot error... + expect(rejection).toBe(boom); + // ...and the plugin started before the throwing one is still torn down. + expect(log).toEqual(['connect', 'refuser:start', 'refuser:kernel:ready', 'refuser:destroy', 'disconnect']); + expect(driver.connected).toBe(false); + }); + + it('the rollback runs with the kernel `stopping`, so a shutdown() issued from inside a destroy() is absorbed, not a second teardown', async () => { + const log: string[] = []; + const driver: FakeDriver = { connected: false }; + const statesSeenByDestroy: string[] = []; + const kernel = newKernel(); + await kernel.use( + driverHost(log, driver, async () => { + statesSeenByDestroy.push(kernel.getState()); + await kernel.shutdown(); + }), + ); + await kernel.use(refuser(log, 'kernel:ready', new Error('refused at kernel:ready'))); + + await kernel.bootstrap().catch(() => undefined); + + expect(statesSeenByDestroy).toEqual(['stopping']); + expect(log.filter((entry) => entry === 'disconnect')).toHaveLength(1); + expect(kernel.getState()).toBe('stopped'); + }); + }); + + describe('is still the one contract', () => { + it('CONTROL: the Phase 2 rollback (a start() that throws) runs once per started plugin, not twice', async () => { + const log: string[] = []; + const driver: FakeDriver = { connected: false }; + const kernel = newKernel(); + await kernel.use(driverHost(log, driver)); + await kernel.use({ + name: 'test.middle', + version: '1.0.0', + dependencies: ['test.driver-host'], + init: async () => {}, + start: async () => { log.push('middle:start'); }, + destroy: async () => { log.push('middle:destroy'); }, + }); + await kernel.use({ + name: 'test.starter-that-throws', + version: '1.0.0', + dependencies: ['test.middle'], + init: async () => {}, + start: async () => { + log.push('thrower:start'); + throw new Error('thrower exploded'); + }, + // Never started, so never rolled back — its destroy() must not run. + destroy: async () => { log.push('thrower:destroy'); }, + }); + + await expect(kernel.bootstrap()).rejects.toThrow('failed to start - rollback complete: thrower exploded'); + + expect(log).toEqual(['connect', 'middle:start', 'thrower:start', 'middle:destroy', 'disconnect']); + expect(driver.connected).toBe(false); + expect(kernel.getState()).toBe('stopped'); + }); + + it.each(ALL_SITES)('shutdown() after a failed boot at $site stays a no-op: no second destroy', async (site) => { + const { kernel, log } = await refusedBoot(site); + const before = [...log]; + + await kernel.shutdown(); + + expect(log).toEqual(before); + expect(kernel.getState()).toBe('stopped'); + }); + + it.each(HOOK_SITES)('the rejection at $site is the original boot error, not a wrapper', async (site) => { + const { rejection, boom } = await refusedBoot(site); + + // Identity, not a message match: a rollback that rewrapped the error would still match a message. + expect(rejection).toBe(boom); + }); + + it('the readiness check rejects with its own error', async () => { + const { rejection } = await refusedBoot(READINESS_SITE); + + expect(rejection).toBeInstanceOf(Error); + expect((rejection as Error).message).toContain('Missing critical services: data'); + }); + + it.each(ALL_SITES)('rollbackOnFailure: false leaves the started plugins alone, as it does for a failing start(), at $site', async (site) => { + const log: string[] = []; + const driver: FakeDriver = { connected: false }; + const kernel = newKernel({ ...site.config, rollbackOnFailure: false }); + await kernel.use(driverHost(log, driver)); + await kernel.use(refuser(log, site.hook, new Error(`refused at ${site.site}`))); + + await expect(kernel.bootstrap()).rejects.toThrow(); + + expect(log).toEqual(untilRefusal(site)); + expect(driver.connected).toBe(true); + expect(kernel.getState()).toBe('stopped'); + }); + + it('a shutdown() that began mid-boot owns the destroys: the failed boot does not destroy again', async () => { + const log: string[] = []; + const driver: FakeDriver = { connected: false }; + const boom = new Error('refused after a shutdown()'); + const kernel = newKernel(); + await kernel.use(driverHost(log, driver)); + await kernel.use( + refuser(log, 'kernel:ready', boom, async () => { + // The host stops the kernel while the boot is still in a handler. + await kernel.shutdown(); + }), + ); + + const rejection = await kernel.bootstrap().then( + () => undefined, + (e: unknown) => e, + ); + + expect(rejection).toBe(boom); + // shutdown() destroyed each plugin once; the failed boot added nothing. + expect(log).toEqual(['connect', 'refuser:start', 'refuser:kernel:ready', 'refuser:destroy', 'disconnect']); + expect(kernel.getState()).toBe('stopped'); + }); + + it('a boot that fails in Phase 1 destroys nothing: no plugin has started', async () => { + const log: string[] = []; + const kernel = newKernel(); + await kernel.use({ + name: 'test.initialised-only', + version: '1.0.0', + init: async () => { log.push('first:init'); }, + start: async () => { log.push('first:start'); }, + destroy: async () => { log.push('first:destroy'); }, + }); + await kernel.use({ + name: 'test.init-thrower', + version: '1.0.0', + dependencies: ['test.initialised-only'], + init: async () => { + log.push('thrower:init'); + throw new Error('init refused'); + }, + destroy: async () => { log.push('thrower:destroy'); }, + }); + + await expect(kernel.bootstrap()).rejects.toThrow('init refused'); + + expect(log).toEqual(['first:init', 'thrower:init']); + expect(kernel.getState()).toBe('stopped'); + }); + }); +}); diff --git a/packages/core/src/kernel.ts b/packages/core/src/kernel.ts index e468e3a7450..db1892c3406 100644 --- a/packages/core/src/kernel.ts +++ b/packages/core/src/kernel.ts @@ -619,6 +619,33 @@ export class ObjectKernel { this.logger.info('✅ Bootstrap complete'); } catch (error) { + // A boot that fails once its plugins have STARTED — the readiness + // check, or a `kernel:ready` / `kernel:bootstrapped` / + // `kernel:listening` handler — leaves them holding what `start()` + // took: a connected driver, an armed timer. This kernel is stopped + // for good below, so `shutdown()` on it is a no-op and nothing + // else will ever destroy them: the failed boot tears them down + // itself, through the SAME rollback a failing `start()` runs (reverse + // start order, a throwing `destroy()` logged and never replacing + // `error`) and under the same `rollbackOnFailure` gate, so one + // contract answers a failed boot at whatever phase it fails. + // + // `startedPlugins` is empty once the Phase 2 branch above has + // rolled back, so a boot that already rolled back never destroys a + // plugin twice. Only a kernel still `running` rolls back: a + // `shutdown()` that began mid-boot already owns the destroys. The + // `stopping` state holds the same ownership the other way round — + // a `shutdown()` or a signal that arrives during the rollback is + // absorbed (listeners stay installed until `stopped`, below). + if ( + this.config.rollbackOnFailure + && this.state === 'running' + && this.startedPlugins.size > 0 + ) { + this.state = 'stopping'; + this.logger.warn('Bootstrap failed after plugins started — rolling back started plugins...'); + await this.rollbackStartedPlugins(); + } this.state = 'stopped'; // A kernel whose boot failed is stopped for good (`shutdown()` on it // is a no-op), so this is the only place its listeners can go. From 4fab80d9707fed4cf6d26fb7a93ea14e67e88946 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 10:20:19 +0000 Subject: [PATCH 2/3] chore(changeset): patch for the failed-boot rollback Claude-Session: https://claude.ai/code/session_01ADzJtzYTLUfgrRZHxkagkX Co-authored-by: Claude --- .changeset/23644-failed-boot-rollback.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) create mode 100644 .changeset/23644-failed-boot-rollback.md diff --git a/.changeset/23644-failed-boot-rollback.md b/.changeset/23644-failed-boot-rollback.md new file mode 100644 index 00000000000..8a2c3f01ea4 --- /dev/null +++ b/.changeset/23644-failed-boot-rollback.md @@ -0,0 +1,20 @@ +--- +'@objectstack/core': patch +--- + +fix(core): a bootstrap that fails after every plugin started rolls them back too, so a driver connected in `start()` is disconnected + +Clause-②: no + +`ObjectKernel.bootstrap()` rolled back started plugins only when a plugin's `start()` failed. A boot refused later (the system-requirement check, or a `kernel:ready`, `kernel:bootstrapped` or `kernel:listening` handler that throws) left the kernel `stopped` with every plugin still holding what its `start()` took: a datasource driver stayed connected, because `shutdown()` on a stopped kernel is a no-op and nothing else ran any `destroy()`. + +- **What changes.** A boot that fails after the last `start()` now destroys the plugins that started, in reverse start order, through the same rollback a failing `start()` already ran. One teardown answers a failed boot at whatever phase it fails. A kernel built with `rollbackOnFailure: false` keeps opting out of both. +- **What does not change.** + - The rejection is still the original error, unwrapped. + - A `destroy()` that throws during the rollback is logged at `error` and does not replace it; the plugins started before it are still destroyed. + - A plugin whose `start()` failed or never ran is not destroyed. + - A failed `start()` still rolls back exactly once. + - `shutdown()` after any failed boot is still a no-op, so nothing is destroyed twice. +- **Ownership of the teardown.** While the rollback runs the kernel reads `stopping`, so a `shutdown()` or a signal that arrives meanwhile is absorbed. A `shutdown()` that already began mid-boot owns the destroys, and the failed boot adds none. +- **Who is affected.** In-process hosts that await a boot which can be refused after connect: `Runtime.start()`, `bootStack`, tests. `os serve` exits on a refused boot and is not. +- **What to check in a plugin.** A `destroy()` may now run on a plugin whose `kernel:ready` handler never ran, the same tolerance `bootPhaseHooks: false` already asks for. A host that closed such a resource by hand after a refused boot now closes it a second time, so that close should tolerate an already-closed resource. From e26fb88f5a8ad05592c543b263a517b1ad8369d1 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 15 Oct 2026 10:54:30 +0000 Subject: [PATCH 3/3] docs(core): state the failed-boot rollback on rollbackOnFailure and in the changeset Claude-Session: https://claude.ai/code/session_01ADzJtzYTLUfgrRZHxkagkX Co-authored-by: Claude --- .changeset/23644-failed-boot-rollback.md | 4 ++-- packages/core/src/kernel.ts | 13 ++++++++++++- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.changeset/23644-failed-boot-rollback.md b/.changeset/23644-failed-boot-rollback.md index 8a2c3f01ea4..969f41398af 100644 --- a/.changeset/23644-failed-boot-rollback.md +++ b/.changeset/23644-failed-boot-rollback.md @@ -10,8 +10,8 @@ Clause-②: no - **What changes.** A boot that fails after the last `start()` now destroys the plugins that started, in reverse start order, through the same rollback a failing `start()` already ran. One teardown answers a failed boot at whatever phase it fails. A kernel built with `rollbackOnFailure: false` keeps opting out of both. - **What does not change.** - - The rejection is still the original error, unwrapped. - - A `destroy()` that throws during the rollback is logged at `error` and does not replace it; the plugins started before it are still destroyed. + - A readiness or hook failure still rejects with its own error, unwrapped; a failing `start()` still rejects with the error naming the plugin. + - A `destroy()` that throws during the rollback is logged at `error` and does not replace the boot error; the plugins started before it are still destroyed. - A plugin whose `start()` failed or never ran is not destroyed. - A failed `start()` still rolls back exactly once. - `shutdown()` after any failed boot is still a no-op, so nothing is destroyed twice. diff --git a/packages/core/src/kernel.ts b/packages/core/src/kernel.ts index db1892c3406..f042f2d1aa5 100644 --- a/packages/core/src/kernel.ts +++ b/packages/core/src/kernel.ts @@ -109,7 +109,18 @@ export interface ObjectKernelConfig { /** Graceful shutdown timeout in milliseconds */ shutdownTimeout?: number; - /** Whether to rollback on startup failure */ + /** + * Whether a failed boot destroys the plugins that started, in reverse start + * order. Default `true`: a `start()` that throws or times out, the + * system-requirement check, and a `kernel:ready`, `kernel:bootstrapped` or + * `kernel:listening` handler that throws each fail the boot after that + * rollback. A failing `start()` rejects with an error naming the plugin + * (its `cause` is the original); the other failures reject with their own + * error, unwrapped. `false`: a failing `start()` is logged and the boot + * goes on without that plugin; any later failure rejects the boot with the + * started plugins left started (the kernel is `stopped` and `shutdown()` on + * it is a no-op), so the host owns whatever they hold. + */ rollbackOnFailure?: boolean; /** Whether to skip strict system requirement validation (Critical for testing) */