diff --git a/.changeset/23645-delete-heal-boot-hydrated.md b/.changeset/23645-delete-heal-boot-hydrated.md new file mode 100644 index 00000000000..237014ecb66 --- /dev/null +++ b/.changeset/23645-delete-heal-boot-hydrated.md @@ -0,0 +1,19 @@ +--- +'@objectstack/metadata-protocol': patch +--- + +fix(metadata-protocol): deleting a runtime object that was loaded at boot unregisters it at once, instead of leaving it registered and served until a restart + +Clause-②: no + +Before this fix, deleting such an object could leave it registered. This happened on a kernel whose ObjectQL plugin has no `environmentId`: the CLI's host-config boot, or a control plane. The object was a runtime object created before the server's last start. `DELETE /api/v1/meta/object/NAME` answered "it no longer exists" and removed the stored row, but the object stayed registered. Until a restart, data CRUD kept serving its table's records. `GET /api/v1/meta/object/NAME`, the object list and the layered read kept serving the deleted definition. + +The cause: at boot the ObjectQL plugin copies every registered object into the `metadata` service. The delete's registry heal read that copy as the code baseline and registered it again. An object created in the same process was never copied, so its delete already worked. + +Now an object body in the `metadata` service stamped tenant-authored (`_provenance: 'org'`) is never read as the object's code layer. What changes on those kernels: + +- After the delete, the object is unregistered, as one created in the same process is. Data CRUD answers `404 OBJECT_NOT_FOUND`. The by-name read and the object list no longer serve it. Its table and records are kept: a delete without `dropStorage` drops nothing. This holds whether or not the row was bound to a package. +- A save afterwards that names a package makes that package the object's owner at once, over the kept table. +- The layered read's `code` layer for a runtime object is `null`, as it already was on a kernel with an `environmentId`. Before, it was the boot copy. + +Unchanged: an object a code package ships keeps its baseline through the same delete. The `metadata` service itself still holds its boot copy. Other types are not affected: the security catalog's `permission` and `position` entries in the `metadata` service are still read as before. diff --git a/packages/metadata-protocol/src/protocol.ts b/packages/metadata-protocol/src/protocol.ts index 3724616527f..7216e63ab3d 100644 --- a/packages/metadata-protocol/src/protocol.ts +++ b/packages/metadata-protocol/src/protocol.ts @@ -9436,6 +9436,37 @@ export class ObjectStackProtocolImplementation implements * `degraded` must be the verdict of the WHOLE lookup: a first read that * failed is not made trustworthy by an alternate spelling that cleanly * missed. + * + * ## [#23645] A tenant-authored OBJECT body is not a code layer + * + * Every caller reads this as the CODE layer: the delete heal's baseline + * ({@link restoreArtifactRegistryView}, tier 2), the by-name read's step 2, + * the layered read's `code` layer and the carry-forward's code layer. For + * an `object` the service can also hold a copy of a stored row. On a kernel + * with no `environmentId`, ObjectQL's boot bridge + * (`bridgeObjectsToMetadataService`) copies every object the registry holds + * into the service, so every runtime object hydrated from `sys_metadata` + * lands there too, stamped `_provenance: 'org'`. Nothing removes that copy + * when the row is deleted. Measured on a real `ObjectKernel` with + * `MetadataPlugin` and `ObjectQLPlugin`: after `DELETE` of a boot-hydrated + * runtime object answered "it no longer exists", tier 2 found the copy, + * re-registered it and returned before tier 3's `unregisterObject`. The + * owner, the data plane and `getMetaItem` (step 2, the same copy) all kept + * serving the deleted definition until a restart. An object saved in the + * same process was never copied, so its delete worked. + * + * So an `object` body stamped tenant-authored answers here as a clean miss. + * This is the discriminator {@link getMetaItemLayered} already applies to + * the registry's plain key ([#21059]): the ADR-0010 marker the hydrator + * writes, read through `isTenantAuthored`. ⛔ No new marker. A stored row is + * read from the store, never from this code-layer read. A miss here is not + * degraded, because the service answered. + * + * ⛔ `object` only. The security catalog's echo (`plugin-security`) + * registers stored `permission` and `position` rows in the service on + * purpose, stamped the same way. It keeps them current on every mutation, + * and this read serves them as the code layer by design: its own pins read + * the layered `code` layer as the echo. */ private async readItemFromMetadataService( type: string, @@ -9452,15 +9483,21 @@ export class ObjectStackProtocolImplementation implements // occupant of the `metadata` slot makes of a third argument today is // unchanged by which of the two methods answers. const read = async (t: string): Promise<{ data: unknown; degraded: boolean; errors: string[] }> => { + let answer: { data: unknown; degraded: boolean; errors: string[] }; if (typeof metadataService.getDiagnosed === 'function') { const diagnosed = await metadataService.getDiagnosed(t, name, packageId); - return { + answer = { data: diagnosed?.data, degraded: diagnosed?.degraded === true, errors: Array.isArray(diagnosed?.errors) ? diagnosed.errors : [], }; + } else { + answer = { data: await metadataService.get(t, name, packageId), degraded: false, errors: [] }; } - return { data: await metadataService.get(t, name, packageId), degraded: false, errors: [] }; + // [#23645] A stored object row's copy is not a code layer. See the header. + return canonicalMetaType(t) === 'object' && isTenantAuthored(answer.data) + ? { ...answer, data: undefined } + : answer; }; const primary = await read(type); @@ -9986,6 +10023,14 @@ export class ObjectStackProtocolImplementation implements const metadataService = services?.get('metadata'); if (metadataService && typeof metadataService.list === 'function') { let runtimeItems = await metadataService.list(request.type); + // [#23645] The service's copy of a stored object row is not the + // base this merge lays the stored rows over: the rows above + // already serve every live one, and a copy of a deleted one + // would list it until a restart. `object` only, as in + // {@link readItemFromMetadataService}. + if (request.type === 'object' && Array.isArray(runtimeItems)) { + runtimeItems = runtimeItems.filter((item: unknown) => !isTenantAuthored(item)); + } // When filtering by packageId, only include runtime items that // belong to the requested package. MetadataService.list() returns // items from ALL packages, so we must filter here to respect the @@ -20473,6 +20518,12 @@ export class ObjectStackProtocolImplementation implements * MetadataService baseline (FilesystemLoader-sourced types) and * re-register it, preserving the historical refresh behaviour * for items the SchemaRegistry never held as artifacts. + * [#23645] A tenant-authored object body is no baseline: on a kernel + * with no `environmentId` the service holds a copy of every object + * hydrated at boot, which is the deleted row itself. The read + * answers it as a miss ({@link readItemFromMetadataService}), so the + * walk reaches step 3 for a runtime object, as for one saved in the + * same process. * [#22057] Tier 1 also declines when there is no plain-key entry, * which is what a VIEW row bound to one package leaves under a name * another package ships ({@link hydrateOverlayIntoRegistry}). So a diff --git a/packages/objectql/src/protocol-delete-boot-hydrated-object-heal.test.ts b/packages/objectql/src/protocol-delete-boot-hydrated-object-heal.test.ts new file mode 100644 index 00000000000..9d45db885d4 --- /dev/null +++ b/packages/objectql/src/protocol-delete-boot-hydrated-object-heal.test.ts @@ -0,0 +1,341 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * #23645 — deleting a runtime object that was HYDRATED AT BOOT unregisters it + * at once, as the delete of one saved in the same process always did. + * + * ## The defect + * + * On a kernel with no `environmentId`, `ObjectQLPlugin.start()` copies every + * object the registry holds into the `metadata` service + * (`bridgeObjectsToMetadataService`). That includes each runtime object + * hydrated from `sys_metadata`, whose copy carries the registry's + * `_provenance: 'org'`. Nothing removes the copy when the row is deleted. + * + * The delete heal (`restoreArtifactRegistryView`) reads the `metadata` + * service as its tier-2 BASELINE. It found the copy, re-registered it, and + * returned before tier 3's `unregisterObject`. Measured on this file's kernel + * before the fix: `DELETE` answered "it no longer exists" and the stored row + * was gone. Yet the registry owner stayed, the data plane kept serving the + * table's records, and `getMetaItem`, the list and the layered `code` layer + * served the same copy, all until the next boot. The two other kernel shapes + * never had the copy, so they never had the defect: an object saved in the + * same process, and a kernel with an `environmentId` (no bridge). + * + * The fix is in `readItemFromMetadataService`, the one read every code-layer + * reader goes through, the heal's tier 2 included. A tenant-authored OBJECT + * body answers there as a clean miss, by the marker #21059 already reads + * (`isTenantAuthored`). The list's merge drops such objects from its base. + * + * ## Why a real kernel, with `MetadataPlugin` + * + * The copy is made by `ObjectQLPlugin.start()` into the `metadata` service + * `MetadataPlugin` provides (a `NodeMetadataManager`, `watch: false`), and the + * boot half is `loadMetaFromDb`'s. A second kernel over the same store is the + * boot. + * + * ## The pins + * + * - a package-less runtime object and one bound to a package (the variant the + * card left unmeasured): after the delete, the owner, the by-name read, the + * list, the layered read and the data plane agree it is gone, and its table + * keeps its records; + * - then a save naming a package makes that package the owner, with the table + * and records kept, and a boot agrees: the way out #23514's refusal names; + * - CONTROL: a code package's object that reaches the same delete keeps its + * baseline (tier 2 still re-registers it). + * + * The negative control (red with the fix removed) is a one-time ablation, + * cited in the PR, not a permanent case. + */ + +import { randomUUID } from 'node:crypto'; +import { mkdtempSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { join } from 'node:path'; +import { afterEach, describe, expect, it } from 'vitest'; +import { ObjectKernel } from '@objectstack/core'; +import { MetadataPlugin } from '@objectstack/metadata'; +import { ObjectQL } from './engine.js'; +import { ObjectQLPlugin } from './plugin.js'; + +type Stores = Map>>; + +/** + * In-memory driver, copied from `protocol-runtime-owner-overlay-refusal.test.ts`; + * equality-only WHERE. `dropTable` records the tables a run drops, so a pin can + * say none was. + */ +function makeMemoryDriver(stores: Stores, drops: string[]) { + const storeFor = (obj: string) => { + let s = stores.get(obj); + if (!s) { s = new Map(); stores.set(obj, s); } + return s; + }; + // `$and` / `$or` are conjoined WITH their sibling keys, the way a real + // driver ANDs them. + const matchesWhere = (row: Record, where: any): boolean => { + if (!where || typeof where !== 'object') return true; + for (const [k, v] of Object.entries(where)) { + if (k === '$and' && Array.isArray(v)) { + if (!v.every((w: any) => matchesWhere(row, w))) return false; + continue; + } + if (k === '$or' && Array.isArray(v)) { + if (!v.some((w: any) => matchesWhere(row, w))) return false; + continue; + } + if (k.startsWith('$')) continue; + const rowVal = row[k]; + const expected = (v && typeof v === 'object' && '$eq' in (v as any)) ? (v as any).$eq : v; + const a = rowVal === undefined ? null : rowVal; + const b = expected === undefined ? null : expected; + if (a !== b) return false; + } + return true; + }; + const driver: any = { + name: 'memory', version: '0.0.0', supports: {} as any, + async connect() {}, async disconnect() {}, async checkHealth() { return true; }, + async execute() { return null; }, + async find(object: string, ast: any) { + const rows = Array.from(storeFor(object).values()).filter((r) => matchesWhere(r, ast?.where)); + // The caller's bound, after the filter, by presence (`check:objectql-double-limit`). + return typeof ast?.limit === 'number' ? rows.slice(0, ast.limit) : rows; + }, + async findOne(object: string, ast: any) { + for (const r of storeFor(object).values()) if (matchesWhere(r, ast?.where)) return r; + return null; + }, + async create(object: string, data: Record) { + const id = (data.id as string) ?? `r_${randomUUID()}`; + const row = { ...data, id }; + storeFor(object).set(id, row); + return row; + }, + async update(object: string, id: string, data: Record) { + const s = storeFor(object); + const cur = s.get(id); + if (!cur) throw new Error(`not found: ${object}/${id}`); + const updated = { ...cur, ...data, id }; + s.set(id, updated); + return updated; + }, + async upsert(object: string, data: Record) { + const id = data.id as string | undefined; + if (id && storeFor(object).has(id)) return this.update(object, id, data); + return this.create(object, data); + }, + async delete(object: string, id: string) { return storeFor(object).delete(id); }, + async count(object: string, ast: any) { return (await this.find(object, ast)).length; }, + async bulkCreate(object: string, rows: Record[]) { + return Promise.all(rows.map((r) => this.create(object, r))); + }, + async bulkUpdate() { return []; }, async bulkDelete() {}, + async beginTransaction() { return { commit: async () => {}, rollback: async () => {} }; }, + async commit() {}, async rollback() {}, + async dropTable(object: string) { drops.push(object); stores.delete(object); }, + }; + return driver; +} + +const P = 'com.qa.p'; +const Q = 'com.qa.q'; +const NAME = 'qa_probe_gadget'; +const SENTINEL = 'sys_metadata'; +const SHIPPED_PKG = 'com.qa.shipped'; +const SHIPPED = 'qa_shipped_widget'; +const SYSTEM = { context: { isSystem: true } } as any; + +/** A runtime object body; `label` tells the writes apart. */ +const gadgetBody = (label: string, name = NAME) => ({ + name, + label, + // [#8310] The runtime object door requires an authored OWD. + sharingModel: 'private', + fields: { + name: { name: 'name', type: 'text', label: 'Name' }, + }, +}); + +interface Booted { + kernel: ObjectKernel; + engine: ObjectQL; + protocol: any; + metadata: any; +} + +describe('#23645 — a runtime object hydrated at boot, then deleted', () => { + const live: ObjectKernel[] = []; + const roots: string[] = []; + afterEach(async () => { + while (live.length) { + const kernel = live.pop()!; + if (kernel.getState() === 'running') await kernel.shutdown(); + } + while (roots.length) rmSync(roots.pop()!, { recursive: true, force: true }); + }); + + /** + * One boot over `stores`: `MetadataPlugin` provides the `metadata` service + * (an empty temp root, so it scans nothing), and `ObjectQLPlugin` with no + * `environmentId` hydrates `sys_metadata` and runs the boot bridge. + * `beforeBoot` registers what a code package ships, ahead of the bridge. + */ + async function boot(stores: Stores, drops: string[], beforeBoot?: (engine: ObjectQL) => void): Promise { + // `logger.level` is the real config key, and gracefulShutdown:false keeps + // process signal handlers out of vitest's fork workers (#4250). + const kernel = new ObjectKernel({ logger: { level: 'silent' }, gracefulShutdown: false }); + live.push(kernel); + const root = mkdtempSync(join(tmpdir(), 'os-23645-')); + roots.push(root); + const engine = new ObjectQL(); + engine.registerDriver(makeMemoryDriver(stores, drops), true); + await kernel.use(new MetadataPlugin({ watch: false, rootDir: root })); + await kernel.use(new ObjectQLPlugin({ ql: engine })); + beforeBoot?.(engine); + await kernel.bootstrap(); + return { + kernel, + engine, + protocol: kernel.getService('protocol'), + metadata: kernel.getService('metadata'), + }; + } + + /** The bindings of the object's stored rows, as the driver holds them. */ + const storedBindings = (stores: Stores) => + Array.from(stores.get('sys_metadata')?.values() ?? []) + .filter((r) => r.type === 'object' && r.name === NAME) + .map((r) => r.package_id ?? null); + + const save = (k: Booted, packageId: string | undefined, label: string) => + k.protocol.saveMetaItem({ + type: 'object', name: NAME, ...(packageId ? { packageId } : {}), item: gadgetBody(label), + }); + + /** + * The answers that must agree on a live object: the registry's owner, the + * definition the data plane dispatches on, the by-name read, and the + * records the data plane serves. + */ + async function served(k: Booted, name = NAME) { + const read = await k.protocol.getMetaItem({ type: 'object', name }); + return { + owner: k.engine.registry.getObjectOwner(name)?.packageId, + dataPlaneLabel: (k.engine.registry.getObject(name) as { label?: string } | undefined)?.label, + metaReadLabel: (read?.item as { label?: string } | undefined)?.label, + records: (await k.engine.find(name, {}, SYSTEM)).length, + }; + } + + /** The object saved under `binding` (or none), two records in its table, then a second boot. */ + async function bootHydrated(stores: Stores, drops: string[], binding: string | undefined): Promise { + const k1 = await boot(stores, drops); + expect((await save(k1, binding, 'Gadget')).success).toBe(true); + await k1.engine.insert(NAME, { name: 'rec-1' }, SYSTEM); + await k1.engine.insert(NAME, { name: 'rec-2' }, SYSTEM); + await k1.kernel.shutdown(); + + const k2 = await boot(stores, drops); + expect(await served(k2)).toEqual({ + owner: binding ?? SENTINEL, dataPlaneLabel: 'Gadget', metaReadLabel: 'Gadget', records: 2, + }); + // The defect's ingredient: the boot bridge's copy of the hydrated row, + // stamped tenant-authored. Without it the pins below would pass for an + // empty reason. + const copy = await k2.metadata.get('object', NAME); + expect(copy).toMatchObject({ label: 'Gadget', _packageId: binding ?? SENTINEL, _provenance: 'org' }); + return k2; + } + + /** Every read that must say the object is gone, and the table that must keep its records. */ + async function expectGone(k: Booted, stores: Stores, drops: string[]) { + expect(storedBindings(stores)).toEqual([]); + expect(k.engine.registry.getObjectOwner(NAME)).toBeUndefined(); + expect(k.engine.registry.getObject(NAME)).toBeUndefined(); + // The by-name read, the list and the layered read. + expect((await k.protocol.getMetaItem({ type: 'object', name: NAME }))?.item).toBeUndefined(); + const listed = (await k.protocol.getMetaItems({ type: 'object' })).items as Array<{ name?: string }>; + expect(listed.map((it) => it?.name)).not.toContain(NAME); + const layered = await k.protocol.getMetaItemLayered({ type: 'object', name: NAME }); + expect({ code: layered.code, overlay: layered.overlay, effective: layered.effective }) + .toEqual({ code: null, overlay: null, effective: null }); + // The data plane refuses with the ADR-0112 envelope. + await expect(k.engine.find(NAME, {}, SYSTEM)).rejects.toMatchObject({ code: 'OBJECT_NOT_FOUND', status: 404 }); + // The table and its records are kept (no `dropStorage`). + expect(stores.get(NAME)?.size).toBe(2); + expect(drops).toEqual([]); + } + + const bindings: Array<{ label: string; binding: string | undefined }> = [ + { label: 'package-less', binding: undefined }, + { label: 'bound to a package', binding: P }, + ]; + + for (const { label, binding } of bindings) { + it(`${label}: the owner, the reads and the data plane agree it is gone at once, and the table keeps its records`, async () => { + const stores: Stores = new Map(); + const drops: string[] = []; + const k2 = await bootHydrated(stores, drops, binding); + + const del = await k2.protocol.deleteMetaItem({ type: 'object', name: NAME }); + + expect(del).toMatchObject({ success: true, reset: true }); + await expectGone(k2, stores, drops); + }); + + it(`${label}: then a save naming a package makes it the owner, with the table and records kept, and a boot agrees`, async () => { + const stores: Stores = new Map(); + const drops: string[] = []; + const k2 = await bootHydrated(stores, drops, binding); + expect((await k2.protocol.deleteMetaItem({ type: 'object', name: NAME })).success).toBe(true); + await expectGone(k2, stores, drops); + + expect((await save(k2, Q, 'Gadget Q')).success).toBe(true); + + expect(storedBindings(stores)).toEqual([Q]); + expect(await served(k2)).toEqual({ + owner: Q, dataPlaneLabel: 'Gadget Q', metaReadLabel: 'Gadget Q', records: 2, + }); + await k2.engine.insert(NAME, { name: 'rec-3' }, SYSTEM); + + await k2.kernel.shutdown(); + const k3 = await boot(stores, drops); + expect(await served(k3)).toEqual({ + owner: Q, dataPlaneLabel: 'Gadget Q', metaReadLabel: 'Gadget Q', records: 3, + }); + expect(storedBindings(stores)).toEqual([Q]); + expect(drops).toEqual([]); + }); + } + + it('CONTROL: a code package\'s object reaching the same delete keeps its baseline, still owned, served and dispatched', async () => { + const stores: Stores = new Map(); + const drops: string[] = []; + const k = await boot(stores, drops, (engine) => { + engine.registerApp({ + id: SHIPPED_PKG, name: SHIPPED_PKG, version: '1.0.0', type: 'app', + objects: [gadgetBody('Shipped', SHIPPED)], + }); + }); + await k.engine.insert(SHIPPED, { name: 'w-1' }, SYSTEM); + // The boot bridge's copy of a CODE object: the baseline tier 2 restores. + expect(await k.metadata.get('object', SHIPPED)) + .toMatchObject({ label: 'Shipped', _packageId: SHIPPED_PKG, _provenance: 'package' }); + const plainKey = () => ((k.engine.registry as any).metadata as Map>) + .get('object')?.get(SHIPPED); + expect(plainKey()).toBeUndefined(); + + // No row is stored, so the delete runs the heal alone. + const del = await k.protocol.deleteMetaItem({ type: 'object', name: SHIPPED }); + + expect(del).toMatchObject({ success: true, reset: false }); + // Tier 2 re-registered the baseline, and nothing was retired. + expect(plainKey()).toMatchObject({ label: 'Shipped' }); + expect(await served(k, SHIPPED)).toEqual({ + owner: SHIPPED_PKG, dataPlaneLabel: 'Shipped', metaReadLabel: 'Shipped', records: 1, + }); + expect(drops).toEqual([]); + }); +});