fix(core): a rejecting sys_api_key read is the store outage (503), not "no key", so a cookie beside the key never answers for it

.changeset/24299-api-key-read-outage.md

@@ -0,0 +1,22 @@
+---
+'@objectstack/core': minor
+---
+
+fix(core)!: an API key whose `sys_api_key` read fails is answered as the store outage (`503 SERVICE_UNAVAILABLE`), never as "no key", so the request no longer falls on to a cookie sent beside the key
+
+Clause-②: no (narrowing)
+
+<!-- adr-0087: not-required (no-migration-prescription) Nothing an author writes or stores moves: the change is the answer a request gets when the engine's read of `sys_api_key` throws. No spec key, export, option, object definition or stored row is added, removed, renamed or re-shaped, so there is nothing for `objectstack migrate meta` to rewrite. -->
+
+**BREAKING** (a served-set narrowing), shipped as `minor` under the launch-window convention for breaking changes.
+
+`resolveApiKeyAdmission` (and `resolveApiKeyPrincipal` and `resolveAuthzContext`, which call it) read the key's row in a bare `catch` that answered "this request carries no key", and logged nothing. Two wrong answers followed on every door that admits API keys:
+
+- **A key-only caller got `401 UNAUTHENTICATED`** during the outage, where a session caller got `503 SERVICE_UNAVAILABLE` for the same fault.
+- **A cookie sent beside the key answered for the request.** "No key" sent the request on to the session arm, so a request that led with a key was served as the cookie's user. Measured on the REST data door and the runtime dispatcher (served as the cookie's user) and on the settings door (`200`).
+
+The read now raises the branded `AuthzStoreUnavailableError` (object `sys_api_key`, the driver's error kept as `cause`), the answer the same function already gave when its lifecycle-policy or owner-posture source failed. Every door relays it as `503 SERVICE_UNAVAILABLE`, and the session is never read in the key's place.
+
+- **Unchanged.** A `sys_api_key` table that was never provisioned is still "no key": no key can exist there. This is the boundary `resolveAuthzContext` already draws for its permission reads (the shared `isMissingTableError`). No key header, no engine, no matching row and a row with no `user_id` also still answer "no key".
+
+What to do. Nothing, unless a client treated `401` during a database outage as "the key is bad": it now sees `503` and should retry, not rotate the key.

packages/core/src/security/api-key-read-outage.test.ts

@@ -0,0 +1,142 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * [#24299] A `sys_api_key` read that THROWS is the branded store outage at the
+ * shared key-admission seam (`resolveApiKeyAdmission`), never "no key".
+ *
+ * Before: the read sat in a bare `catch { return { outcome: 'none' }; }`. A key
+ * holder got the anonymous floor's `401` where a session caller got `503`, and
+ * `none` let the request fall on to the session arm, so a cookie sent beside
+ * the key answered for it (measured: served as the cookie's user).
+ *
+ * What these pin:
+ *  - a rejecting read raises `AuthzStoreUnavailableError` (`SERVICE_UNAVAILABLE`
+ *    / 503) naming `sys_api_key`, with the driver's error kept as `cause`;
+ *  - through `resolveAuthzContext`, a cookie beside that key never answers:
+ *    the session read is never asked;
+ *  - a NEVER-PROVISIONED `sys_api_key` is still "no key", the boundary the
+ *    2026-08-30 ruling (option A) draws for `tryFind` — and a missing-table
+ *    phrase naming another relation stays loud;
+ *  - the other `none` exits are unchanged: no header, no engine, no row, a
+ *    row with no `user_id`.
+ */
+
+import { describe, expect, it } from 'vitest';
+
+import { hashApiKey, resolveApiKeyAdmission } from './api-key.js';
+import {
+  AUTHZ_STORE_UNAVAILABLE_CODE,
+  AUTHZ_STORE_UNAVAILABLE_STATUS,
+  isAuthzStoreUnavailableError,
+} from './authz-store-unavailable.js';
+import { resolveAuthzContext } from './resolve-authz-context.js';
+
+const RAW = 'osk_read_outage_key';
+const OWNER = 'usr_key_owner';
+const KEY_ROW = { id: 'key_1', key: hashApiKey(RAW), user_id: OWNER, revoked: false, scopes: '[]' };
+const headers = (extra: Record<string, string> = {}) => ({ 'x-api-key': RAW, ...extra });
+
+/** `sys_api_key` reads throw `fault()`; every other table is empty. */
+function qlWhoseKeyReadThrows(fault: () => unknown) {
+  return {
+    find: async (object: string) => {
+      if (object === 'sys_api_key') throw fault();
+      return [];
+    },
+  };
+}
+
+/** Equality and `$in` — the two shapes the resolver's reads use here; any other operator is refused, never matched. */
+function matchesWhere(row: any, where: any): boolean {
+  for (const [field, cond] of Object.entries(where ?? {})) {
+    if (field.startsWith('$')) throw new Error(`fixture where-matcher: unsupported combinator '${field}'`);
+    if (cond !== null && typeof cond === 'object') {
+      const ops = Object.keys(cond as object);
+      if (ops.length !== 1 || ops[0] !== '$in' || !Array.isArray((cond as any).$in)) {
+        throw new Error(`fixture where-matcher: unsupported operator shape on '${field}'`);
+      }
+      if (!(cond as any).$in.includes(row[field])) return false;
+      continue;
+    }
+    if (row[field] !== cond) return false;
+  }
+  return true;
+}
+
+/** `sys_api_key` holds `rows`, matched by `where` and bounded by `limit`. */
+function qlWithKeyRows(rows: any[]) {
+  return {
+    find: async (object: string, q: any = {}) => {
+      const table = object === 'sys_api_key' ? rows : [];
+      const found = table.filter((r) => matchesWhere(r, q?.where));
+      return typeof q?.limit === 'number' ? found.slice(0, q.limit) : found;
+    },
+  };
+}
+
+const settle = (p: Promise<unknown>) => p.then((v) => ({ ok: true as const, v }), (e) => ({ ok: false as const, e }));
+
+describe('[#24299] a rejecting sys_api_key read is the store outage, not "no key"', () => {
+  it('raises AuthzStoreUnavailableError (SERVICE_UNAVAILABLE / 503) naming sys_api_key, the driver error kept as cause', async () => {
+    const driverFault = new Error('connection terminated unexpectedly');
+    const r = await settle(resolveApiKeyAdmission(qlWhoseKeyReadThrows(() => driverFault), headers(), Date.now(), undefined, undefined, undefined));
+    expect(r.ok, 'the admission RESOLVED — the failed read was answered as a verdict').toBe(false);
+    const e = (r as any).e;
+    expect(isAuthzStoreUnavailableError(e)).toBe(true);
+    expect(e.code).toBe(AUTHZ_STORE_UNAVAILABLE_CODE);
+    expect(e.status).toBe(AUTHZ_STORE_UNAVAILABLE_STATUS);
+    expect(e.object).toBe('sys_api_key');
+    expect(e.cause).toBe(driverFault);
+  });
+
+  it('through resolveAuthzContext, a cookie beside the key never answers: the session is never read', async () => {
+    let sessionReads = 0;
+    const r = await settle(resolveAuthzContext({
+      ql: qlWhoseKeyReadThrows(() => new Error('engine down')),
+      headers: headers({ cookie: 'session=other-user' }),
+      getSession: async () => { sessionReads += 1; return { user: { id: 'usr_cookie' }, session: { userId: 'usr_cookie' } } as any; },
+      userAuthGates: undefined,
+      apiKeyLifecycle: undefined,
+    }));
+    expect(r.ok, 'the context RESOLVED — the request fell on to another credential').toBe(false);
+    expect(isAuthzStoreUnavailableError((r as any).e)).toBe(true);
+    expect((r as any).e.object).toBe('sys_api_key');
+    expect(sessionReads).toBe(0);
+  });
+
+  it('a NEVER-PROVISIONED sys_api_key is still "no key" (ruling 2026-08-30, option A)', async () => {
+    for (const fault of [
+      () => new Error('no such table: sys_api_key'),
+      () => Object.assign(new Error('relation "sys_api_key" does not exist'), { code: '42P01' }),
+    ]) {
+      const a = await resolveApiKeyAdmission(qlWhoseKeyReadThrows(fault), headers(), Date.now(), undefined, undefined, undefined);
+      expect(a).toEqual({ outcome: 'none' });
+    }
+  });
+
+  it('a missing-table phrase naming ANOTHER relation is no evidence about sys_api_key, and stays loud', async () => {
+    const r = await settle(resolveApiKeyAdmission(
+      qlWhoseKeyReadThrows(() => new Error('no such table: main.some_other_base')),
+      headers(), Date.now(), undefined, undefined, undefined,
+    ));
+    expect(r.ok).toBe(false);
+    expect(isAuthzStoreUnavailableError((r as any).e)).toBe(true);
+  });
+});
+
+describe('[#24299] the other "none" exits are unchanged', () => {
+  it('no key header, no engine, an engine with no find, no row, and a row with no user_id each answer none', async () => {
+    const args = [Date.now(), undefined, undefined, undefined] as const;
+    expect(await resolveApiKeyAdmission(qlWithKeyRows([KEY_ROW]), {}, ...args)).toEqual({ outcome: 'none' });
+    expect(await resolveApiKeyAdmission(undefined, headers(), ...args)).toEqual({ outcome: 'none' });
+    expect(await resolveApiKeyAdmission({}, headers(), ...args)).toEqual({ outcome: 'none' });
+    expect(await resolveApiKeyAdmission(qlWithKeyRows([]), headers(), ...args)).toEqual({ outcome: 'none' });
+    expect(await resolveApiKeyAdmission(qlWithKeyRows([{ ...KEY_ROW, user_id: undefined }]), headers(), ...args)).toEqual({ outcome: 'none' });
+  });
+
+  it('control: the same key, read healthily, is admitted as its owner', async () => {
+    const a = await resolveApiKeyAdmission(qlWithKeyRows([KEY_ROW]), headers(), Date.now(), undefined, undefined, undefined);
+    expect(a.outcome).toBe('admitted');
+    expect((a as any).principal.userId).toBe(OWNER);
+  });
+});

packages/core/src/security/api-key.ts

@@ -28,6 +28,7 @@ import type { TenancyPosture } from '@objectstack/spec/security';
 import type { ApiKeyLifecyclePolicy } from '@objectstack/spec/contracts';
 
 import { accountScopeAuthGate, type AuthGate, type UserAuthGateSource } from './auth-gate.js';
+import { isMissingTableError } from '@objectstack/types';
 import { AuthzStoreUnavailableError } from './authz-store-unavailable.js';
 
 /** Default visible prefix for generated keys (helps users identify a key). */
@@ -587,10 +588,12 @@ export function effectiveTenancyPosture(
  * the verdict of {@link resolveApiKeyAdmission}, collapsed to a principal.
  *
  * Fail-closed: returns `undefined` for a missing key, an unusable data engine,
- * a lookup error, a key that is unknown / owner-less, a key its own lifecycle
+ * a key that is unknown / owner-less, a key its own lifecycle
  * refuses (revoked / expired / inactive — `lifecycle`), a key the tenancy
  * posture refuses, and a key whose OWNER holds an `account`-scope ADR-0069
- * posture (`held`).
+ * posture (`held`). A `sys_api_key` read that THROWS is no answer at all: it
+ * raises the branded `AuthzStoreUnavailableError` (#24299), as an unreadable
+ * lifecycle-policy or posture source already does.
  *
  * ⚠️ The collapse is lossy by construction: a principal-or-nothing answer cannot
  * carry the posture's code, so a host that admits keys through this function
@@ -696,8 +699,22 @@ export async function resolveApiKeyAdmission(
       limit: 1,
       context: { isSystem: true },
     });
-  } catch {
-    return { outcome: 'none' };
+  } catch (err) {
+    // [#24299] A read that was ISSUED and THREW is the branded store outage,
+    // never "no key". `none` here sent a key holder the anonymous floor's
+    // `401` where a session caller got `503`, and let the request fall on to
+    // the session arm, so a cookie sent beside the key answered for a
+    // credential the caller did not lead with (measured: served as the
+    // cookie's user on the REST data door and the runtime dispatcher).
+    //
+    // The one read failure that IS a true "no key" is a `sys_api_key` table
+    // that was never provisioned: no key can exist there, so nothing was
+    // withheld. That is the boundary the 2026-08-30 ruling (option A) draws
+    // for `resolveAuthzContext`'s `tryFind`, asked through the same
+    // `isMissingTableError` predicate — ⛔ never a second classifier here;
+    // its signed-off false-positive risk is written beside `tryFind`.
+    if (isMissingTableError(err, 'sys_api_key')) return { outcome: 'none' };
+    throw new AuthzStoreUnavailableError('sys_api_key', err);
   }
   if (rows && (rows as any).value) rows = (rows as any).value;
   const row = Array.isArray(rows) ? rows[0] : undefined;

packages/rest/src/rest-api-key-read-outage.pin.test.ts

@@ -0,0 +1,162 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * [#24299] The REST data door answers a `sys_api_key` read that REJECTS as the
+ * store outage — `503 SERVICE_UNAVAILABLE`, the answer a session caller
+ * already got for the same fault — on the real `GET /api/v1/data/:object`.
+ *
+ * Measured on the unrepaired tree through this harness, `sys_api_key` reads
+ * rejecting and every other table healthy:
+ *
+ *   caller                          | before                      | after
+ *   --------------------------------|-----------------------------|------------------------
+ *   key only                        | 401 UNAUTHENTICATED         | 503 SERVICE_UNAVAILABLE
+ *   key + another user's cookie     | 200, served AS the cookie's | 503, the service is
+ *                                   | user                        | never called
+ *   session only                    | 200 as the cookie's user    | unchanged
+ *
+ * The middle row is the defect's worse half: `resolveApiKeyAdmission` answered
+ * the failed read "no key", so the request fell on to the session arm and a
+ * credential the caller did not lead with answered for it.
+ *
+ * Controls: a healthy key is served as its owner, and a NEVER-PROVISIONED
+ * `sys_api_key` stays "no key" (the anonymous floor's 401), the boundary the
+ * 2026-08-30 ruling (option A) draws.
+ */
+
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+import { hashApiKey } from '@objectstack/core';
+import { RestServer } from './rest-server.js';
+
+const KEY = 'osk_rest_read_outage_pin';
+const KEY_OWNER = 'u_key_owner';
+const COOKIE_USER = 'u_cookie';
+const DATA_COLLECTION = '/api/v1/data/:object';
+
+type KeyRead = 'healthy' | 'rejects' | 'unprovisioned';
+
+/** Equality and `$in` — the two shapes the resolver's reads use here; any other operator is refused, never matched. */
+function matchesWhere(row: any, where: any): boolean {
+  for (const [field, cond] of Object.entries(where ?? {})) {
+    if (field.startsWith('$')) throw new Error(`fixture where-matcher: unsupported combinator '${field}'`);
+    if (cond !== null && typeof cond === 'object') {
+      const ops = Object.keys(cond as object);
+      if (ops.length !== 1 || ops[0] !== '$in' || !Array.isArray((cond as any).$in)) {
+        throw new Error(`fixture where-matcher: unsupported operator shape on '${field}'`);
+      }
+      if (!(cond as any).$in.includes(row[field])) return false;
+      continue;
+    }
+    if (row[field] !== cond) return false;
+  }
+  return true;
+}
+
+function makeQl(keyRead: KeyRead) {
+  const tables: Record<string, any[]> = {
+    sys_api_key: [{ id: 'key_1', key: hashApiKey(KEY), user_id: KEY_OWNER, revoked: false, scopes: '[]' }],
+    sys_member: [],
+    sys_user: [{ id: KEY_OWNER }, { id: COOKIE_USER }],
+    sys_user_permission_set: [],
+    sys_permission_set: [],
+  };
+  return {
+    find: async (object: string, q: any = {}) => {
+      if (object === 'sys_api_key' && keyRead === 'rejects') throw new Error('connection terminated unexpectedly');
+      if (object === 'sys_api_key' && keyRead === 'unprovisioned') throw new Error('no such table: sys_api_key');
+      const rows = (tables[object] ?? []).filter((row: any) => matchesWhere(row, q?.where));
+      return typeof q?.limit === 'number' ? rows.slice(0, q.limit) : rows;
+    },
+    update: async () => ({}),
+  };
+}
+
+/** The single-kernel wiring: an auth provider whose session read answers the cookie's user, and the objectql provider. */
+function setup(keyRead: KeyRead) {
+  const servedAs: Array<string | undefined> = [];
+  const protocol: any = {
+    getDiscovery: vi.fn().mockResolvedValue({ version: 'v0', routes: { data: '', metadata: '', ui: '', auth: '/auth' } }),
+    getMetaTypes: vi.fn().mockResolvedValue([]),
+    getMetaItems: vi.fn().mockResolvedValue([]),
+    getMetaItem: vi.fn().mockResolvedValue({}),
+    findData: vi.fn(async (r: any) => { servedAs.push(r?.context?.userId); return { value: [], total: 0 }; }),
+  };
+  const server: any = {
+    get: vi.fn(), post: vi.fn(), put: vi.fn(), delete: vi.fn(), patch: vi.fn(), use: vi.fn(),
+    listen: vi.fn().mockResolvedValue(undefined), close: vi.fn().mockResolvedValue(undefined),
+  };
+  const authServiceProvider = async () => ({
+    api: {
+      getSession: async ({ headers }: { headers: Headers }) => (
+        headers.get('cookie') ? { user: { id: COOKIE_USER }, session: { userId: COOKIE_USER } } : null
+      ),
+    },
+  });
+  const ql = makeQl(keyRead);
+  const rest = new RestServer(server, protocol, {} as any, undefined, undefined, undefined, authServiceProvider, async () => ql);
+  rest.registerRoutes();
+  return { rest, servedAs };
+}
+
+async function get(rest: any, headers: Record<string, string>) {
+  const res: any = { statusCode: 200, body: undefined };
+  res.status = (c: number) => { res.statusCode = c; return res; };
+  res.json = (b: any) => { res.body = b; return res; };
+  res.header = () => res; res.setHeader = () => {}; res.write = () => {}; res.end = () => {}; res.send = () => {};
+  const route = rest.getRoutes().find((r: any) => r.method === 'GET' && r.path === DATA_COLLECTION);
+  if (!route) throw new Error(`GET ${DATA_COLLECTION} route not registered`);
+  await route.handler({ method: 'GET', path: '/api/v1/data/account', params: { object: 'account' }, query: {}, headers }, res);
+  return { status: res.statusCode as number, body: res.body };
+}
+
+// This file silences the REST fault log it does not assert on, so it declares the level it runs at.
+beforeAll(() => { vi.stubEnv('OS_REST_LOG', 'info'); });
+afterAll(() => { vi.unstubAllEnvs(); });
+
+let errorSpy: ReturnType<typeof vi.spyOn>;
+let warnSpy: ReturnType<typeof vi.spyOn>;
+beforeEach(() => {
+  errorSpy = vi.spyOn(console, 'error').mockImplementation(() => {});
+  warnSpy = vi.spyOn(console, 'warn').mockImplementation(() => {});
+});
+afterEach(() => { errorSpy.mockRestore(); warnSpy.mockRestore(); });
+
+describe('[#24299] REST data door — a rejecting sys_api_key read is the store outage', () => {
+  it('key only: 503 SERVICE_UNAVAILABLE, not the anonymous floor\'s 401', async () => {
+    const { rest, servedAs } = setup('rejects');
+    const { status, body } = await get(rest, { 'x-api-key': KEY });
+    expect(status).toBe(503);
+    expect(body?.code).toBe('SERVICE_UNAVAILABLE');
+    expect(servedAs).toEqual([]);
+  });
+
+  it('key + another user\'s cookie: 503, and the request is never served as the cookie\'s user', async () => {
+    const { rest, servedAs } = setup('rejects');
+    const { status, body } = await get(rest, { 'x-api-key': KEY, cookie: 'session=other-user' });
+    expect(status).toBe(503);
+    expect(body?.code).toBe('SERVICE_UNAVAILABLE');
+    expect(servedAs, 'the data service was called — a credential the caller did not lead with answered').toEqual([]);
+  });
+
+  it('session only: unchanged — the same fault does not touch a caller who sends no key', async () => {
+    const { rest, servedAs } = setup('rejects');
+    const { status } = await get(rest, { cookie: 'session=other-user' });
+    expect(status).toBe(200);
+    expect(servedAs).toEqual([COOKIE_USER]);
+  });
+
+  it('control: a healthy key read is served as the key\'s owner, cookie or not', async () => {
+    const { rest, servedAs } = setup('healthy');
+    expect((await get(rest, { 'x-api-key': KEY })).status).toBe(200);
+    expect((await get(rest, { 'x-api-key': KEY, cookie: 'session=other-user' })).status).toBe(200);
+    expect(servedAs).toEqual([KEY_OWNER, KEY_OWNER]);
+  });
+
+  it('control: a never-provisioned sys_api_key is still "no key" — the anonymous floor\'s 401', async () => {
+    const { rest, servedAs } = setup('unprovisioned');
+    const { status, body } = await get(rest, { 'x-api-key': KEY });
+    expect(status).toBe(401);
+    expect(body?.code).toBe('UNAUTHENTICATED');
+    expect(servedAs).toEqual([]);
+  });
+});