fix(driver-memory): a composed ReadableIds leaf hashes its readable ids once per query, not once per row (ADR-0139 D3)

.changeset/23679-memory-readable-ids-hashed-once.md

@@ -0,0 +1,12 @@
+---
+'@objectstack/driver-memory': patch
+---
+
+fix(driver-memory): a composed `ReadableIds` leaf hashes its readable ids once per query, not once per row, so a read under a large readable-parent set no longer costs rows × parents (ADR-0139 D3)
+
+Clause-②: no
+
+- **Before.** The driver rewrote a composed `ReadableIds` leaf into the ids of every readable parent row, with no cap, and handed mingo that list as a plain `$in`. mingo's `$in` rebuilds a hash map of its whole list for every document it tests. So a read over R child rows under N readable parents made R × N hash inserts: with 2,700 rows and 2,400 readable parents, one `find`, `count` or `aggregate` took about a second.
+- **After.** The leaf's list lowers to a member test that hashes it once, when the query compiles. The same read takes milliseconds. This holds on every door that evaluates the leaf (`find`, `count`, `aggregate`, `distinct`, `updateMany`, `deleteMany`) and under `$and`, `$or` and `$not`.
+- **Answers are unchanged.** The member test keeps every rule of mingo's `$in`: a nil value matches only a literal `null` member, an array value matches when any element is a member, objects compare by structure, and string values are tested against RegExp members.
+- **Scope.** Every other `$in` is still mingo's own operator. Only the list the leaf was rewritten into takes the new test. The test's operator name cannot be written in a filter: the filter shape gate refuses it.

packages/drivers/driver-memory/src/memory-driver.ts

@@ -23,7 +23,11 @@ import { Logger, createLogger, compensatedSum } from '@objectstack/core';
 // one source for every driver.
 import { findMisplacedReadableIds, refuseReadableIdsOnce, replaceComposedReadableIds } from '@objectstack/core';
 import type { ComposedReadableIds } from '@objectstack/spec/data';
-import { Query, Aggregator } from 'mingo';
+import { Aggregator } from 'mingo';
+// [ADR-0139 D3] mingo's `Query`, able to compile the member test a composed
+// `ReadableIds` leaf lowers to: its readable ids hashed once per query, not
+// once per row (#23679). Every other operator is mingo's.
+import { Query, READABLE_IDS_MEMBER_OP, isReadableIdsList, markReadableIdsList } from './memory-readable-ids-member.js';
 import {
   assertSingleTenantPosture,
   assertObjectsNotTenantScoped,
@@ -1487,7 +1491,9 @@ export class InMemoryDriver implements IDataDriver {
     let rows: Record<string, any>[] = this.getTable(leaf.object);
     const mongoQuery = this.convertToMongoQuery(leaf.scope, leaf.object);
     if (mongoQuery && Object.keys(mongoQuery).length > 0) rows = new Query(mongoQuery).find(rows).all() as Record<string, any>[];
-    return rows.map((row) => row.id);
+    // [#23679] Marked, so the `$in` arm lowers this list to the member test
+    // that hashes it once (`memory-readable-ids-member.ts`).
+    return markReadableIdsList(rows.map((row) => row.id));
   }
 
   /**
@@ -1890,7 +1896,13 @@ export class InMemoryDriver implements IDataDriver {
         // `$between` arm above).
         case '$eq': case '$ne': case '$gt': case '$gte': case '$lt': case '$lte':
         case '$in': case '$nin':
-          put(op, store(val));
+          // [ADR-0139 D3, #23679] The list a composed `ReadableIds` leaf was
+          // rewritten into is answered by a member test that hashes it once per
+          // query. mingo's `$in` re-hashes its whole list for every row tested,
+          // which made a read cost rows × readable parents. Only the D3 leaf's
+          // list carries the mark; every other `$in` stays mingo's.
+          if (op === '$in' && isReadableIdsList(val)) put(READABLE_IDS_MEMBER_OP, store(val));
+          else put(op, store(val));
           break;
         // [commit 9dac1ae01] `$exists` means "the field HAS A VALUE" (`!= null`), never
         // key presence — #5298 leg 3 / #5369, landed in PR #5962, and ruled

packages/drivers/driver-memory/src/memory-readable-ids-hashed-once.test.ts

@@ -0,0 +1,226 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * [ADR-0139 D3, #23679] A composed `ReadableIds` leaf hashes its readable ids
+ * once per query, never once per row.
+ *
+ * The leaf is rewritten into the ids of every readable parent row. Handed to
+ * mingo as a plain `$in`, that list was re-hashed for every row tested, so a
+ * read over R rows under N readable parents made R × N hash inserts. The E3
+ * conformance pins (`memory-readable-ids-conformance.test.ts`) took 2–3 s per
+ * test on an idle box, within reach of vitest's 5 s default under CI's
+ * per-core worker share, and `main`'s hourly full run went red on them.
+ * `memory-readable-ids-member.ts` lowers the leaf's list to a member test that
+ * hashes it when the query compiles.
+ *
+ * Three parts, all deterministic, with no wall clock anywhere:
+ *   - COST: the `HashMap#set` calls a composed-leaf read makes, counted on
+ *     mingo's own class, are linear in rows + ids on every read door, and under
+ *     every combinator. The bound is a tenth of R × N, and the per-row hashing
+ *     made R × N.
+ *   - ANSWERS: over a matrix of values, lists and selectors, the member test
+ *     answers exactly what mingo's `$in` answers, and the write doors answer a
+ *     composed leaf as they did.
+ *   - NOT AUTHORABLE: a filter naming the private operator is refused by the
+ *     shape gate.
+ */
+
+import { describe, it, expect, beforeAll, afterEach, vi } from 'vitest';
+import { Query as MingoQuery } from 'mingo';
+import { HashMap } from 'mingo/util';
+import { ReadableIds, lowerFilterCondition } from '@objectstack/spec/data';
+import type { DriverQuery } from '@objectstack/spec/contracts';
+import { InMemoryDriver } from './memory-driver.js';
+import { Query, READABLE_IDS_MEMBER_OP, isReadableIdsList, markReadableIdsList } from './memory-readable-ids-member.js';
+
+const seamed = (where: unknown): DriverQuery['where'] =>
+  lowerFilterCondition(where, { isDatetimeColumn: () => false }) as DriverQuery['where'];
+
+const TEXT = (name: string) => ({ type: 'text', name });
+const PARENT = 'hashed_once_parent';
+const CHILD = 'hashed_once_child';
+const ROWS = 500;
+const READABLE = 400;
+/** What the per-row hashing made for the list alone: one insert per readable id, per row tested. */
+const PER_ROW_INSERTS = ROWS * READABLE;
+const LINEAR_BOUND = PER_ROW_INSERTS / 10;
+
+const parentId = (i: number) => `p${String(i).padStart(4, '0')}`;
+const composed = () => ReadableIds.compose(ReadableIds.of(PARENT), { owner: 'reader' });
+
+async function seed(): Promise<InMemoryDriver> {
+  const driver = new InMemoryDriver({ persistence: false });
+  await driver.connect();
+  await driver.syncSchema(PARENT, { fields: { id: TEXT('id'), owner: TEXT('owner') } });
+  await driver.syncSchema(CHILD, { fields: { id: TEXT('id'), record_id: TEXT('record_id'), label: TEXT('label') } });
+  for (let i = 0; i < ROWS; i++) {
+    await driver.create(PARENT, { id: parentId(i), owner: i < READABLE ? 'reader' : 'someone_else' });
+    await driver.create(CHILD, { id: `c${String(i).padStart(4, '0')}`, record_id: parentId(i), label: 'x' });
+  }
+  return driver;
+}
+
+/** The `HashMap#set` calls `read` makes, counted on mingo's own class. */
+async function insertsDuring<T>(read: () => Promise<T>): Promise<{ result: T; inserts: number }> {
+  const set = vi.spyOn(HashMap.prototype, 'set');
+  try {
+    const result = await read();
+    return { result, inserts: set.mock.calls.length };
+  } finally {
+    set.mockRestore();
+  }
+}
+
+describe('[ADR-0139 D3] InMemoryDriver — a composed ReadableIds leaf hashes its ids once per query', () => {
+  let driver: InMemoryDriver;
+
+  beforeAll(async () => {
+    driver = await seed();
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  describe('cost: linear in rows + ids, on every read door and under every combinator', () => {
+    const where = () => seamed({ record_id: { $in: composed() } });
+
+    it('find', async () => {
+      const { result, inserts } = await insertsDuring(() => driver.find(CHILD, { where: where() }));
+      expect(result).toHaveLength(READABLE);
+      expect(inserts).toBeLessThan(LINEAR_BOUND);
+    });
+
+    it('count', async () => {
+      const { result, inserts } = await insertsDuring(() => driver.count(CHILD, { where: where() }));
+      expect(result).toBe(READABLE);
+      expect(inserts).toBeLessThan(LINEAR_BOUND);
+    });
+
+    it('aggregate', async () => {
+      const { result, inserts } = await insertsDuring(() =>
+        driver.aggregate(CHILD, { where: where(), aggregations: [{ function: 'count', alias: 'n' }] }),
+      );
+      expect(Number((result[0] as Record<string, unknown>).n)).toBe(READABLE);
+      expect(inserts).toBeLessThan(LINEAR_BOUND);
+    });
+
+    it('distinct', async () => {
+      const { result, inserts } = await insertsDuring(() => driver.distinct(CHILD, 'record_id', { where: where() }));
+      expect(result).toHaveLength(READABLE);
+      expect(inserts).toBeLessThan(LINEAR_BOUND);
+    });
+
+    it('under $not: the leaf compiles inside the sub-query mingo builds for $nor', async () => {
+      const { result, inserts } = await insertsDuring(() =>
+        driver.find(CHILD, { where: seamed({ $not: { record_id: { $in: composed() } } }) }),
+      );
+      expect(result).toHaveLength(ROWS - READABLE);
+      expect(inserts).toBeLessThan(LINEAR_BOUND);
+    });
+
+    it('under $or', async () => {
+      const { result, inserts } = await insertsDuring(() =>
+        driver.find(CHILD, { where: seamed({ $or: [{ record_id: { $in: composed() } }, { id: 'c0499' }] }) }),
+      );
+      expect(result).toHaveLength(READABLE + 1);
+      expect(inserts).toBeLessThan(LINEAR_BOUND);
+    });
+  });
+
+  describe('answers', () => {
+    it('the write doors answer a composed leaf as before: updateMany, then deleteMany', async () => {
+      const fresh = await seed();
+      const where = seamed({ record_id: { $in: composed() } });
+      expect(await fresh.updateMany(CHILD, { where }, { label: 'readable' })).toBe(READABLE);
+      expect(await fresh.count(CHILD, { where: seamed({ label: 'readable' }) })).toBe(READABLE);
+      expect(await fresh.deleteMany(CHILD, { where })).toBe(READABLE);
+      expect(await fresh.count(CHILD, {})).toBe(ROWS - READABLE);
+    });
+
+    it("only a list the driver marked is the leaf's: a filter-written array is not", () => {
+      expect(isReadableIdsList(['p0000'])).toBe(false);
+      expect(isReadableIdsList(markReadableIdsList(['p0000']))).toBe(true);
+    });
+
+    const VALUES: Array<[string, unknown]> = [
+      ['a string', 'a'],
+      ['another string', 'abc'],
+      ['a number', 1],
+      ['a numeric string', '1'],
+      ['a boolean', true],
+      ['null', null],
+      ['an array of strings', ['a', 'z']],
+      ['an empty array', []],
+      ['an array holding null', [null]],
+      ['an object', { x: 1 }],
+      ['an array of objects', [{ x: 1 }]],
+      ['a date', new Date(0)],
+      ['a nested array', [['a', 'z']]],
+    ];
+    const LISTS: Array<[string, unknown[]]> = [
+      ['empty', []],
+      ['one string', ['a']],
+      ['one number', [1]],
+      ['one numeric string', ['1']],
+      ['a number and its string', [1, '1']],
+      ['null', [null]],
+      ['null and a string', [null, 'a']],
+      ['a RegExp', [/^a/]],
+      ['a string and a RegExp', ['z', /b/]],
+      ['an object', [{ x: 1 }]],
+      ['an array', [['a', 'z']]],
+      ['a date', [new Date(0)]],
+      ['a boolean', [true]],
+    ];
+    const DOCS = (value: unknown): Array<[string, string, Record<string, unknown>]> => [
+      ['top-level field', 'f', { f: value }],
+      ['missing field', 'f', { g: value }],
+      ['dotted path', 'g.f', { g: { f: value } }],
+      ['dotted path through an array of subdocuments', 'g.f', { g: [{ f: value }, { f: 'b' }] }],
+    ];
+
+    for (const [listName, list] of LISTS) {
+      it(`the member test answers what mingo's $in answers — list: ${listName}`, () => {
+        for (const [valueName, value] of VALUES) {
+          for (const [docName, selector, doc] of DOCS(value)) {
+            const expected = new MingoQuery({ [selector]: { $in: list } }).test(doc);
+            const actual = new Query({ [selector]: { [READABLE_IDS_MEMBER_OP]: list } }).test(doc);
+            expect(actual, `${valueName} at ${docName}`).toBe(expected);
+          }
+        }
+      });
+    }
+
+    it('a compiled member test answers the same on every row it is asked (the hash is built once, then only read)', () => {
+      const query = new Query({ f: { [READABLE_IDS_MEMBER_OP]: ['a', 1, null, /^z/] } });
+      const docs = [{ f: 'a' }, { f: 2 }, {}, { f: 'zed' }, { f: ['q', 1] }];
+      const first = docs.map((d) => query.test(d));
+      expect(docs.map((d) => query.test(d))).toEqual(first);
+      expect(first).toEqual([true, false, true, true, true]);
+    });
+
+    it("a comparand that is not an array is mingo's $in to answer, unchanged", () => {
+      const run = (test: () => boolean) => {
+        try {
+          return { matched: test() };
+        } catch (e) {
+          return { threw: (e as Error).message };
+        }
+      };
+      for (const doc of [{ f: 'a' }, {}]) {
+        expect(run(() => new Query({ f: { [READABLE_IDS_MEMBER_OP]: 'a' } }).test(doc))).toEqual(
+          run(() => new MingoQuery({ f: { $in: 'a' } }).test(doc)),
+        );
+      }
+    });
+  });
+
+  describe('not authorable', () => {
+    it('a filter naming the private operator is refused by the shape gate', async () => {
+      await expect(
+        driver.find(CHILD, { where: { record_id: { [READABLE_IDS_MEMBER_OP]: [parentId(0)] } } }),
+      ).rejects.toMatchObject({ code: 'INVALID_FILTER', status: 400 });
+    });
+  });
+});

packages/drivers/driver-memory/src/memory-readable-ids-member.ts

@@ -0,0 +1,101 @@
+// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.
+
+/**
+ * [ADR-0139 D3] The member test a composed `ReadableIds` leaf compiles to in
+ * this driver: the readable parent ids are hashed ONCE, when the query
+ * compiles, and each row is then one lookup.
+ *
+ * ## Why (#23679)
+ *
+ * `convertToMongoQuery` rewrites a composed leaf into the ids of every
+ * readable parent row (`readableParentIds`), with no cap. Handed to mingo as a
+ * plain `$in`, that list was re-hashed for every row tested: mingo's `$in`
+ * runs `intersection([value, list])` per document, and `intersection` builds a
+ * hash map of the WHOLE list on each call. A read over R child rows under N
+ * readable parents therefore made R × N hash inserts. On the E3 conformance
+ * fixture (2,700 rows, 2,400 readable parents) that was about 950 ms of mingo
+ * per read, against 0.8 ms for one prebuilt set. The E3 pins make two and three
+ * such reads per test, so under CI's per-core worker share they crossed
+ * vitest's 5 s default and turned `main`'s hourly full run red.
+ *
+ * ## Scope: the D3 leaf only
+ *
+ * Every other `$in` keeps mingo's own operator. Only a list `readableParentIds`
+ * produced is marked ({@link markReadableIdsList}), and only a marked list is
+ * lowered to {@link READABLE_IDS_MEMBER_OP}. The mark is object identity, held
+ * in a `WeakSet`, so it cannot be written in a filter. The operator name is
+ * not authorable either: the shape gate, which runs before the lowering,
+ * refuses a field operator it does not declare.
+ *
+ * ## The answers are mingo's `$in`
+ *
+ * {@link readableIdsMember} is mingo's `$in` predicate with the loop-invariant
+ * work moved out of the per-document closure, and nothing else changed: the
+ * same `resolve` with `unwrapArray`, the same nil rule (a nil value matches
+ * only a literal `null` member), the same structural `HashMap` membership
+ * `intersection` asks, and the same fallback that tests string values against
+ * RegExp members. `memory-readable-ids-hashed-once.test.ts` holds both
+ * halves: the answers equal mingo's `$in` over a case matrix, and the
+ * hash-insert count of a composed-leaf read is linear.
+ *
+ * ## How mingo finds the operator
+ *
+ * mingo's `Query` merges a caller's context into its default one, and a name
+ * the default does not have is added (an existing name would win, which is why
+ * this is a new name and not a replacement `$in`). {@link Query} passes that
+ * context on every construction, and mingo hands the same options to the
+ * sub-queries it builds for `$and`, `$or` and `$nor`, so a leaf under any
+ * combinator resolves too. A filter that reached mingo without it would fail
+ * loudly (`unknown query operator`), never answer silently.
+ */
+
+import { Context, Query as MingoQuery } from 'mingo';
+import { HashMap, ensureArray, isNil, isRegExp, isString, resolve } from 'mingo/util';
+import type { AnyObject, Criteria, Options } from 'mingo/types';
+import { $in as mingoIn } from 'mingo/operators/query';
+
+/** The private mingo operator a marked `ReadableIds` list lowers to. */
+export const READABLE_IDS_MEMBER_OP = '$readableIdsMember';
+
+const readableIdsLists = new WeakSet<object>();
+
+/** Mark a list `readableParentIds` produced, so the lowering knows it is the D3 leaf's. */
+export function markReadableIdsList<T extends unknown[]>(list: T): T {
+  readableIdsLists.add(list);
+  return list;
+}
+
+/** Whether `value` is a list {@link markReadableIdsList} marked. */
+export function isReadableIdsList(value: unknown): value is unknown[] {
+  return Array.isArray(value) && readableIdsLists.has(value);
+}
+
+/**
+ * mingo's `$in`, with the list hashed once when the query compiles. The
+ * returned closure is the per-document test.
+ */
+export const readableIdsMember: typeof mingoIn = (selector, value, options) => {
+  if (!Array.isArray(value)) return mingoIn(selector, value, options);
+  const list: unknown[] = value;
+  const members = HashMap.init();
+  for (const member of list) members.set(member, true);
+  const nullIsMember = list.some((member) => member === null);
+  const patterns = list.filter(isRegExp);
+  const resolveOptions = { unwrapArray: true };
+  return (doc) => {
+    const lhs = resolve(doc, selector, resolveOptions);
+    if (isNil(lhs)) return nullIsMember;
+    const values = ensureArray(lhs);
+    if (values.some((v) => members.has(v))) return true;
+    return patterns.length > 0 && values.some((v) => isString(v) && patterns.some((re) => re.test(v)));
+  };
+};
+
+const READABLE_IDS_CONTEXT = Context.init({ query: { [READABLE_IDS_MEMBER_OP]: readableIdsMember } });
+
+/** mingo's `Query`, able to compile {@link READABLE_IDS_MEMBER_OP}. Every other operator is mingo's. */
+export class Query<T = AnyObject> extends MingoQuery<T> {
+  constructor(condition: Criteria<T>, options?: Partial<Options>) {
+    super(condition, { ...options, context: READABLE_IDS_CONTEXT });
+  }
+}