diff --git a/.changeset/23679-memory-readable-ids-hashed-once.md b/.changeset/23679-memory-readable-ids-hashed-once.md new file mode 100644 index 00000000000..ae36136ce5f --- /dev/null +++ b/.changeset/23679-memory-readable-ids-hashed-once.md @@ -0,0 +1,12 @@ +--- +'@objectstack/driver-memory': patch +--- + +fix(driver-memory): a composed `ReadableIds` leaf hashes its readable ids once per query, not once per row, so a read under a large readable-parent set no longer costs rows × parents (ADR-0139 D3) + +Clause-②: no + +- **Before.** The driver rewrote a composed `ReadableIds` leaf into the ids of every readable parent row, with no cap, and handed mingo that list as a plain `$in`. mingo's `$in` rebuilds a hash map of its whole list for every document it tests. So a read over R child rows under N readable parents made R × N hash inserts: with 2,700 rows and 2,400 readable parents, one `find`, `count` or `aggregate` took about a second. +- **After.** The leaf's list lowers to a member test that hashes it once, when the query compiles. The same read takes milliseconds. This holds on every door that evaluates the leaf (`find`, `count`, `aggregate`, `distinct`, `updateMany`, `deleteMany`) and under `$and`, `$or` and `$not`. +- **Answers are unchanged.** The member test keeps every rule of mingo's `$in`: a nil value matches only a literal `null` member, an array value matches when any element is a member, objects compare by structure, and string values are tested against RegExp members. +- **Scope.** Every other `$in` is still mingo's own operator. Only the list the leaf was rewritten into takes the new test. The test's operator name cannot be written in a filter: the filter shape gate refuses it. diff --git a/packages/drivers/driver-memory/src/memory-driver.ts b/packages/drivers/driver-memory/src/memory-driver.ts index 4f091158381..d6609e52f3e 100644 --- a/packages/drivers/driver-memory/src/memory-driver.ts +++ b/packages/drivers/driver-memory/src/memory-driver.ts @@ -23,7 +23,11 @@ import { Logger, createLogger, compensatedSum } from '@objectstack/core'; // one source for every driver. import { findMisplacedReadableIds, refuseReadableIdsOnce, replaceComposedReadableIds } from '@objectstack/core'; import type { ComposedReadableIds } from '@objectstack/spec/data'; -import { Query, Aggregator } from 'mingo'; +import { Aggregator } from 'mingo'; +// [ADR-0139 D3] mingo's `Query`, able to compile the member test a composed +// `ReadableIds` leaf lowers to: its readable ids hashed once per query, not +// once per row (#23679). Every other operator is mingo's. +import { Query, READABLE_IDS_MEMBER_OP, isReadableIdsList, markReadableIdsList } from './memory-readable-ids-member.js'; import { assertSingleTenantPosture, assertObjectsNotTenantScoped, @@ -1487,7 +1491,9 @@ export class InMemoryDriver implements IDataDriver { let rows: Record[] = this.getTable(leaf.object); const mongoQuery = this.convertToMongoQuery(leaf.scope, leaf.object); if (mongoQuery && Object.keys(mongoQuery).length > 0) rows = new Query(mongoQuery).find(rows).all() as Record[]; - return rows.map((row) => row.id); + // [#23679] Marked, so the `$in` arm lowers this list to the member test + // that hashes it once (`memory-readable-ids-member.ts`). + return markReadableIdsList(rows.map((row) => row.id)); } /** @@ -1890,7 +1896,13 @@ export class InMemoryDriver implements IDataDriver { // `$between` arm above). case '$eq': case '$ne': case '$gt': case '$gte': case '$lt': case '$lte': case '$in': case '$nin': - put(op, store(val)); + // [ADR-0139 D3, #23679] The list a composed `ReadableIds` leaf was + // rewritten into is answered by a member test that hashes it once per + // query. mingo's `$in` re-hashes its whole list for every row tested, + // which made a read cost rows × readable parents. Only the D3 leaf's + // list carries the mark; every other `$in` stays mingo's. + if (op === '$in' && isReadableIdsList(val)) put(READABLE_IDS_MEMBER_OP, store(val)); + else put(op, store(val)); break; // [commit 9dac1ae01] `$exists` means "the field HAS A VALUE" (`!= null`), never // key presence — #5298 leg 3 / #5369, landed in PR #5962, and ruled diff --git a/packages/drivers/driver-memory/src/memory-readable-ids-hashed-once.test.ts b/packages/drivers/driver-memory/src/memory-readable-ids-hashed-once.test.ts new file mode 100644 index 00000000000..51e87d16cd5 --- /dev/null +++ b/packages/drivers/driver-memory/src/memory-readable-ids-hashed-once.test.ts @@ -0,0 +1,226 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [ADR-0139 D3, #23679] A composed `ReadableIds` leaf hashes its readable ids + * once per query, never once per row. + * + * The leaf is rewritten into the ids of every readable parent row. Handed to + * mingo as a plain `$in`, that list was re-hashed for every row tested, so a + * read over R rows under N readable parents made R × N hash inserts. The E3 + * conformance pins (`memory-readable-ids-conformance.test.ts`) took 2–3 s per + * test on an idle box, within reach of vitest's 5 s default under CI's + * per-core worker share, and `main`'s hourly full run went red on them. + * `memory-readable-ids-member.ts` lowers the leaf's list to a member test that + * hashes it when the query compiles. + * + * Three parts, all deterministic, with no wall clock anywhere: + * - COST: the `HashMap#set` calls a composed-leaf read makes, counted on + * mingo's own class, are linear in rows + ids on every read door, and under + * every combinator. The bound is a tenth of R × N, and the per-row hashing + * made R × N. + * - ANSWERS: over a matrix of values, lists and selectors, the member test + * answers exactly what mingo's `$in` answers, and the write doors answer a + * composed leaf as they did. + * - NOT AUTHORABLE: a filter naming the private operator is refused by the + * shape gate. + */ + +import { describe, it, expect, beforeAll, afterEach, vi } from 'vitest'; +import { Query as MingoQuery } from 'mingo'; +import { HashMap } from 'mingo/util'; +import { ReadableIds, lowerFilterCondition } from '@objectstack/spec/data'; +import type { DriverQuery } from '@objectstack/spec/contracts'; +import { InMemoryDriver } from './memory-driver.js'; +import { Query, READABLE_IDS_MEMBER_OP, isReadableIdsList, markReadableIdsList } from './memory-readable-ids-member.js'; + +const seamed = (where: unknown): DriverQuery['where'] => + lowerFilterCondition(where, { isDatetimeColumn: () => false }) as DriverQuery['where']; + +const TEXT = (name: string) => ({ type: 'text', name }); +const PARENT = 'hashed_once_parent'; +const CHILD = 'hashed_once_child'; +const ROWS = 500; +const READABLE = 400; +/** What the per-row hashing made for the list alone: one insert per readable id, per row tested. */ +const PER_ROW_INSERTS = ROWS * READABLE; +const LINEAR_BOUND = PER_ROW_INSERTS / 10; + +const parentId = (i: number) => `p${String(i).padStart(4, '0')}`; +const composed = () => ReadableIds.compose(ReadableIds.of(PARENT), { owner: 'reader' }); + +async function seed(): Promise { + const driver = new InMemoryDriver({ persistence: false }); + await driver.connect(); + await driver.syncSchema(PARENT, { fields: { id: TEXT('id'), owner: TEXT('owner') } }); + await driver.syncSchema(CHILD, { fields: { id: TEXT('id'), record_id: TEXT('record_id'), label: TEXT('label') } }); + for (let i = 0; i < ROWS; i++) { + await driver.create(PARENT, { id: parentId(i), owner: i < READABLE ? 'reader' : 'someone_else' }); + await driver.create(CHILD, { id: `c${String(i).padStart(4, '0')}`, record_id: parentId(i), label: 'x' }); + } + return driver; +} + +/** The `HashMap#set` calls `read` makes, counted on mingo's own class. */ +async function insertsDuring(read: () => Promise): Promise<{ result: T; inserts: number }> { + const set = vi.spyOn(HashMap.prototype, 'set'); + try { + const result = await read(); + return { result, inserts: set.mock.calls.length }; + } finally { + set.mockRestore(); + } +} + +describe('[ADR-0139 D3] InMemoryDriver — a composed ReadableIds leaf hashes its ids once per query', () => { + let driver: InMemoryDriver; + + beforeAll(async () => { + driver = await seed(); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + describe('cost: linear in rows + ids, on every read door and under every combinator', () => { + const where = () => seamed({ record_id: { $in: composed() } }); + + it('find', async () => { + const { result, inserts } = await insertsDuring(() => driver.find(CHILD, { where: where() })); + expect(result).toHaveLength(READABLE); + expect(inserts).toBeLessThan(LINEAR_BOUND); + }); + + it('count', async () => { + const { result, inserts } = await insertsDuring(() => driver.count(CHILD, { where: where() })); + expect(result).toBe(READABLE); + expect(inserts).toBeLessThan(LINEAR_BOUND); + }); + + it('aggregate', async () => { + const { result, inserts } = await insertsDuring(() => + driver.aggregate(CHILD, { where: where(), aggregations: [{ function: 'count', alias: 'n' }] }), + ); + expect(Number((result[0] as Record).n)).toBe(READABLE); + expect(inserts).toBeLessThan(LINEAR_BOUND); + }); + + it('distinct', async () => { + const { result, inserts } = await insertsDuring(() => driver.distinct(CHILD, 'record_id', { where: where() })); + expect(result).toHaveLength(READABLE); + expect(inserts).toBeLessThan(LINEAR_BOUND); + }); + + it('under $not: the leaf compiles inside the sub-query mingo builds for $nor', async () => { + const { result, inserts } = await insertsDuring(() => + driver.find(CHILD, { where: seamed({ $not: { record_id: { $in: composed() } } }) }), + ); + expect(result).toHaveLength(ROWS - READABLE); + expect(inserts).toBeLessThan(LINEAR_BOUND); + }); + + it('under $or', async () => { + const { result, inserts } = await insertsDuring(() => + driver.find(CHILD, { where: seamed({ $or: [{ record_id: { $in: composed() } }, { id: 'c0499' }] }) }), + ); + expect(result).toHaveLength(READABLE + 1); + expect(inserts).toBeLessThan(LINEAR_BOUND); + }); + }); + + describe('answers', () => { + it('the write doors answer a composed leaf as before: updateMany, then deleteMany', async () => { + const fresh = await seed(); + const where = seamed({ record_id: { $in: composed() } }); + expect(await fresh.updateMany(CHILD, { where }, { label: 'readable' })).toBe(READABLE); + expect(await fresh.count(CHILD, { where: seamed({ label: 'readable' }) })).toBe(READABLE); + expect(await fresh.deleteMany(CHILD, { where })).toBe(READABLE); + expect(await fresh.count(CHILD, {})).toBe(ROWS - READABLE); + }); + + it("only a list the driver marked is the leaf's: a filter-written array is not", () => { + expect(isReadableIdsList(['p0000'])).toBe(false); + expect(isReadableIdsList(markReadableIdsList(['p0000']))).toBe(true); + }); + + const VALUES: Array<[string, unknown]> = [ + ['a string', 'a'], + ['another string', 'abc'], + ['a number', 1], + ['a numeric string', '1'], + ['a boolean', true], + ['null', null], + ['an array of strings', ['a', 'z']], + ['an empty array', []], + ['an array holding null', [null]], + ['an object', { x: 1 }], + ['an array of objects', [{ x: 1 }]], + ['a date', new Date(0)], + ['a nested array', [['a', 'z']]], + ]; + const LISTS: Array<[string, unknown[]]> = [ + ['empty', []], + ['one string', ['a']], + ['one number', [1]], + ['one numeric string', ['1']], + ['a number and its string', [1, '1']], + ['null', [null]], + ['null and a string', [null, 'a']], + ['a RegExp', [/^a/]], + ['a string and a RegExp', ['z', /b/]], + ['an object', [{ x: 1 }]], + ['an array', [['a', 'z']]], + ['a date', [new Date(0)]], + ['a boolean', [true]], + ]; + const DOCS = (value: unknown): Array<[string, string, Record]> => [ + ['top-level field', 'f', { f: value }], + ['missing field', 'f', { g: value }], + ['dotted path', 'g.f', { g: { f: value } }], + ['dotted path through an array of subdocuments', 'g.f', { g: [{ f: value }, { f: 'b' }] }], + ]; + + for (const [listName, list] of LISTS) { + it(`the member test answers what mingo's $in answers — list: ${listName}`, () => { + for (const [valueName, value] of VALUES) { + for (const [docName, selector, doc] of DOCS(value)) { + const expected = new MingoQuery({ [selector]: { $in: list } }).test(doc); + const actual = new Query({ [selector]: { [READABLE_IDS_MEMBER_OP]: list } }).test(doc); + expect(actual, `${valueName} at ${docName}`).toBe(expected); + } + } + }); + } + + it('a compiled member test answers the same on every row it is asked (the hash is built once, then only read)', () => { + const query = new Query({ f: { [READABLE_IDS_MEMBER_OP]: ['a', 1, null, /^z/] } }); + const docs = [{ f: 'a' }, { f: 2 }, {}, { f: 'zed' }, { f: ['q', 1] }]; + const first = docs.map((d) => query.test(d)); + expect(docs.map((d) => query.test(d))).toEqual(first); + expect(first).toEqual([true, false, true, true, true]); + }); + + it("a comparand that is not an array is mingo's $in to answer, unchanged", () => { + const run = (test: () => boolean) => { + try { + return { matched: test() }; + } catch (e) { + return { threw: (e as Error).message }; + } + }; + for (const doc of [{ f: 'a' }, {}]) { + expect(run(() => new Query({ f: { [READABLE_IDS_MEMBER_OP]: 'a' } }).test(doc))).toEqual( + run(() => new MingoQuery({ f: { $in: 'a' } }).test(doc)), + ); + } + }); + }); + + describe('not authorable', () => { + it('a filter naming the private operator is refused by the shape gate', async () => { + await expect( + driver.find(CHILD, { where: { record_id: { [READABLE_IDS_MEMBER_OP]: [parentId(0)] } } }), + ).rejects.toMatchObject({ code: 'INVALID_FILTER', status: 400 }); + }); + }); +}); diff --git a/packages/drivers/driver-memory/src/memory-readable-ids-member.ts b/packages/drivers/driver-memory/src/memory-readable-ids-member.ts new file mode 100644 index 00000000000..e403a200d04 --- /dev/null +++ b/packages/drivers/driver-memory/src/memory-readable-ids-member.ts @@ -0,0 +1,101 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * [ADR-0139 D3] The member test a composed `ReadableIds` leaf compiles to in + * this driver: the readable parent ids are hashed ONCE, when the query + * compiles, and each row is then one lookup. + * + * ## Why (#23679) + * + * `convertToMongoQuery` rewrites a composed leaf into the ids of every + * readable parent row (`readableParentIds`), with no cap. Handed to mingo as a + * plain `$in`, that list was re-hashed for every row tested: mingo's `$in` + * runs `intersection([value, list])` per document, and `intersection` builds a + * hash map of the WHOLE list on each call. A read over R child rows under N + * readable parents therefore made R × N hash inserts. On the E3 conformance + * fixture (2,700 rows, 2,400 readable parents) that was about 950 ms of mingo + * per read, against 0.8 ms for one prebuilt set. The E3 pins make two and three + * such reads per test, so under CI's per-core worker share they crossed + * vitest's 5 s default and turned `main`'s hourly full run red. + * + * ## Scope: the D3 leaf only + * + * Every other `$in` keeps mingo's own operator. Only a list `readableParentIds` + * produced is marked ({@link markReadableIdsList}), and only a marked list is + * lowered to {@link READABLE_IDS_MEMBER_OP}. The mark is object identity, held + * in a `WeakSet`, so it cannot be written in a filter. The operator name is + * not authorable either: the shape gate, which runs before the lowering, + * refuses a field operator it does not declare. + * + * ## The answers are mingo's `$in` + * + * {@link readableIdsMember} is mingo's `$in` predicate with the loop-invariant + * work moved out of the per-document closure, and nothing else changed: the + * same `resolve` with `unwrapArray`, the same nil rule (a nil value matches + * only a literal `null` member), the same structural `HashMap` membership + * `intersection` asks, and the same fallback that tests string values against + * RegExp members. `memory-readable-ids-hashed-once.test.ts` holds both + * halves: the answers equal mingo's `$in` over a case matrix, and the + * hash-insert count of a composed-leaf read is linear. + * + * ## How mingo finds the operator + * + * mingo's `Query` merges a caller's context into its default one, and a name + * the default does not have is added (an existing name would win, which is why + * this is a new name and not a replacement `$in`). {@link Query} passes that + * context on every construction, and mingo hands the same options to the + * sub-queries it builds for `$and`, `$or` and `$nor`, so a leaf under any + * combinator resolves too. A filter that reached mingo without it would fail + * loudly (`unknown query operator`), never answer silently. + */ + +import { Context, Query as MingoQuery } from 'mingo'; +import { HashMap, ensureArray, isNil, isRegExp, isString, resolve } from 'mingo/util'; +import type { AnyObject, Criteria, Options } from 'mingo/types'; +import { $in as mingoIn } from 'mingo/operators/query'; + +/** The private mingo operator a marked `ReadableIds` list lowers to. */ +export const READABLE_IDS_MEMBER_OP = '$readableIdsMember'; + +const readableIdsLists = new WeakSet(); + +/** Mark a list `readableParentIds` produced, so the lowering knows it is the D3 leaf's. */ +export function markReadableIdsList(list: T): T { + readableIdsLists.add(list); + return list; +} + +/** Whether `value` is a list {@link markReadableIdsList} marked. */ +export function isReadableIdsList(value: unknown): value is unknown[] { + return Array.isArray(value) && readableIdsLists.has(value); +} + +/** + * mingo's `$in`, with the list hashed once when the query compiles. The + * returned closure is the per-document test. + */ +export const readableIdsMember: typeof mingoIn = (selector, value, options) => { + if (!Array.isArray(value)) return mingoIn(selector, value, options); + const list: unknown[] = value; + const members = HashMap.init(); + for (const member of list) members.set(member, true); + const nullIsMember = list.some((member) => member === null); + const patterns = list.filter(isRegExp); + const resolveOptions = { unwrapArray: true }; + return (doc) => { + const lhs = resolve(doc, selector, resolveOptions); + if (isNil(lhs)) return nullIsMember; + const values = ensureArray(lhs); + if (values.some((v) => members.has(v))) return true; + return patterns.length > 0 && values.some((v) => isString(v) && patterns.some((re) => re.test(v))); + }; +}; + +const READABLE_IDS_CONTEXT = Context.init({ query: { [READABLE_IDS_MEMBER_OP]: readableIdsMember } }); + +/** mingo's `Query`, able to compile {@link READABLE_IDS_MEMBER_OP}. Every other operator is mingo's. */ +export class Query extends MingoQuery { + constructor(condition: Criteria, options?: Partial) { + super(condition, { ...options, context: READABLE_IDS_CONTEXT }); + } +}