diff --git a/packages/create-objectstack/src/scaffold-e2e-boot-probe.test.ts b/packages/create-objectstack/src/scaffold-e2e-boot-probe.test.ts index 6b3163931e5..ded56c76254 100644 --- a/packages/create-objectstack/src/scaffold-e2e-boot-probe.test.ts +++ b/packages/create-objectstack/src/scaffold-e2e-boot-probe.test.ts @@ -345,27 +345,42 @@ interface Ran { seconds: number; } -/** Execute a step's script the way GitHub does: `bash -e `. */ -function runBlock(h: Harness, script: string): Ran { +/** + * Execute a step's script the way GitHub does: `bash -e `. + * + * Awaited, never `execFileSync`: a block's wall is its own waits — the loop's + * `sleep 2`, the stub's boot delay — spent in child processes, so this worker + * stays free while one block waits and the cases below run concurrently. The + * answer is the synchronous one's: the exit status (`-1` when a signal ended + * it, the timeout's included), stdout alone on success, stdout then stderr on + * failure. + */ +function runBlock(h: Harness, script: string): Promise { const file = path.join(h.dir, 'step.sh'); fs.writeFileSync(file, script); const started = Date.now(); - try { - const out = execFileSync('bash', ['-e', file], { + return new Promise((resolve) => { + const child = spawn('bash', ['-e', file], { env: h.env, cwd: h.dir, - encoding: 'utf8', timeout: 240_000, stdio: ['ignore', 'pipe', 'pipe'], }); - return { status: 0, out, seconds: (Date.now() - started) / 1000 }; - } catch (err: any) { - return { - status: typeof err.status === 'number' ? err.status : -1, - out: `${err.stdout ?? ''}${err.stderr ?? ''}`, - seconds: (Date.now() - started) / 1000, - }; - } + let stdout = ''; + let stderr = ''; + child.stdout.setEncoding('utf8').on('data', (chunk: string) => { + stdout += chunk; + }); + child.stderr.setEncoding('utf8').on('data', (chunk: string) => { + stderr += chunk; + }); + const settle = (status: number, out: string) => + resolve({ status, out, seconds: (Date.now() - started) / 1000 }); + child.once('error', () => settle(-1, `${stdout}${stderr}`)); + child.once('close', (code) => + code === 0 ? settle(0, stdout) : settle(typeof code === 'number' ? code : -1, `${stdout}${stderr}`), + ); + }); } /** @@ -387,12 +402,13 @@ function said(r: Ran): string { * A neighbouring run's healthy server, answering everything with 200. * * A CHILD PROCESS, not an in-process `http.createServer`, and that is not a - * style choice: `runBlock` below uses `execFileSync`, which blocks this worker's - * event loop for the whole run. An in-process neighbour would accept the TCP - * connection (the kernel backlog does that) and then never write a response, so - * the block's `curl` — which carries no `--max-time`, exactly as the workflow - * spells it — would hang until the harness timeout. Measured while writing this - * file: every case sat at its 240s ceiling. + * style choice: `runBlock` used `execFileSync`, which blocked this worker's + * event loop for the whole run, and `curlBody` still does for its probe. An + * in-process neighbour would accept the TCP connection (the kernel backlog does + * that) and then never write a response, so a `curl` — which carries no + * `--max-time`, exactly as the workflow spells it — would hang until the + * harness timeout. Measured while writing this file: every case sat at its + * 240s ceiling. * * ## Why this waits on the child's own word instead of on a clock (commit c27e16059) * @@ -530,6 +546,14 @@ async function neighbour(options: NeighbourOptions = {}): Promise { child.on('error', (err) => giveUp(diagnosis(`it never spawned: ${err.message}`))); child.on('exit', (code, signal) => { state.exited = { code, signal }; + }); + // The death is diagnosed on 'close', never on 'exit': Node does not order + // 'exit' after this side has read the child's pipes, and a refused bind's + // errno arrives on the same stdout. Measured with the cases running + // concurrently: 'exit' first, the errno unread, and the diagnosis "it + // exited" in place of "refused to bind: EADDRINUSE". 'close' waits for both + // pipes to drain, so the child's last words are read before its death. + child.on('close', (code, signal) => { giveUp(diagnosis(`it exited before announcing a listener (code ${code}, signal ${signal})`)); }); const backstop = setTimeout( @@ -670,7 +694,16 @@ const OS_START_STEPS: Array<[string, string]> = [ ['registry-canary', 'Boot and probe health (blank only)'], ]; -describe.skipIf(!RUNNABLE)('[#9779] scaffold-e2e.yml boot-and-probe blocks assert on their OWN server', () => { +/** + * Concurrent, both suites: each case's wall is the waits of its own child + * processes (the loop's `sleep 2`, the stub's boot delay, a backstop), and the + * cases share nothing — each gets its own kernel-assigned port, harness + * directory and stub state. Run one after another, those waits summed to the + * file's wall; run together, they overlap. Nothing waited on is shortened. + */ +const CASES_RUN_TOGETHER = { concurrent: true } as const; + +describe.skipIf(!RUNNABLE)('[#9779] scaffold-e2e.yml boot-and-probe blocks assert on their OWN server', CASES_RUN_TOGETHER, () => { for (const [job, step] of OS_START_STEPS) { describe(`${job} / ${step}`, () => { it('refuses a neighbour already answering the URL its loop accepts as proof', async () => { @@ -684,7 +717,7 @@ describe.skipIf(!RUNNABLE)('[#9779] scaffold-e2e.yml boot-and-probe blocks asser // Vacuity guard: the neighbour must really be reachable at the exact // spelling the loop probes, or "refused" below proves nothing. expect(curlBody(`http://localhost:${port}/api/v1/health`)).toContain('NEIGHBOUR'); - const r = runBlock(harness(), script); + const r = await runBlock(harness(), script); expect(r.status, said(r)).not.toBe(0); expect(r.out).toContain('already serving'); // The whole card: it must NOT have gone on to assert /api/v1/ready @@ -713,7 +746,7 @@ describe.skipIf(!RUNNABLE)('[#9779] scaffold-e2e.yml boot-and-probe blocks asser expect(occupier.message).toContain('its listener refused to bind: EADDRINUSE'); const script = stepScript(step).replaceAll('8080', String(port)); - const r = runBlock(harness({ STUB_PORT_HOLDER_PID: String(held.pid) }), script); + const r = await runBlock(harness({ STUB_PORT_HOLDER_PID: String(held.pid) }), script); expect(r.status, said(r)).toBe(0); // `curl -fsS .../api/v1/ready` echoes the body, so the run says out loud // whose app it asserted on. This is the guard that keeps the case above @@ -730,7 +763,7 @@ describe.skipIf(!RUNNABLE)('[#9779] scaffold-e2e.yml boot-and-probe blocks asser const held = await portHolder(); try { const script = stepScript(step).replaceAll('8080', String(held.port)); - const r = runBlock( + const r = await runBlock( harness({ OS_STUB_DIE_ON_BOOT: '1', STUB_PORT_HOLDER_PID: String(held.pid) }), script, ); @@ -754,7 +787,7 @@ describe.skipIf(!RUNNABLE)('[#9779] scaffold-e2e.yml boot-and-probe blocks asser const held = await portHolder(); try { const script = stepScript(step).replaceAll('18080', String(held.port)); - const r = runBlock( + const r = await runBlock( harness({ STUB_CONTAINER_DIE_MS: '1500', STUB_PORT_HOLDER_PID: String(held.pid) }), script, ); @@ -773,7 +806,7 @@ describe.skipIf(!RUNNABLE)('[#9779] scaffold-e2e.yml boot-and-probe blocks asser const held = await portHolder(); try { const script = stepScript(step).replaceAll('18080', String(held.port)); - const r = runBlock(harness({ STUB_PORT_HOLDER_PID: String(held.pid) }), script); + const r = await runBlock(harness({ STUB_PORT_HOLDER_PID: String(held.pid) }), script); expect(r.status, said(r)).toBe(0); } finally { held.stop(); @@ -826,7 +859,7 @@ async function failedNeighbour( * harness these pin is reached only where that suite runs, so pinning it * elsewhere would pin an instrument nothing uses. */ -describe.skipIf(!RUNNABLE)('[#19424] the neighbour harness says WHY it did not come up', () => { +describe.skipIf(!RUNNABLE)('[#19424] the neighbour harness says WHY it did not come up', CASES_RUN_TOGETHER, () => { it('reports the port it really bound, and answers there', async () => { const n = await neighbour(); try { diff --git a/packages/rest/src/import-template-route.test.ts b/packages/rest/src/import-template-route.test.ts index a5a23b16198..4a02af2ad1b 100644 --- a/packages/rest/src/import-template-route.test.ts +++ b/packages/rest/src/import-template-route.test.ts @@ -148,11 +148,56 @@ async function boot(opts: BootOptions = {}) { return { engine, protocol, findData, get, importRoute, rest }; } +/** + * A workbook as this file reads it: sheet names and cell values, never the + * dropdowns. exceljs's reader expands each range-keyed `` into + * one entry per covered cell, and the template covers `TEMPLATE_VALIDATED_ROWS` + * (50 000) rows per dropdown column: measured on the `deal` template, 100 000 + * entries and 39 ms a parse in full, 4 ms with the node skipped, every sheet's + * values identical. No case here asserts on a dropdown; `import-template.test.ts` + * pins them, read in full. The one case that writes a workbook back for the + * import door reads it in full, so the file it imports carries its dropdowns. + */ +const readWorkbook = (bytes: Buffer) => loadXlsxWorkbook(bytes, { ignoreNodes: ['dataValidations'] }); + const headerRow = async (bytes: Buffer, sheet = 0) => { - const wb = await loadXlsxWorkbook(bytes); + const wb = await readWorkbook(bytes); return { wb, header: (wb.worksheets[sheet].getRow(1).values as unknown[]).slice(1) as string[] }; }; +/** + * The process's first loads, paid during collection rather than inside the + * first cases' clocks (AGENTS.md: clocked windows measure behaviour, never + * loading). The route `import()`s `exceljs` lazily (`xlsx-module.ts`), and + * each path the cases take — the SQL driver, the template writer in both + * locales, the workbook reader and writer, both import formats, the three + * export formats — runs cold the first time. One throwaway engine walks each + * path once and is destroyed before the first case runs. No case sees its + * engine, its rows or its spies: every case boots an engine of its own. + */ +async function warmUpFirstLoads(): Promise { + const { engine, get, importRoute } = await boot(); + liveEngines.splice(liveEngines.indexOf(engine), 1); + try { + const bytes = (await get({ template: 'true' })).body(); + await parseXlsxToRows(bytes); + const wb = await loadXlsxWorkbook(bytes); + const headers = (wb.worksheets[0].getRow(1).values as unknown[]).slice(1) as string[]; + const fields = (wb.worksheets[1].getColumn(2).values as unknown[]).slice(5, 5 + headers.length) as string[]; + const mapping = Object.fromEntries(headers.map((h, i) => [h, fields[i]])); + await importRows(importRoute, 'deal', { format: 'xlsx', xlsxBase64: bytes.toString('base64'), mapping }); + await importRows(importRoute, 'deal', { format: 'json', rows: [{ title: 'warm-up' }] }); + await wb.xlsx.writeBuffer(); + await readWorkbook((await get({ template: 'true', locale: 'zh-CN' })).body()); + for (const format of ['xlsx', 'csv', 'json']) await get({ format }); + } finally { + await engine.destroy(); + } +} + +// At module scope, never in a hook: vitest clocks a hook as it clocks a test. +await warmUpFirstLoads(); + // --------------------------------------------------------------------------- describe('?template=true — the import template, through the real stack', () => { @@ -175,7 +220,7 @@ describe('?template=true — the import template, through the real stack', () => const schema: any = (await (protocol as any).getMetaItem({ type: 'object', name: 'deal' })).item; for (const f of INJECTED) expect(Object.keys(schema.fields), `${f} is injected`).toContain(f); const { header } = await headerRow((await get({ template: 'true' })).body()); - const wb = await loadXlsxWorkbook((await get({ template: 'true' })).body()); + const wb = await readWorkbook((await get({ template: 'true' })).body()); const fieldColumn = (wb.worksheets[1].getColumn(2).values as unknown[]).slice(5).filter(Boolean); for (const f of INJECTED) expect(fieldColumn, f).not.toContain(f); expect(header).not.toContain('Owner'); @@ -193,7 +238,7 @@ describe('?template=true — the import template, through the real stack', () => it('the lookup column names its target by the target object\'s label', async () => { const { get } = await boot(); - const wb = await loadXlsxWorkbook((await get({ template: 'true' })).body()); + const wb = await readWorkbook((await get({ template: 'true' })).body()); const howToFill = String(wb.worksheets[1].getCell('E12').value); expect(wb.worksheets[1].getCell('B12').value).toBe('account'); expect(howToFill).toContain('客户'); @@ -207,19 +252,19 @@ describe('?template=true — the import template, through the real stack', () => it('?locale=zh-CN answers the Chinese sheets', async () => { const { get } = await boot(); - const wb = await loadXlsxWorkbook((await get({ template: 'true', locale: 'zh-CN' })).body()); + const wb = await readWorkbook((await get({ template: 'true', locale: 'zh-CN' })).body()); expect(wb.worksheets.map((w) => w.name)).toEqual(['模板', '填写说明']); }); it('an Accept-Language of zh-CN answers the Chinese sheets too', async () => { const { get } = await boot(); - const wb = await loadXlsxWorkbook((await get({ template: 'true' }, 'deal', { 'accept-language': 'zh-CN,zh;q=0.9' })).body()); + const wb = await readWorkbook((await get({ template: 'true' }, 'deal', { 'accept-language': 'zh-CN,zh;q=0.9' })).body()); expect(wb.worksheets.map((w) => w.name)).toEqual(['模板', '填写说明']); }); it('with no locale at all the sheets are English', async () => { const { get } = await boot(); - const wb = await loadXlsxWorkbook((await get({ template: 'true' })).body()); + const wb = await readWorkbook((await get({ template: 'true' })).body()); expect(wb.worksheets.map((w) => w.name)).toEqual(['Template', 'Instructions']); }); @@ -393,7 +438,7 @@ describe('?template=true — the template filled in and imported back', () => { // from the header (the `*` stripped) — objectui's wizard does the same. const rows = await parseXlsxToRows(bytes); expect(rows).toHaveLength(1); - const wb = await loadXlsxWorkbook(bytes); + const wb = await readWorkbook(bytes); const headers = (wb.worksheets[0].getRow(1).values as unknown[]).slice(1) as string[]; const fields = (wb.worksheets[1].getColumn(2).values as unknown[]).slice(5, 5 + headers.length) as string[]; const mapping = Object.fromEntries(headers.map((h, i) => [h, fields[i]])); @@ -535,7 +580,9 @@ const PARITY: Record; blank: 'refused' | * the workbook, and imports a row through the real import door, about a third * of the loop each. The engine boot is already warm at this point in the file, * and boot plus registering and syncing the 20 objects is 2% of the case, so - * moving the boot into a `beforeAll` would not fix this. + * moving the boot into a `beforeAll` would not fix this. The parse has since + * stopped reading the dropdowns (`readWorkbook` above), so its share is smaller + * than the table measured. The budget below is unchanged. * * The budget per shape: the slowest per-shape cost measured above is 419ms * (8371ms / 20, at 24 busy loops). 1000ms is about 2.4x that, and about 18x the @@ -563,7 +610,7 @@ describe('the `*` agrees with the engine: starred exactly when the import door r const def = served ?? authored; const object = objectOf(key); // The template's own verdict — its header, via `?fields=` so every shape is a column. - const wb = await loadXlsxWorkbook((await get({ template: 'true', fields: 'f' }, object)).body()); + const wb = await readWorkbook((await get({ template: 'true', fields: 'f' }, object)).body()); const starred = wb.worksheets[0].getCell('A1').value === 'F *'; expect(wb.worksheets[1].getCell('D5').value, key).toBe(starred ? 'Yes' : 'No'); expect(isTemplateRequired({ required: true, ...def }), key).toBe(starred); diff --git a/packages/rest/src/xlsx-test-loader.ts b/packages/rest/src/xlsx-test-loader.ts index 63e3b596385..67e94abdf88 100644 --- a/packages/rest/src/xlsx-test-loader.ts +++ b/packages/rest/src/xlsx-test-loader.ts @@ -64,7 +64,7 @@ */ import ExcelJS from 'exceljs'; -import type { Workbook, Xlsx } from 'exceljs'; +import type { Workbook, Xlsx, XlsxReadOptions } from 'exceljs'; /** * The parameter type `Xlsx.load` actually declares, read off the dependency's @@ -80,9 +80,13 @@ type XlsxLoadInput = Parameters[0]; * The single site in this package where the exceljs declaration defect above is * asserted away. Callers pass the Node `Buffer` their fixture produced and get * back a workbook; no call site needs to know about any of this. + * + * `options` is exceljs's own read options, handed through unchanged — for a + * caller that reads cell values only and skips a node it never asserts on + * (`ignoreNodes`). Omitted, the workbook is read in full. */ -export async function loadXlsxWorkbook(bytes: Buffer): Promise { +export async function loadXlsxWorkbook(bytes: Buffer, options?: Partial): Promise { const wb = new ExcelJS.Workbook(); - await wb.xlsx.load(bytes as unknown as XlsxLoadInput); + await wb.xlsx.load(bytes as unknown as XlsxLoadInput, options); return wb; } diff --git a/packages/verify/src/boot-warm-up.testkit.ts b/packages/verify/src/boot-warm-up.testkit.ts index 8f3c9d9743e..05d6ff0d2d7 100644 --- a/packages/verify/src/boot-warm-up.testkit.ts +++ b/packages/verify/src/boot-warm-up.testkit.ts @@ -30,10 +30,40 @@ * import, and a list would drift as the boot changes. Running the boot, a * sign-in and a sign-up pays all of it through this package's own harness. * - * ⛔ It shares nothing a case asserts on. It boots an app of its own (no - * objects), on its own in-memory database, and stops it before the file's - * first hook runs. What stays warm is process-level: the module registry, - * sql.js's compiled WASM and the JIT. + * What it shares. It boots an app of its own (no objects), under a + * configuration object of its own, on its own in-memory database, and stops + * it before the file's first hook runs, so no kernel, row, service instance or + * configuration claim outlives it. What outlives it is the PROCESS's state — + * under vitest's per-file isolation, the file's — which the file's own boots + * then inherit instead of creating: + * + * - the harness's data key: `harnessCryptoProvider()` in `harness.ts` builds + * the process's crypto provider on the process's FIRST boot and keeps it, so + * every later boot seals and opens under the provider the warm-up built; + * - `NODE_ENV`: `bootStack` sets it to `development` and never restores it, + * so the module code after the warm-up, and every case before the file's + * own first boot, runs under `development` instead of vitest's `test`; + * - what the loaded modules keep for the life of the process: the module + * registry, sql.js's compiled WASM, the JIT, better-auth's process-global + * context holder, and each module's own memos and once-per-process notices. + * + * That is safe for a case that asserts on none of it and changes none of its + * inputs after the warm-up: the provider is built by the same code from the + * same input whichever boot builds it (a random key, no environment read, no + * key file); every `bootStack` sets `NODE_ENV` before it builds its kernel; + * and a module's memo answers the file's boots as it would have answered the + * file's first. Only WHEN that state is created moves, and so under which + * environment. + * + * ⛔ So a file whose case asserts on that state takes no warm-up: on how or + * under which environment the data key is built, on `NODE_ENV` before its + * first boot, on a notice a module says once per process, or on a memo whose + * input the case changes after the warm-up (an environment variable a module + * reads once). + * `harness.key-custody.test.ts` is the measured case: it sets its keyless + * posture in a hook, a warm-up would build the provider at module scope before + * that posture exists, and a provider that minted a key file would then mint it + * outside the posture the file inspects, so the pin would stay green over it. * * ⛔ Call it at module scope, on a line of its own, never in a hook: vitest * clocks a hook with `hookTimeout` exactly as it clocks a test with