diff --git a/.changeset/15953-s8-cli-dashboard-windows-flag.md b/.changeset/15953-s8-cli-dashboard-windows-flag.md new file mode 100644 index 00000000000..28c4b97b22b --- /dev/null +++ b/.changeset/15953-s8-cli-dashboard-windows-flag.md @@ -0,0 +1,9 @@ +--- +'@objectstack/cli': patch +--- + +fix(cli): `os verify --dashboard-windows` describes the measures it proves: counts, sums and derived measures + +Clause-②: no (the flag, its default and what it accepts are unchanged; only its help text moves) + +The `--dashboard-windows` flag description ended "(date and datetime fields; count measures)". The date-window family in `@objectstack/verify` now also proves sums and derived measures over counts or sums, so it reads FROM "(date and datetime fields; count measures)" TO "(date and datetime fields; count, sum and derived measures)". Nothing else about the flag changes. diff --git a/.changeset/15953-s8-window-sum-derived.md b/.changeset/15953-s8-window-sum-derived.md new file mode 100644 index 00000000000..233853c6e96 --- /dev/null +++ b/.changeset/15953-s8-window-sum-derived.md @@ -0,0 +1,16 @@ +--- +'@objectstack/verify': patch +--- + +fix(verify): the dashboard date-window family also proves sums and derived measures, and says why an average stays out + +Clause-②: no (no export, option or type moves: the family reads the measures it already read, and a widget it used to exclude can now be proven or contradicted) + +`runDashboardWindowProofs` (and `os verify --dashboard-windows`) now judges every verdict in fixture rows, so it compares every measure whose difference can be read back as rows: + +- **A `sum`** is probed. Each fixture row holds the summed field to the value the CRUD derivation synthesizes for it, or to the value the widget's own selection already holds there. The sum's difference over that value is the rows the window took in. FROM: a sum-only widget was `excluded` (`no-count-measure`), and a sum beside a count was an `excluded` measure (`not-a-count`). TO: `proven` or `contradicted` like a count. A sum of a relationship path, a summed field with no synthesizable value, and a held value that is not a non-zero number are `excluded` as `count-field-unsynthesizable`, and the reason names which. +- **A `derived` measure** (`ratio`, `sum`, `difference`, `product`) is probed through its operands, which the widget's own answer carries beside it. Each operand must be a count or a sum of the same dataset, and the fixture rows meet every operand's filter. Under each preset every operand must take in exactly the rows the window selects. The value the widget shows must equal `evaluateDerivedMeasures` (the server's own evaluator, from `@objectstack/service-analytics`) over that row's operands, or the preset is `contradicted` and the detail names both values. A derived measure over a derived one, or over a measure that is not a count or a sum, stays `excluded` (`not-a-count`), and the reason names the operand. +- **An `avg`, `min`, `max` or `count_distinct`** stays `excluded` (`not-a-count`, or `no-count-measure` for a widget with nothing else), with a reason that says why. What a fixture row does to it depends on rows the app already holds in the window: their number for an average, their extreme for a minimum or maximum, their values for a distinct count. Nothing the widget answers reveals that, so deriving it would mean guessing the data. The reason text changes: FROM `"avg_effort" aggregates avg of effort, not a row count` TO `"avg_effort" averages effort: an average is not additive — …`. +- The widget-level `no-count-measure` reason reads `no measure the widget selects can be compared (…)`. The `excludedKind` values are unchanged. +- **A new `not-run` cause.** The family gives up on a preset when a sum's own total is so large that its floating-point difference cannot resolve one fixture row. That needs 2^34 times the held value, and the detail says so. +- **Upgrading:** on an app whose dashboards bind sums or derived measures, `os verify --dashboard-windows` now probes those widgets. A window that does not reach them now fails the run (`hardFailures`), where before they were only counted as `excluded`. diff --git a/content/docs/deployment/cli.mdx b/content/docs/deployment/cli.mdx index bd35864b43e..a3a720e076b 100644 --- a/content/docs/deployment/cli.mdx +++ b/content/docs/deployment/cli.mdx @@ -2209,8 +2209,9 @@ os verify --dashboard-windows # Also prove each dashboard date `filterBindings.dateRange` names, or on the dashboard's `dateRange.field` (`created_at` when unauthored), resolved against the object behind the widget's dataset, and must be a `date` or `datetime` field; a widget with - `filterBindings: { dateRange: false }` is reported `unbound`. For each count - measure of a bound widget, fixture rows are written through the seed door + `filterBindings: { dateRange: false }` is reported `unbound`. For each + measure of a bound widget that is a count, a sum, or a derived measure over + those, fixture rows are written through the seed door on every preset's first and last day and the day either side (never after today), stamped at UTC noon, owned by the caller and holding every column the widget's own filter, the dataset's filter and the measure's filter @@ -2230,8 +2231,9 @@ os verify --dashboard-windows # Also prove each dashboard date probe whose fixtures do not meet the widget's own selection, or that cannot be written or read back as written, is reported inconclusive, never proven. A held column is an equality, an `$in`, a `$nin` or `$ne` on a field that - declares its options, or `{current_user_id}`; an average, a sum or a derived - measure, and a filter the ground truth cannot model (`$or`, a range, a + declares its options, or `{current_user_id}`; an average, a minimum or + maximum, a distinct count, or a derived measure over one, and a filter the + ground truth cannot model (`$or`, a range, a relationship path, a condition on the window's own field), are reported `excluded`. `--multi-tenant` boots org-scoped so tenant-isolation RLS policies diff --git a/packages/cli/src/commands/verify.ts b/packages/cli/src/commands/verify.ts index 196b60a29ab..5901741f6c1 100644 --- a/packages/cli/src/commands/verify.ts +++ b/packages/cli/src/commands/verify.ts @@ -155,7 +155,7 @@ export default class Verify extends Command { default: false, }), 'dashboard-windows': Flags.boolean({ - description: "Also prove, for every dashboard that declares a dateRange, that each widget it binds counts, under each date-range preset of the picker, the rows that window selects: fixture rows written on known days, the widget asked through the dataset door with and without the window (date and datetime fields; count measures)", + description: "Also prove, for every dashboard that declares a dateRange, that each widget it binds counts, under each date-range preset of the picker, the rows that window selects: fixture rows written on known days, the widget asked through the dataset door with and without the window (date and datetime fields; count, sum and derived measures)", default: false, }), 'parent-write-reach': Flags.boolean({ diff --git a/packages/verify/README.md b/packages/verify/README.md index fbee074b7e9..560b8e221ac 100644 --- a/packages/verify/README.md +++ b/packages/verify/README.md @@ -38,7 +38,8 @@ derived from your own metadata: master. - **Dashboard date windows** (opt-in) — *every widget a dashboard's `dateRange` binds counts, under each preset of the picker, the rows that window selects*: - per bound widget and count measure, fixture rows are written on known days + per bound widget and measure (a count, a sum, or a derived measure over + those), fixture rows are written on known days around every preset's edges, and the widget is asked through the dataset door with no window and under each of the 13 presets, before and after the fixtures. Each difference must equal the fixture rows the window selects; a @@ -502,18 +503,19 @@ compute one, so it is not read as "no grant anywhere". `summary.hardFailures` is `contradicted + failed` objects plus personas that could not be provisioned. **Dashboard date windows** (`runDashboardWindowProofs`), per widget a -dashboard's `dateRange` binds. Probes per count measure: the widget with no +dashboard's `dateRange` binds. Probes per measure that is a count, a sum or +derived over those: the widget with no window, then each of the 13 presets, each counted before and after the fixture rows: | status | meaning | | --- | --- | | `proven` | under every preset the window counted exactly the fixture rows it selects | -| `contradicted` | under some preset the widget showed nothing where the window selects fixture rows (`blank`), or counted another number of them; a window that answers what no window answers under every preset is also named `inert` **(failure)** | +| `contradicted` | under some preset the widget showed nothing where the window selects fixture rows (`blank`), or counted another number of them, or a derived measure showed another value than the server's evaluator makes of its operands; a window that answers what no window answers under every preset is also named `inert` **(failure)** | | `failed` | the door refused or faulted on a windowed query while the widget's own query answered **(failure)** | | `inconclusive` | a fixture row could not be written or did not read back as written (an app hook rewrote it, or the read omits the column), the widget with no window did not count every fixture row (its own selection does not meet the fixtures), its own query is refused, the UTC day turned after the run started, or the caller's reference timezone is not UTC | | `unbound` | the widget opts out of the window (`filterBindings: { dateRange: false }`) | -| `excluded` | the family cannot compare it: no count measure, a filter the ground truth cannot model, a window on a field that is missing or is not `date` / `datetime`, a dataset or dashboard that is undeclared or does not parse, or an object the app does not declare or the CRUD derivation cannot synthesize (`excludedKind` names which) | +| `excluded` | the family cannot compare it: no measure it can compare (an `avg`, `min`, `max` or `count_distinct` is not additive), a filter the ground truth cannot model, a window on a field that is missing or is not `date` / `datetime`, a dataset or dashboard that is undeclared or does not parse, or an object the app does not declare or the CRUD derivation cannot synthesize (`excludedKind` names which) | `member-visible` everywhere usually means the app is single-tenant; pass `--multi-tenant` (or `{ multiTenant: true }`) to register org-scoping so tenant diff --git a/packages/verify/src/dashboard-window.test.ts b/packages/verify/src/dashboard-window.test.ts index 49740f917f0..bf112f8057b 100644 --- a/packages/verify/src/dashboard-window.test.ts +++ b/packages/verify/src/dashboard-window.test.ts @@ -8,7 +8,11 @@ // published seam (`registerMiddleware`), no engine change: one drops the // window from the query (a floor that matches every row), one makes a // windowed query match nothing (a window that zeroes the widget). Both are -// found, and the first is named inert; +// found — on a count, a sum and a derived ratio alike — and the first is +// named inert; +// • a DERIVED-VALUE control: a door whose derived value is not the op over +// the operands it answers beside it is contradicted, while its operands +// hold; // • the ATTRIBUTION control: an app hook that rewrites the window's field on // write makes the probe not run, never contradicted — the family judges a // window only over rows that read back as written; @@ -88,6 +92,12 @@ const source = { { name: 'assigned_count', label: 'Assigned', aggregate: 'count', field: 'assignee' }, { name: 'avg_effort', label: 'Effort', aggregate: 'avg', field: 'effort' }, { name: 'closed_rate', label: 'Closed rate', derived: { op: 'ratio', of: ['closed_count', 'ticket_count'] } }, + { name: 'effort_total', label: 'Effort total', aggregate: 'sum', field: 'effort' }, + { name: 'open_count', label: 'Not closed', derived: { op: 'difference', of: ['ticket_count', 'closed_count'] } }, + { name: 'effort_per_ticket', label: 'Effort per ticket', derived: { op: 'ratio', of: ['effort_total', 'ticket_count'] } }, + { name: 'effort_index', label: 'Effort index', derived: { op: 'ratio', of: ['avg_effort', 'ticket_count'] } }, + { name: 'rate_of_rate', label: 'Rate of rate', derived: { op: 'ratio', of: ['closed_rate', 'ticket_count'] } }, + { name: 'distinct_effort', label: 'Distinct effort', aggregate: 'count_distinct', field: 'effort' }, ], }, { @@ -114,10 +124,19 @@ const source = { // A declared option the condition does not exclude holds the column. { id: 'not_closed', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filter: { status: { $nin: ['closed'] } } }, { id: 'mixed', type: 'table', dataset: 'ticket_metrics', dimensions: ['priority'], values: ['ticket_count', 'avg_effort'] }, + // A sum, alone, grouped and beside a count; derived measures over counts and sums. + { id: 'effort_sum', type: 'metric', dataset: 'ticket_metrics', values: ['effort_total'] }, + { id: 'workload', type: 'bar', dataset: 'ticket_metrics', dimensions: ['priority'], values: ['effort_total'] }, + { id: 'count_and_sum', type: 'table', dataset: 'ticket_metrics', values: ['ticket_count', 'effort_total'] }, + { id: 'rate', type: 'metric', dataset: 'ticket_metrics', values: ['closed_rate'] }, + { id: 'still_open', type: 'metric', dataset: 'ticket_metrics', values: ['open_count'] }, + { id: 'per_ticket', type: 'bar', dataset: 'ticket_metrics', dimensions: ['status'], values: ['effort_per_ticket'] }, // Outside the population, each with its reason. { id: 'trend', type: 'line', dataset: 'ticket_metrics', values: ['ticket_count'], filterBindings: { dateRange: false } }, { id: 'effort', type: 'metric', dataset: 'ticket_metrics', values: ['avg_effort'] }, - { id: 'rate', type: 'metric', dataset: 'ticket_metrics', values: ['closed_rate'] }, + { id: 'index', type: 'metric', dataset: 'ticket_metrics', values: ['effort_index'] }, + { id: 'nested', type: 'metric', dataset: 'ticket_metrics', values: ['rate_of_rate'] }, + { id: 'distinct', type: 'metric', dataset: 'ticket_metrics', values: ['distinct_effort'] }, { id: 'recent', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filter: { opened_at: { $gte: '{30_days_ago}' } } }, { id: 'either', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filter: { $or: [{ status: 'open' }, { is_closed: true }] } }, { id: 'not_x', type: 'metric', dataset: 'ticket_metrics', values: ['ticket_count'], filter: { subject: { $ne: 'x' } } }, @@ -137,7 +156,10 @@ const source = { ], }; -const PROVEN = ['open_tickets', 'by_status', 'due_window', 'mine', 'closed_tickets', 'with_assignee', 'high_or_low', 'not_closed', 'mixed', 'all_tickets']; +const PROVEN = [ + 'open_tickets', 'by_status', 'due_window', 'mine', 'closed_tickets', 'with_assignee', 'high_or_low', 'not_closed', 'mixed', + 'effort_sum', 'workload', 'count_and_sum', 'rate', 'still_open', 'per_ticket', 'all_tickets', +]; const widgetOf = (report: DashboardWindowReport, widget: string) => { const r = report.results.find((x) => x.widget === widget); @@ -195,8 +217,17 @@ describe('deriveDashboardWindowCases — what the family reads from the metadata expect(cases.get('wnd_desk/with_assignee')!.measures).toEqual([{ measure: 'assigned_count', countField: 'assignee' }]); expect(cases.get('wnd_desk/closed_tickets')!.measures).toEqual([{ measure: 'closed_count', filter: { is_closed: true } }]); expect(cases.get('wnd_desk/mixed')!.measures.map((m) => m.excluded?.kind ?? 'probed')).toEqual(['probed', 'not-a-count']); + // A sum and a derived measure over counts or sums are in the population. + expect(cases.get('wnd_desk/effort_sum')!.measures).toEqual([{ measure: 'effort_total' }]); + expect(cases.get('wnd_desk/rate')!.measures).toEqual([{ measure: 'closed_rate' }]); + expect(cases.get('wnd_desk/rate')!.excluded).toBeUndefined(); + expect(cases.get('wnd_desk/per_ticket')!.excluded).toBeUndefined(); + // What is not additive stays out, and says why. expect(cases.get('wnd_desk/effort')!.excluded).toMatchObject({ kind: 'no-count-measure' }); - expect(cases.get('wnd_desk/rate')!.excluded!.reason).toMatch(/"closed_rate" is derived \(ratio of closed_count, ticket_count\)/); + expect(cases.get('wnd_desk/effort')!.excluded!.reason).toMatch(/"avg_effort" averages effort: an average is not additive/); + expect(cases.get('wnd_desk/index')!.excluded!.reason).toMatch(/"effort_index" is derived \(ratio of avg_effort, ticket_count\), and its operand "avg_effort" averages effort/); + expect(cases.get('wnd_desk/nested')!.excluded!.reason).toMatch(/its operand "closed_rate" is itself derived/); + expect(cases.get('wnd_desk/distinct')!.excluded!.reason).toMatch(/"distinct_effort" counts the distinct values of effort/); expect(cases.get('wnd_desk/users')!.excluded).toMatchObject({ kind: 'object-undeclared' }); }); @@ -263,13 +294,19 @@ describe('runDashboardWindowProofs — the fixture app', () => { expect(widgetOf(report, 'open_tickets').measures[0]!.fixture).toBe('smallest'); expect(widgetOf(report, 'due_window').fieldType).toBe('date'); expect(widgetOf(report, 'all_tickets').fieldType).toBe('datetime'); + // A derived measure shows its operands' value, which its answer carries beside it. + const rate = widgetOf(report, 'rate').measures[0]!; + expect(rate.fixture).toBe('every-field'); + for (const p of rate.presets) expect(p.shown).toBeGreaterThan(0); }); it('names every widget outside the population, with its reason', () => { expect(widgetOf(report, 'trend').status).toBe('unbound'); const excluded = (w: string) => [widgetOf(report, w).status, widgetOf(report, w).excludedKind]; expect(excluded('effort')).toEqual(['excluded', 'no-count-measure']); - expect(excluded('rate')).toEqual(['excluded', 'no-count-measure']); + expect(excluded('index')).toEqual(['excluded', 'no-count-measure']); + expect(excluded('nested')).toEqual(['excluded', 'no-count-measure']); + expect(excluded('distinct')).toEqual(['excluded', 'no-count-measure']); expect(excluded('recent')).toEqual(['excluded', 'filter-unmodelled']); expect(widgetOf(report, 'recent').reason).toMatch(/a condition on opened_at, the field the window lands on/); expect(excluded('either')).toEqual(['excluded', 'filter-unmodelled']); @@ -286,11 +323,12 @@ describe('runDashboardWindowProofs — the fixture app', () => { it('counts every dashboard, widget, measure and window', () => { expect(report.summary).toEqual({ dashboards: { declared: 3, windowed: 2, notWindowed: 1, excluded: 0 }, - widgets: { total: 19, unbound: 1, excluded: 8, probed: 10, proven: 10, contradicted: 0, failed: 0, inconclusive: 0 }, - excludedBy: { 'no-count-measure': 2, 'filter-unmodelled': 3, 'field-type': 1, 'field-missing': 1, 'object-undeclared': 1 }, - measures: { probed: 10, excluded: 1, excludedBy: { 'not-a-count': 1 } }, - probes: 10 * DATE_RANGE_PRESETS.length, - held: 10 * DATE_RANGE_PRESETS.length, + widgets: { total: 27, unbound: 1, excluded: 10, probed: 16, proven: 16, contradicted: 0, failed: 0, inconclusive: 0 }, + excludedBy: { 'no-count-measure': 4, 'filter-unmodelled': 3, 'field-type': 1, 'field-missing': 1, 'object-undeclared': 1 }, + // 11 counts, 3 sums (one beside a count), 3 derived (a ratio and a difference of counts, a ratio over a sum). + measures: { probed: 17, excluded: 1, excludedBy: { 'not-a-count': 1 } }, + probes: 17 * DATE_RANGE_PRESETS.length, + held: 17 * DATE_RANGE_PRESETS.length, blank: 0, contradictions: 0, faults: 0, @@ -315,10 +353,26 @@ const controlSource = { opened_at: { type: 'datetime', label: 'Opened at' }, logged_at: { type: 'datetime', label: 'Logged at' }, audited_at: { type: 'datetime', label: 'Audited at', internal: true }, + closed_at: { type: 'datetime', label: 'Closed at' }, + hours: { type: 'number', label: 'Hours' }, + flagged: { type: 'boolean', label: 'Flagged' }, }, }, ], - datasets: [{ name: 'case_metrics', label: 'Cases', object: 'wnc_case', dimensions: [], measures: [{ name: 'case_count', label: 'Cases', aggregate: 'count' }] }], + datasets: [ + { + name: 'case_metrics', + label: 'Cases', + object: 'wnc_case', + dimensions: [], + measures: [ + { name: 'case_count', label: 'Cases', aggregate: 'count' }, + { name: 'hours_total', label: 'Hours', aggregate: 'sum', field: 'hours' }, + { name: 'flagged_count', label: 'Flagged', aggregate: 'count', filter: { flagged: true } }, + { name: 'flagged_share', label: 'Flagged share', derived: { op: 'ratio', of: ['flagged_count', 'case_count'] } }, + ], + }, + ], dashboards: [ { name: 'wnc_board', @@ -328,6 +382,10 @@ const controlSource = { { id: 'cases', type: 'metric', dataset: 'case_metrics', values: ['case_count'] }, { id: 'logged', type: 'metric', dataset: 'case_metrics', values: ['case_count'], filterBindings: { dateRange: 'logged_at' } }, { id: 'audited', type: 'metric', dataset: 'case_metrics', values: ['case_count'], filterBindings: { dateRange: 'audited_at' } }, + { id: 'hours', type: 'metric', dataset: 'case_metrics', values: ['hours_total'] }, + { id: 'share', type: 'metric', dataset: 'case_metrics', values: ['flagged_share'] }, + // On a field the plugin leaves alone: the derived-value control (below). + { id: 'tampered', type: 'metric', dataset: 'case_metrics', values: ['flagged_share'], filterBindings: { dateRange: 'closed_at' } }, ], }, ], @@ -374,6 +432,26 @@ const windowBreaker = (mode: 'drop' | 'zero') => ({ }, }); +/** + * The derived-value control: the door as `stack` answers it, except that a + * `flagged_share` windowed on `closed_at` comes back as half the ratio its own + * operands make — a derived value computed from something other than the + * operands the answer carries beside it. The operands are left as answered. + */ +function tamperingDerived(stack: VerifyStack): VerifyStack { + return { + ...stack, + apiAs: async (...args: Parameters) => { + const res = await stack.apiAs(...args); + const selection = (args[3] as { selection?: { measures?: string[]; runtimeFilter?: unknown } } | undefined)?.selection; + if (!selection?.measures?.includes('flagged_share') || !mentions(selection.runtimeFilter, 'closed_at') || res.status !== 200) return res; + const body = (await res.json()) as { rows: Array> }; + for (const row of body.rows) if (typeof row.flagged_share === 'number') row.flagged_share = row.flagged_share / 2; + return new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' } }); + }, + }; +} + describe.each(['drop', 'zero'] as const)("NEGATIVE CONTROL — an app plugin that keeps the window from storage (%s)", (mode) => { let stack: VerifyStack; let report: DashboardWindowReport; @@ -383,7 +461,7 @@ describe.each(['drop', 'zero'] as const)("NEGATIVE CONTROL — an app plugin tha stack = await bootStack(app as never); const token = await stack.signIn(); wallClockAtMidnight(); - report = await onFrozenClock(() => runDashboardWindowProofs(stack, token, app)); + report = await onFrozenClock(() => runDashboardWindowProofs(tamperingDerived(stack), token, app)); }, BOOT_TIMEOUT + RUN_TIMEOUT); afterAll(async () => { @@ -405,10 +483,42 @@ describe.each(['drop', 'zero'] as const)("NEGATIVE CONTROL — an app plugin tha expect(m.presets.map((p) => p.outcome)).toEqual(DATE_RANGE_PRESETS.map(() => 'blank')); expect(m.inert).toBe(false); } - expect(report.summary.hardFailures).toBe(1); expect(formatDashboardWindowReport(report)).toContain('✗ wnc_board/cases (wnc_case.opened_at, datetime) CONTRADICTED'); }); + it('is found on a sum, and on a derived ratio through its operands', () => { + for (const widget of ['hours', 'share']) { + const r = widgetOf(report, widget); + expect(r.status, `${widget}\n${formatDashboardWindowReport(report)}`).toBe('contradicted'); + const m = r.measures[0]!; + expect(m.unwindowed!.answered).toBe(m.days!.length); + if (mode === 'drop') { + expect(m.presets.map((p) => p.outcome)).toEqual(DATE_RANGE_PRESETS.map(() => 'contradicted')); + expect(m.presets.every((p) => p.answered === m.days!.length)).toBe(true); + expect(m.inert).toBe(true); + } else { + expect(m.presets.map((p) => p.outcome)).toEqual(DATE_RANGE_PRESETS.map(() => 'blank')); + } + } + expect(widgetOf(report, 'hours').measures[0]!.presets[0]!.detail).toMatch( + mode === 'drop' ? /^the window counted \d+ fixture row\(s\) \(a sum of \d+ at 7 per row\), and it selects \d+ \(wider\)$/ : /^the widget showed nothing/, + ); + if (mode === 'drop') { + expect(widgetOf(report, 'share').measures[0]!.presets[0]!.detail).toMatch(/^the window counted \d+ fixture row\(s\) in flagged_count, and it selects \d+ \(wider\)$/); + } + }); + + it('DERIVED-VALUE CONTROL — a derived value that is not the op over its operands is contradicted, while they hold', () => { + const m = widgetOf(report, 'tampered').measures[0]!; + expect(widgetOf(report, 'tampered').status).toBe('contradicted'); + // Every operand took in exactly the rows each window selects… + expect(m.presets.every((p) => p.answered === p.expected)).toBe(true); + // …and the value shown is not the ratio they make. + expect(m.presets.map((p) => p.outcome)).toEqual(DATE_RANGE_PRESETS.map(() => 'contradicted')); + expect(m.presets[0]!.detail).toMatch(/^the widget shows flagged_share = [\d.]+ where its operands in the same row \(flagged_count = \d+, case_count = \d+\) make [\d.]+$/); + expect(report.summary.hardFailures).toBe(4); + }); + it("ATTRIBUTION CONTROL — a hook rewriting the window's field makes the probe not run, never contradicted", () => { const r = widgetOf(report, 'logged'); expect(r.status).toBe('inconclusive'); @@ -418,7 +528,7 @@ describe.each(['drop', 'zero'] as const)("NEGATIVE CONTROL — an app plugin tha const audited = widgetOf(report, 'audited'); expect(audited.status).toBe('inconclusive'); expect(audited.reason).toMatch(/reads back no audited_at \(the read does not return that field\), so what was written there cannot be confirmed$/); - expect(report.summary.widgets).toMatchObject({ contradicted: 1, inconclusive: 2, proven: 0 }); + expect(report.summary.widgets).toMatchObject({ contradicted: 4, inconclusive: 2, proven: 0 }); }); }); diff --git a/packages/verify/src/dashboard-window.ts b/packages/verify/src/dashboard-window.ts index de49162e021..3fa6388f7ef 100644 --- a/packages/verify/src/dashboard-window.ts +++ b/packages/verify/src/dashboard-window.ts @@ -7,8 +7,8 @@ // ## The promise, and where each half of it is read from // // A dashboard's `dateRange` is a picker. Each widget it binds is narrowed to -// the picked window on one of the widget's own fields, and the widget's count -// is the count of the rows that window selects. Three published sources: +// the picked window on one of the widget's own fields, and the widget's +// measure is taken over the rows that window selects. Three published sources: // // - WHICH field a widget's window lands on: `DashboardWidgetSchema.filterBindings` // (`@objectstack/spec/ui`). A string binds the built-in date range to that @@ -34,36 +34,58 @@ // ## The door, and who walks through it // // `POST {basePath}/analytics/dataset/query` with the saved dataset's name and -// the widget's own selection (its dimensions, one count measure, and its -// filter with the window ANDed in as `runtimeFilter`) — the request a -// dashboard widget makes — as the signed-in principal the caller hands in. -// `{current_user_id}` in a widget's filter is that principal. +// the widget's own selection (its dimensions, one measure, and its filter with +// the window ANDed in as `runtimeFilter`) — the request a dashboard widget +// makes — as the signed-in principal the caller hands in. `{current_user_id}` +// in a widget's filter is that principal. // // ## The ground truth, and the control that keeps it honest // -// Per widget and count measure, fixture rows are written through the seed door +// Per widget and measure, fixture rows are written through the seed door // (system, hooks and validations live) on known days that straddle every // preset's past edges, each row stamped at UTC noon (a `date` field takes the // day itself), with every column the widget's own filter, the dataset's filter // and the measure's filter read held to a value those filters select, and // owned by the principal. Each row is read back: a row whose stamp or held // columns do not read back as written (an app hook rewrote them) makes the -// probe not run, never contradicted. Every answer is a DIFFERENCE — the count -// after the fixtures minus the count before, under the same window — so the -// app's own rows cancel out. The control leg is the widget with no window: it -// must count every fixture row, or the fixtures do not meet the widget's own -// selection and no window is judged. +// probe not run, never contradicted. Every answer is a DIFFERENCE — the +// measure after the fixtures minus the measure before, under the same window — +// so the app's own rows cancel out. The control leg is the widget with no +// window: it must take in every fixture row, or the fixtures do not meet the +// widget's own selection and no window is judged. +// +// Every verdict is in fixture rows, so a measure is compared only where a +// difference can be read back as rows — where the measure is ADDITIVE: +// +// - a `count` adds one per fixture row (a count of a named field: the +// fixtures carry a value there); +// - a `sum` adds the value each fixture row carries in the summed field, +// held to the one the CRUD derivation synthesizes, so its difference over +// that value is the rows it took in; +// - a `derived` measure (`ratio`, `sum`, `difference`, `product`) is read +// through its operands, which the widget's own answer carries beside it: +// each operand must be a count or a sum, the fixtures meet every operand's +// filter, every operand must take in exactly the rows the window selects, +// and the value the widget shows must be the server's own evaluator +// (`evaluateDerivedMeasures`) applied to those operands, row by row. +// +// An `avg`, `min`, `max` or `count_distinct` is not additive: what a fixture +// row does to it depends on the rows the app already holds in the window +// (their number, their extreme, their values), which nothing the widget +// answers reveals. Deriving it would mean guessing that data, so such a +// measure, and a derived measure over one, is excluded with that reason. // // Three verdicts per preset: the widget is not blank where the window holds -// fixture rows; its count equals the rows the window selects; and a narrower -// preset never answers more than a wider one (a window that never narrows is -// inert, a finding). +// fixture rows; it takes in exactly the rows the window selects; and a +// narrower preset never takes in more than a wider one (a window that never +// narrows is inert, a finding). // // Run it on a fresh stack of its own: it writes fixtures. import { DashboardSchema, DatasetSchema } from '@objectstack/spec/ui'; import { DATE_RANGE_PRESETS, DATE_RANGE_PRESET_MACRO_WINDOWS, type DateRangePreset } from '@objectstack/spec/data'; import { resolveFilterTokens } from '@objectstack/core'; +import { evaluateDerivedMeasures } from '@objectstack/service-analytics'; import { declaredCollection } from './artifact-collections.js'; import { deriveCrudCases, fillRelationalRefs, fixtureBody, type CrudCase } from './derive.js'; import type { VerifyStack } from './harness.js'; @@ -86,7 +108,14 @@ const CURRENT_USER_TOKEN = '{current_user_id}'; // ── Types ─────────────────────────────────────────────────────────────────── -/** Why the family cannot compare a widget, or one of its measures. */ +/** + * Why the family cannot compare a widget, or one of its measures. Three name + * the measure: `not-a-count` — it is not additive (`avg`, `min`, `max`, + * `count_distinct`), or derived over a measure that is not, or derived over a + * derived one; `count-field-unsynthesizable` — the field a count or a sum + * reads cannot be held on a fixture row; `no-count-measure` — no measure the + * widget selects can be compared. + */ export type DashboardWindowExclusion = | 'dashboard-unparsed' | 'dataset-undeclared' @@ -158,7 +187,7 @@ export interface DashboardWindowPreset { export interface DashboardWindowAnswer { kind: 'answered' | 'refused' | 'faulted'; status: number; - /** The measure summed over every returned row. */ + /** The measure summed over every returned row (a null value adds nothing). */ count?: number; rows?: number; code?: string; @@ -170,16 +199,24 @@ export interface DashboardWindowPresetProbe { preset: DateRangePreset; /** Fixture rows the window selects (the ground truth). */ expected: number; - /** The count the window added once the fixtures were written (after minus before). */ + /** + * The fixture rows the window added once the fixtures were written (after + * minus before): a count's difference; a sum's difference over the value + * each fixture row carries; a derived measure's operands', which must agree + * (the first one that does not, when they do not). + */ answered?: number; - /** The count the widget showed after the fixtures, all rows included. */ + /** The measure the widget showed after the fixtures, summed over its rows. */ shown?: number; /** - * - `held` — the window counted exactly the fixture rows it selects; + * - `held` — the window took in exactly the fixture rows it selects + * (and a derived measure showed its operands' value); * - `blank` — it showed nothing where it selects fixture rows; - * - `contradicted` — it counted another number of them; + * - `contradicted` — it took in another number of them, or a derived measure + * showed another value than its operands make; * - `failed` — the door refused or faulted on the windowed query (the unwindowed one answered); - * - `not-run` — the fixtures could not be written, or the day turned mid-probe. + * - `not-run` — the fixtures could not be written, the day turned mid-probe, + * or a sum's own total is too large to resolve one fixture row. */ outcome: 'held' | 'blank' | 'contradicted' | 'failed' | 'not-run'; answer?: DashboardWindowAnswer; @@ -195,7 +232,7 @@ export interface DashboardWindowMeasureResult { days?: string[]; /** `smallest` — the required fields only; `every-field` — the seed door refused that, so every field the CRUD derivation synthesizes. */ fixture?: 'smallest' | 'every-field'; - /** The control leg: the widget with no window must count every fixture row. */ + /** The control leg: the widget with no window must take in every fixture row. */ unwindowed?: { expected: number; answered?: number; answer?: DashboardWindowAnswer }; presets: DashboardWindowPresetProbe[]; /** Every preset answered what the unwindowed query answered: the window never narrows. */ @@ -279,6 +316,92 @@ const firstIssue = (error: any): string => { return `${issue?.path?.join('.') || 'root'}: ${clip(issue?.message, 120)}`; }; +/** An additive measure the family reads back as fixture rows: a count, or a sum. */ +interface AdditiveMeasure { + /** The measure's name — the column the door answers it under. */ + name: string; + aggregate: 'count' | 'sum'; + /** The field a count of a named field counts, or the field a sum sums. */ + field?: string; + /** The measure's own scoped filter, as parsed. */ + filter?: Record; +} + +/** How one probed measure is read back: its additive measures, and the derived value over them. */ +interface MeasurePlan { + /** The measure itself (a count or a sum), or a derived measure's operands. */ + additive: AdditiveMeasure[]; + derived?: { name: string; op: 'ratio' | 'sum' | 'difference' | 'product'; of: string[] }; +} + +type Excluded = { excluded: { kind: DashboardWindowExclusion; reason: string } }; + +/** Why a non-additive aggregate cannot be read back as fixture rows. */ +function notAdditive(name: string, m: any): string { + const of = m.field ? ` of ${m.field}` : ''; + switch (m.aggregate) { + case 'avg': + return `"${name}" averages${m.field ? ` ${m.field}` : ''}: an average is not additive — the rows the app already holds in the window enter its denominator, and an average does not say how many there are, so what the fixture rows change cannot be derived without guessing that number`; + case 'min': + case 'max': + return `"${name}" takes the ${m.aggregate === 'min' ? 'minimum' : 'maximum'}${of}: the fixture rows move it only where theirs is the extreme, which the rows the app already holds in the window decide`; + case 'count_distinct': + return `"${name}" counts the distinct values${of}: whether a fixture row adds one depends on the values the rows the app already holds in the window carry`; + default: + return `"${name}" aggregates ${m.aggregate ?? 'nothing'}${of}, which is not additive`; + } +} + +/** A count or a sum, read from the dataset's parsed measure; anything else names why not. */ +function additiveOf(name: string, m: any): { measure: AdditiveMeasure } | Excluded { + const filter = nonEmpty(m.filter) ? { filter: m.filter } : {}; + if (m.aggregate === 'count') { + if (typeof m.field === 'string' && m.field !== '*') { + if (m.field.includes('.')) { + return { excluded: { kind: 'count-field-unsynthesizable', reason: `"${name}" counts the relationship path ${m.field}, which a fixture row cannot be held to` } }; + } + return { measure: { name, aggregate: 'count', field: m.field, ...filter } }; + } + return { measure: { name, aggregate: 'count', ...filter } }; + } + if (m.aggregate === 'sum' && typeof m.field === 'string') { + if (m.field.includes('.')) { + return { excluded: { kind: 'count-field-unsynthesizable', reason: `"${name}" sums the relationship path ${m.field}, which a fixture row cannot be held to` } }; + } + return { measure: { name, aggregate: 'sum', field: m.field, ...filter } }; + } + return { excluded: { kind: 'not-a-count', reason: notAdditive(name, m) } }; +} + +/** + * The plan for one measure a widget selects: a count or a sum is read back + * itself; a derived measure through its operands, each of which must be a + * count or a sum declared in the same dataset (the spec's parse already + * refuses an operand it does not declare). + */ +function planOf(name: string, measures: Map, dataset: string): { plan: MeasurePlan } | Excluded { + const m = measures.get(name); + if (!m) return { excluded: { kind: 'measure-undeclared', reason: `dataset "${dataset}" declares no measure "${name}"` } }; + if (!m.derived) { + const a = additiveOf(name, m); + return 'excluded' in a ? a : { plan: { additive: [a.measure] } }; + } + const of: string[] = (m.derived.of ?? []).map(String); + const what = `"${name}" is derived (${m.derived.op} of ${of.join(', ')})`; + const additive: AdditiveMeasure[] = []; + for (const o of of) { + const om = measures.get(o); + if (!om) return { excluded: { kind: 'measure-undeclared', reason: `${what}, and dataset "${dataset}" declares no measure "${o}"` } }; + if (om.derived) { + return { excluded: { kind: 'not-a-count', reason: `${what}, and its operand "${o}" is itself derived: a derived measure is read through operands that are counts or sums` } }; + } + const a = additiveOf(o, om); + if ('excluded' in a) return { excluded: { kind: a.excluded.kind, reason: `${what}, and its operand ${a.excluded.reason}` } }; + if (!additive.some((x) => x.name === o)) additive.push(a.measure); + } + return { plan: { additive, derived: { name, op: m.derived.op, of } } }; +} + /** * Derive one case per widget of every dashboard the app declares with a * `dateRange`, as the spec's parse of the dashboard and of its datasets states @@ -290,7 +413,12 @@ export function deriveDashboardWindowCases(config: any): DashboardWindowCase[] { return derive(config).cases; } -function derive(config: any): { cases: DashboardWindowCase[]; dashboards: DashboardWindowDashboard[] } { +function derive(config: any): { + cases: DashboardWindowCase[]; + dashboards: DashboardWindowDashboard[]; + plans: Map; +} { + const plans = new Map(); const crud = new Map(deriveCrudCases(config).map((c) => [c.object, c])); const datasets = new Map(); for (const ds of declaredCollection(config, 'datasets')) if (ds?.name) datasets.set(ds.name, ds); @@ -366,41 +494,29 @@ function derive(config: any): { cases: DashboardWindowCase[]; dashboards: Dashbo const measures = new Map((dataset.measures ?? []).map((m: any) => [m.name, m])); for (const name of (Array.isArray(w?.values) ? w.values : []).map(String)) { - const m = measures.get(name); const mc: DashboardWindowMeasureCase = { measure: name }; c.measures.push(mc); - if (!m) { - mc.excluded = { kind: 'measure-undeclared', reason: `dataset "${w.dataset}" declares no measure "${name}"` }; - continue; - } - if (m.derived) { - mc.excluded = { kind: 'not-a-count', reason: `"${name}" is derived (${m.derived.op} of ${(m.derived.of ?? []).join(', ')}), not a row count` }; - continue; - } - if (m.aggregate !== 'count') { - mc.excluded = { kind: 'not-a-count', reason: `"${name}" aggregates ${m.aggregate ?? 'nothing'}${m.field ? ` of ${m.field}` : ''}, not a row count` }; + const planned = planOf(name, measures, String(w.dataset)); + if ('excluded' in planned) { + mc.excluded = planned.excluded; continue; } - if (typeof m.field === 'string' && m.field !== '*') { - if (m.field.includes('.')) { - mc.excluded = { kind: 'count-field-unsynthesizable', reason: `"${name}" counts the relationship path ${m.field}, which a fixture row cannot be held to` }; - continue; - } - mc.countField = m.field; - } - if (nonEmpty(m.filter)) mc.filter = m.filter; + plans.set(mc, planned.plan); + const own = planned.plan.derived ? undefined : planned.plan.additive[0]!; + if (own?.aggregate === 'count' && own.field) mc.countField = own.field; + if (own?.filter) mc.filter = own.filter; } if (c.measures.length > 0 && c.measures.every((m) => m.excluded)) { - const counts = c.measures.filter((m) => m.excluded!.kind !== 'not-a-count' && m.excluded!.kind !== 'measure-undeclared'); - c.excluded = counts.length > 0 - ? { kind: counts[0]!.excluded!.kind, reason: counts[0]!.excluded!.reason } - : { kind: 'no-count-measure', reason: `no measure the widget selects is a row count (${c.measures.map((m) => m.excluded!.reason).join('; ')})` }; + const fields = c.measures.filter((m) => m.excluded!.kind !== 'not-a-count' && m.excluded!.kind !== 'measure-undeclared'); + c.excluded = fields.length > 0 + ? { kind: fields[0]!.excluded!.kind, reason: fields[0]!.excluded!.reason } + : { kind: 'no-count-measure', reason: `no measure the widget selects can be compared (${c.measures.map((m) => m.excluded!.reason).join('; ')})` }; } else if (c.measures.length === 0) { c.excluded = { kind: 'no-count-measure', reason: 'the widget selects no measure' }; } }); } - return { cases, dashboards }; + return { cases, dashboards, plans }; } // ── Held columns: what the widget's own selection asks of a row ───────────── @@ -512,31 +628,92 @@ const nestedIn = (a: Record, b: Record): boole // ── The verdict ───────────────────────────────────────────────────────────── +/** One door answer, with the rows it returned. */ +interface Asked { + answer: DashboardWindowAnswer; + rows: Array>; +} + +/** A column summed over an answer's rows (a null value adds nothing). */ +const total = (asked: Asked, column: string): number => asked.rows.reduce((s, r) => s + (Number(r?.[column]) || 0), 0); + +/** + * The fixture rows one additive measure took in between two answers under the + * same window: a count's difference, or a sum's difference over the value + * each fixture row carries (`unit`). A sum of float values is snapped to the + * nearest whole row only within its own rounding noise (2^-36 of its + * magnitude: a naive float sum of up to 2^17 rows); where that noise reaches a + * quarter of a row, one row cannot be resolved and `undefined` is returned. + */ +function rowsTakenIn(before: Asked, after: Asked, a: AdditiveMeasure, unit: number): number | undefined { + const b = total(before, a.name); + const x = total(after, a.name); + if (a.aggregate === 'count') return x - b; + const noise = (Math.max(Math.abs(b), Math.abs(x), 1) * 2 ** -36) / Math.abs(unit); + if (noise >= 0.25) return undefined; + const rows = (x - b) / unit; + const whole = Math.round(rows); + return Math.abs(rows - whole) <= noise ? whole : rows; +} + +/** The first row whose derived value is not the server's own evaluator over that row's operands. */ +function inconsistentRow(after: Asked, d: NonNullable): string | undefined { + for (const row of after.rows) { + const made = evaluateDerivedMeasures([row], [d])[0]?.[d.name] ?? null; + const shown = row?.[d.name] ?? null; + if (made === null || shown === null ? made === shown : Number(made) === Number(shown)) continue; + const operands = d.of.map((o) => `${o} = ${JSON.stringify(row?.[o] ?? null)}`).join(', '); + return `the widget shows ${d.name} = ${JSON.stringify(shown)} where its operands in the same row (${operands}) make ${JSON.stringify(made)}`; + } + return undefined; +} + /** * Compare one windowed answer against the ground truth. The whole verdict of * the family is this function: it reads the expectation and the door's two - * answers under the same window, and nothing else. + * answers under the same window, and nothing else. Every additive measure the + * plan reads (the measure itself, or a derived measure's operands) must take + * in exactly the fixture rows the window selects, and a derived measure must + * show the server's evaluator over its operands. */ function judge( expected: number, - before: DashboardWindowAnswer, - after: DashboardWindowAnswer, + before: Asked, + after: Asked, + plan: MeasurePlan, + units: Map, ): Pick { - const failed = before.kind !== 'answered' ? before : after.kind !== 'answered' ? after : undefined; + const failed = before.answer.kind !== 'answered' ? before.answer : after.answer.kind !== 'answered' ? after.answer : undefined; if (failed) { return { outcome: 'failed', detail: `the windowed query answered ${failed.status}${failed.code ? ` ${failed.code}` : ''}: ${failed.detail ?? ''}` }; } - const answered = after.count! - before.count!; - const shown = after.count!; - if (expected > 0 && (after.rows === 0 || shown === 0)) { - return { outcome: 'blank', answered, shown, detail: `the widget showed nothing where the window selects ${expected} fixture row(s)` }; + const shown = after.answer.count!; + const taken: Array<{ a: AdditiveMeasure; rows: number }> = []; + for (const a of plan.additive) { + const rows = rowsTakenIn(before, after, a, units.get(a.name)!); + if (rows === undefined) { + return { outcome: 'not-run', shown, detail: `the sum ${a.name} totals ${total(after, a.name)}: at that size its float difference cannot resolve one fixture row of ${units.get(a.name)}` }; + } + taken.push({ a, rows }); } - if (answered === expected) return { outcome: 'held', answered, shown }; + const off = taken.find((t) => t.rows !== expected); + const answered = (off ?? taken[0]!).rows; + const blank = `the widget showed nothing where the window selects ${expected} fixture row(s)`; + if (expected > 0 && after.answer.rows === 0) return { outcome: 'blank', answered, shown, detail: blank }; + const inconsistent = plan.derived ? inconsistentRow(after, plan.derived) : undefined; + if (!off && !inconsistent) return { outcome: 'held', answered, shown }; + const nothingShown = plan.derived ? after.rows.every((r) => r?.[plan.derived!.name] == null) : shown === 0; + if (expected > 0 && nothingShown) return { outcome: 'blank', answered, shown, detail: blank }; + if (!off) return { outcome: 'contradicted', answered, shown, detail: inconsistent }; + const where = plan.derived ? ` in ${off.a.name}` : ''; + const sum = off.a.aggregate === 'sum' + ? ` (a sum of ${total(after, off.a.name) - total(before, off.a.name)} at ${units.get(off.a.name)} per row)` + : ''; return { outcome: 'contradicted', answered, shown, - detail: `the window counted ${answered} fixture row(s), and it selects ${expected} (${answered > expected ? 'wider' : 'narrower'})`, + detail: `the window counted ${off.rows} fixture row(s)${where}${sum}, and it selects ${expected} (${off.rows > expected ? 'wider' : 'narrower'})`, }; } @@ -544,12 +721,13 @@ function judge( /** * Run the dashboard date-window family against a live stack, as the principal - * `token` signs in: for every widget a `dateRange` binds, every count measure - * it selects is asked through the dataset door with no window and under each - * preset of `DATE_RANGE_PRESETS`, before and after fixture rows are written on - * known days, and each difference is compared with the rows that window - * selects. Returns a structured report; a per-widget failure is collected, - * not thrown. + * `token` signs in: for every widget a `dateRange` binds, every measure it + * selects that is a count, a sum or derived over those is asked through the + * dataset door with no window and under each preset of `DATE_RANGE_PRESETS`, + * before and after fixture rows are written on known days, and each + * difference, read back as fixture rows, is compared with the rows that + * window selects. Returns a structured report; a per-widget failure is + * collected, not thrown. * * The presets resolve once, when the run starts, on the current instant: the * server resolves them per request on its own clock, so no other instant @@ -561,7 +739,7 @@ export async function runDashboardWindowProofs( token: string, config: any, ): Promise { - const { cases, dashboards } = derive(config); + const { cases, dashboards, plans } = derive(config); const crud = new Map(deriveCrudCases(config).map((c) => [c.object, c])); const objectDefs = new Map(); for (const o of declaredCollection(config, 'objects')) if (o?.name) objectDefs.set(o.name, o); @@ -590,22 +768,27 @@ export async function runDashboardWindowProofs( }; // ── The door ────────────────────────────────────────────────────────────── - async function ask(dataset: string, selection: Record, measure: string): Promise { + async function ask(dataset: string, selection: Record, measure: string): Promise { try { const res = await stack.apiAs(token, 'POST', '/analytics/dataset/query', { datasetName: dataset, selection }); const body = (await res.json().catch(() => undefined)) as { rows?: unknown; code?: unknown; message?: unknown } | undefined; if (res.status === 200 && Array.isArray(body?.rows)) { const rows = body!.rows as Array>; - return { kind: 'answered', status: 200, rows: rows.length, count: rows.reduce((s, r) => s + (Number(r?.[measure]) || 0), 0) }; + const asked: Asked = { rows, answer: { kind: 'answered', status: 200, rows: rows.length } }; + asked.answer.count = total(asked, measure); + return asked; } return { - kind: res.status >= 500 ? 'faulted' : 'refused', - status: res.status, - ...(typeof body?.code === 'string' ? { code: body.code } : {}), - detail: clip(body?.message), + rows: [], + answer: { + kind: res.status >= 500 ? 'faulted' : 'refused', + status: res.status, + ...(typeof body?.code === 'string' ? { code: body.code } : {}), + detail: clip(body?.message), + }, }; } catch (e) { - return { kind: 'faulted', status: 0, detail: clip((e as Error)?.message ?? e) }; + return { rows: [], answer: { kind: 'faulted', status: 0, detail: clip((e as Error)?.message ?? e) } }; } } @@ -648,6 +831,66 @@ export async function runDashboardWindowProofs( return made; } + // ── Held columns per measure ────────────────────────────────────────────── + /** + * Hold, onto the widget's own held columns, what every additive measure the + * plan reads asks of a fixture row: its filter, a value in the field a count + * counts, and a non-zero number in the field a sum sums. `units` is what one + * fixture row adds to each (1 for a count, the held value for a sum). + */ + async function holdFor( + c: DashboardWindowCase, + plan: MeasurePlan, + held: Map, + holdCtx: Parameters[2], + ): Promise<{ held: Map; units: Map } | Excluded> { + const units = new Map(); + const cc = crud.get(c.object!)!; + for (const a of plan.additive) { + const filterReason = holdColumns(a.filter, held, holdCtx); + if (filterReason) { + const whose = plan.derived ? `operand ${a.name}'s` : "measure's own"; + return { excluded: { kind: 'filter-unmodelled', reason: `the ${whose} filter carries ${filterReason}, which the ground truth cannot model` } }; + } + if (a.aggregate === 'count') { + units.set(a.name, 1); + // A count of a field counts its non-null values: hold one. + if (!a.field || held.has(a.field)) continue; + const ref = (cc.relationalRefs ?? []).find((x) => x.field === a.field); + if (ref) { + const dep = await baseRecord(ref.target); + if (!dep.id) { + return { excluded: { kind: 'count-field-unsynthesizable', reason: `"${a.name}" counts ${a.field}, and no ${ref.target} record can be written for it: ${dep.reason}` } }; + } + held.set(a.field, ref.multiple ? [dep.id] : dep.id); + } else if (cc.body && a.field in cc.body) { + held.set(a.field, cc.body[a.field]); + } else { + return { excluded: { kind: 'count-field-unsynthesizable', reason: `"${a.name}" counts ${a.field}, and the CRUD derivation synthesizes no value for it` } }; + } + continue; + } + // A sum adds the value each fixture row carries: the one the selection + // already holds there, else the one the CRUD derivation synthesizes (it + // honours the field's declared min / max / step). + const field = a.field!; + const value = held.has(field) ? held.get(field) : cc.body?.[field]; + if (typeof value !== 'number' || !Number.isFinite(value) || value === 0) { + return { + excluded: { + kind: 'count-field-unsynthesizable', + reason: value === undefined + ? `"${a.name}" sums ${field}, and the CRUD derivation synthesizes no value for it` + : `"${a.name}" sums ${field}, and the value a fixture row would carry there (${clip(JSON.stringify(value), 40)}) is not a non-zero number, so the rows it adds cannot be read back`, + }, + }; + } + held.set(field, value); + units.set(a.name, value); + } + return { held, units }; + } + // ── Probes ──────────────────────────────────────────────────────────────── const results: DashboardWindowWidgetResult[] = []; for (const c of cases) { @@ -715,34 +958,16 @@ export async function runDashboardWindowProofs( for (const [i, m] of c.measures.entries()) { const r = measureResults[i]!; if (m.excluded) continue; - const held = new Map(objectHeld); - const measureReason = holdColumns(m.filter, held, holdCtx); - if (measureReason) { - Object.assign(r, { reason: `the measure's own filter carries ${measureReason}, which the ground truth cannot model`, excludedKind: 'filter-unmodelled' }); + const plan = plans.get(m)!; + const holding = await holdFor(c, plan, new Map(objectHeld), holdCtx); + if ('excluded' in holding) { + Object.assign(r, { reason: holding.excluded.reason, excludedKind: holding.excluded.kind }); continue; } - // A count of a field counts its non-null values: hold one. - if (m.countField && !held.has(m.countField)) { - const cc = crud.get(c.object!)!; - const ref = (cc.relationalRefs ?? []).find((x) => x.field === m.countField); - if (ref) { - const dep = await baseRecord(ref.target); - if (!dep.id) { - Object.assign(r, { reason: `"${m.measure}" counts ${m.countField}, and no ${ref.target} record can be written for it: ${dep.reason}`, excludedKind: 'count-field-unsynthesizable' }); - continue; - } - held.set(m.countField, ref.multiple ? [dep.id] : dep.id); - } else if (cc.body && m.countField in cc.body) { - held.set(m.countField, cc.body[m.countField]); - } else { - Object.assign(r, { reason: `"${m.measure}" counts ${m.countField}, and the CRUD derivation synthesizes no value for it`, excludedKind: 'count-field-unsynthesizable' }); - continue; - } - } r.status = 'probed'; delete r.reason; delete r.excludedKind; - await probeMeasure(c, m, r, held, fieldType, ownerColumn); + await probeMeasure(c, m, r, holding.held, fieldType, ownerColumn, plan, holding.units); } const probed = measureResults.filter((m) => m.status === 'probed'); @@ -751,7 +976,7 @@ export async function runDashboardWindowProofs( results.push({ ...base, excludedKind: first?.excludedKind ?? 'no-count-measure', - reason: first?.reason ?? 'no measure the widget selects is a row count', + reason: first?.reason ?? 'no measure the widget selects can be compared', measures: measureResults, }); continue; @@ -766,6 +991,8 @@ export async function runDashboardWindowProofs( held: Map, fieldType: string, ownerColumn: boolean, + plan: MeasurePlan, + units: Map, ): Promise { const selection = (window?: Record): Record => { const windowed = window ? { [c.field!]: window } : undefined; @@ -787,9 +1014,10 @@ export async function runDashboardWindowProofs( // The control leg, before any fixture: the widget's own query must answer. const before0 = await ask(c.dataset!, selection(), m.measure); - if (before0.kind !== 'answered') { - r.unwindowed = { expected: 0, answer: before0 }; - notRun(`the widget's own query, with no window, answered ${before0.status}${before0.code ? ` ${before0.code}` : ''}: ${before0.detail ?? ''}`); + if (before0.answer.kind !== 'answered') { + const refused = before0.answer; + r.unwindowed = { expected: 0, answer: refused }; + notRun(`the widget's own query, with no window, answered ${refused.status}${refused.code ? ` ${refused.code}` : ''}: ${refused.detail ?? ''}`); return; } const day0 = todayAt(new Date()); @@ -800,7 +1028,7 @@ export async function runDashboardWindowProofs( notRun(`the day turned after the run started (${startDay} → ${day0}), so the windows the presets resolved then are not the ones the server resolves now`); return; } - const before = new Map(); + const before = new Map(); for (const p of presets) before.set(p.preset, await ask(c.dataset!, selection(p.condition), m.measure)); // Fixtures: one row per day, every held column held, owned by the principal. @@ -879,21 +1107,37 @@ export async function runDashboardWindowProofs( } const after0 = await ask(c.dataset!, selection(), m.measure); - const after = new Map(); + const after = new Map(); for (const p of presets) after.set(p.preset, await ask(c.dataset!, selection(p.condition), m.measure)); if (todayAt(new Date()) !== day0) { notRun(`the day turned during the probe (${day0} → ${todayAt(new Date())}), so the windows the server resolved moved under the fixtures`); return; } - // The control leg: with no window, the widget counts every fixture row. - const unwindowedAnswered = after0.kind === 'answered' ? after0.count! - before0.count! : undefined; - r.unwindowed = { expected: written.length, ...(unwindowedAnswered !== undefined ? { answered: unwindowedAnswered } : {}), answer: after0 }; + // The control leg: with no window, the widget takes in every fixture row — + // every additive measure the plan reads, each by the rows it took in. A + // derived measure's operands are read from the widget's own answer, so + // that answer must carry them. + const answered0 = after0.answer.kind === 'answered'; + const unread = answered0 && plan.derived + ? plan.additive.find((a) => after0.rows.length > 0 && after0.rows.every((row) => !(a.name in (row ?? {})))) + : undefined; + const taken0 = answered0 && !unread ? plan.additive.map((a) => rowsTakenIn(before0, after0, a, units.get(a.name)!)) : []; + const unresolved = plan.additive.find((_, k) => taken0.length > 0 && taken0[k] === undefined); + const unwindowedAnswered = taken0.length > 0 && !unresolved + ? (taken0.find((t) => t !== written.length) ?? taken0[0]) + : undefined; + r.unwindowed = { expected: written.length, ...(unwindowedAnswered !== undefined ? { answered: unwindowedAnswered } : {}), answer: after0.answer }; if (unwindowedAnswered !== written.length) { + const verb = plan.additive.every((a) => a.aggregate === 'count') ? 'counts' : 'takes in'; notRun( - after0.kind !== 'answered' - ? `the widget's own query, with no window, answered ${after0.status} once the fixtures were written: ${after0.detail ?? ''}` - : `with no window the widget counts ${unwindowedAnswered} of the ${written.length} fixture rows, so the fixtures do not meet its own selection and no window can be judged`, + !answered0 + ? `the widget's own query, with no window, answered ${after0.answer.status} once the fixtures were written: ${after0.answer.detail ?? ''}` + : unread + ? `the widget's answer carries no ${unread.name} column, so the operands of "${m.measure}" cannot be read back` + : unresolved + ? `the sum ${unresolved.name} totals ${total(after0, unresolved.name)}: at that size its float difference cannot resolve one fixture row of ${units.get(unresolved.name)}` + : `with no window the widget ${verb} ${unwindowedAnswered} of the ${written.length} fixture rows, so the fixtures do not meet its own selection and no window can be judged`, ); return; } @@ -902,7 +1146,7 @@ export async function runDashboardWindowProofs( const expected = written.filter((w) => selects(p.resolved, w.day)).length; const b = before.get(p.preset)!; const a = after.get(p.preset)!; - return { preset: p.preset, expected, ...judge(expected, b, a), answer: a }; + return { preset: p.preset, expected, ...judge(expected, b, a, plan, units), answer: a.answer }; }); // An inert window: every preset answered what no window answered. @@ -986,7 +1230,7 @@ function widgetStatus(measures: DashboardWindowMeasureResult[]): { status: Dashb export function formatDashboardWindowReport(report: DashboardWindowReport): string { const lines: string[] = [`\n=== objectstack verify — dashboard date windows — ${report.app} ===`]; lines.push( - ` door: POST /analytics/dataset/query as the signed-in principal; ${report.presets.length} preset(s) per count measure, ` + + ` door: POST /analytics/dataset/query as the signed-in principal; ${report.presets.length} preset(s) per measure, ` + `each window resolved on ${report.timezone} calendar days`, ); for (const d of report.dashboards.filter((x) => x.status === 'excluded')) { @@ -1015,7 +1259,7 @@ export function formatDashboardWindowReport(report: DashboardWindowReport): stri lines.push( ` ── ${s.widgets.proven} proven, ${s.widgets.contradicted} CONTRADICTED, ${s.widgets.failed} FAILED, ` + `${s.widgets.inconclusive} inconclusive, ${s.widgets.unbound} unbound, ${s.widgets.excluded} excluded ` + - `(${s.dashboards.windowed} windowed dashboard(s) of ${s.dashboards.declared}; ${s.measures.probed} count measure(s) probed, ` + + `(${s.dashboards.windowed} windowed dashboard(s) of ${s.dashboards.declared}; ${s.measures.probed} measure(s) probed, ` + `${s.measures.excluded} excluded; ${s.probes} window(s): ${s.held} held, ${s.blank} blank, ${s.contradictions} contradicted, ` + `${s.faults} faulted, ${s.notRun} not run; ${s.inert} inert, ${s.narrowing} narrowing violation(s))`, );