diff --git a/reference/core/control-center/evidence/gap-closure-evidence.json b/reference/core/control-center/evidence/gap-closure-evidence.json index 01c32c836..e4c6fb83e 100644 --- a/reference/core/control-center/evidence/gap-closure-evidence.json +++ b/reference/core/control-center/evidence/gap-closure-evidence.json @@ -9908,6 +9908,23 @@ ], "dependencyDisposition": "accepted-scope", "dependencyRationale": "Coverage of this pack is the coverage of whatever analyser the tenant points it at, never the 138 weaknesses the standard names, and notEvaluableHere records that rather than leaving a reader to infer it. Publishing a rule per weakness would advertise 138 checks nothing performs - GT-585 measured what that produces here. Deliberately NOT done: scoring. A count of weaknesses is not a compliance verdict and the standard's scoring is a policy decision belonging to a tenant's selected ruleset. Two defects were surfaced and fixed rather than recorded: GOV-CORE-UNRESOLVED misdiagnosing a narrowed run as an unresolved Core, and a run reporting passed over skipped rules once that misdiagnosis was out of the way." + }, + { + "id": "GT-663", + "closedAt": "2026-08-09", + "closureCommit": "059c8772", + "evidence": [ + "src/packages/core-domain/src/application/validators/standards/iso-5055-coverage.ts", + "src/packages/core-domain/src/application/validators/standards/iso-5055-coverage.spec.ts", + "src/packages/core-domain/src/application/validators/enforcement/adapters/iso-5055-adapter.ts" + ], + "validationCommands": [ + "npx jest --config src/packages/core-domain/jest.config.js --testPathPatterns iso-5055 -> 35 passed.", + "iso5055CoverageFromSarif over the 75 live CodeQL alerts -> observed 10 of 138; Security 10/74 (34 findings), Reliability 0/74, Performance Efficiency 0/18, Maintainability 2/31; 28 out of scope, 13 untagged.", + "describeIso5055Coverage -> the sentence names the floor as a FLOOR and states it says nothing about the remaining 128." + ], + "dependencyDisposition": "accepted-scope", + "dependencyRationale": "Deliberately NOT computed: what the analyser COULD have found. That needs the scanner's own rule catalogue - a vendor call for CodeQL, a registry fetch for semgrep - and a 'could have' number derived from anything less would invent the assurance this row exists to withhold. observed is a floor and is labelled one. The measurement also makes the next step evidenced rather than asserted: Reliability 0/74 and Performance Efficiency 0/18 say CodeQL does not look there at all, which is the concrete argument for adding a second analyser." } ] } diff --git a/reference/core/control-center/gaps/gap-reference-catalog.es.md b/reference/core/control-center/gaps/gap-reference-catalog.es.md index a2fb7e49f..b0d34ac19 100644 --- a/reference/core/control-center/gaps/gap-reference-catalog.es.md +++ b/reference/core/control-center/gaps/gap-reference-catalog.es.md @@ -8617,3 +8617,37 @@ La lección es la del propio tablero y esta vez la pagó quien medía: un `conta **Lo que la rebanada 1 deliberadamente NO hace:** puntuar, ni descartar hallazgos fuera de alcance. Un conteo de debilidades no es un veredicto de cumplimiento, y la puntuación del estándar es una decisión de política que pertenece al ruleset que el tenant seleccione. Los hallazgos cuyos CWE el estándar no nombra se devuelven con medidas vacías en vez de descartarse, porque «este escaneo no encontró nada que a ISO 5055 le importe» y «este escaneo no encontró nada» son reportes distintos y solo uno habla del código. +#### GT-663 + +**Title:** La medición ISO/IEC 5055 no tenía denominador — «no encontró ninguna» se leía igual que «no busca ninguna» + +- **Purpose:** Hacer legible un veredicto ISO/IEC 5055 publicando de qué es fracción. +- **Evidence:** `GT-662` hizo real la medición: los hallazgos de CodeQL de este repositorio mapean a **34 violaciones sobre 10 debilidades distintas**. Lo que el reporte no podía decir es *de cuántas*. El estándar nombra **138**, así que una corrida que no encuentra nada se lee idéntica tanto si el analizador busca las 138 como si no busca ninguna — **y lo segundo es el caso común**, porque la cobertura aquí es la del ANALIZADOR y nunca la del estándar. Es la forma exacta que `GT-569` arregló para la cobertura de reglas un nivel más abajo: un conteo sin su denominador redefine en silencio lo que cuenta. +- **Impact:** Un veredicto ISO/IEC 5055 verde es la salida más citable que produce este motor para el ICP de la §11.2 del posicionamiento. Enviado sin denominador es una garantía que nadie se ganó. +- **Affected files:** `src/packages/core-domain/src/application/validators/standards/iso-5055-coverage.ts` +- **Component:** `Evolith Core` · **Criticality:** P2 · **Complexity:** S +- **Principal:** `S` · **Interest:** `MED` · **Basis:** `estimate` +- **Acceptance criteria:** + - [x] Todo reporte ISO/IEC 5055 lleva el número de debilidades distintas observadas Y las 138 que nombra el estándar, por medida y en total. + - [x] El número se etiqueta como SUELO en el texto que viaja con él, y nunca afirma que las no observadas estén ausentes. + - [x] «Fuera del estándar» y «el analizador no etiquetó nada» se cuentan por separado, porque significan cosas distintas. + - [x] Medido contra los hallazgos reales de este repositorio, no contra un fixture. +- **Status:** `COMPLETADO` (2026-08-09) + +**ENTREGADO el 2026-08-09.** `iso5055CoverageFromSarif` reporta **observadas / 138**, por medida, junto a los hallazgos; `describeIso5055Coverage` escribe la única frase que comparten todas las superficies, para que el matiz no lo pueda quitar quien renderice. + +**Medido sobre los hallazgos vivos de CodeQL de este repositorio — y el resultado es un hallazgo de producto, no solo de test:** + +| medida | observadas / total | hallazgos | +|---|---|---| +| Security | **10 / 74** | 34 | +| Reliability | **0 / 74** | 0 | +| Performance Efficiency | **0 / 18** | 0 | +| Maintainability | **2 / 31** | 2 | + +**Dos de las cuatro medidas están en cero, y nada lo decía antes.** CodeQL no busca debilidades de Reliability ni de Performance Efficiency en absoluto — que es el argumento concreto y con evidencia para añadir un segundo analizador, y era invisible mientras el reporte no tenía denominador. También sale a la luz: **28 hallazgos llevaban un CWE fuera del estándar y 13 no llevaban ninguno**, contados por separado porque «no está en ISO/IEC 5055» y «el analizador no nos dijo nada mapeable» son hechos distintos y solo uno habla del código. + +**El aviso se emite SOLO en una corrida limpia, y una vez.** Es cuando el denominador importa y cuando el reporte antes callaba; cuando hay hallazgos el lector ya tiene algo concreto que hacer, y repetir el matiz por hallazgo le enseñaría a saltárselo. No bloqueante por construcción —las reglas del pack son `blocking: false`— así que informa un veredicto y nunca lo decide. + +**Lo que se niega a calcular deliberadamente:** qué PODRÍA haber encontrado el analizador. Eso necesita el catálogo de reglas del propio escáner —una llamada al proveedor para CodeQL, una al registro para semgrep— y un número de «podría» derivado de menos que eso inventaría justo la garantía que esta ficha existe para no dar. Por eso `observed` es un suelo y está etiquetado como tal. + diff --git a/reference/core/control-center/gaps/gap-reference-catalog.md b/reference/core/control-center/gaps/gap-reference-catalog.md index 38e4cfc60..8b3028550 100644 --- a/reference/core/control-center/gaps/gap-reference-catalog.md +++ b/reference/core/control-center/gaps/gap-reference-catalog.md @@ -8711,3 +8711,37 @@ The lesson is the board's own, and this time the measurer paid it: a `contains` **What slice 1 deliberately does NOT do:** score, and drop out-of-scope findings. A count of weaknesses is not a compliance verdict, and the standard's scoring is a policy decision belonging to a tenant's selected ruleset. Findings whose CWEs the standard does not name are returned with empty measures rather than discarded, because «this scan found nothing ISO 5055 cares about» and «this scan found nothing» are different reports and only one is about the code. +#### GT-663 + +**Title:** The ISO/IEC 5055 measurement had no denominator — «found none» read the same as «looks for none» + +- **Purpose:** Make an ISO/IEC 5055 verdict legible by publishing what it is a fraction OF. +- **Evidence:** `GT-662` made the measurement real: this repository's CodeQL findings map to **34 violations across 10 distinct weaknesses**. What the report could not say is *of how many*. The standard names **138**, so a run that finds nothing reads identically whether the analyser looks for all 138 or for none of them — **and the second is the common case**, because coverage here is the ANALYSER's and never the standard's. This is the exact shape `GT-569` fixed for rule coverage one level down: a count without its denominator silently redefines what it counts. +- **Impact:** A green ISO/IEC 5055 verdict is the single most quotable output this engine produces for the ICP in §11.2 of the positioning. Shipped without a denominator it is an assurance nobody earned. +- **Affected files:** `src/packages/core-domain/src/application/validators/standards/iso-5055-coverage.ts` +- **Component:** `Evolith Core` · **Criticality:** P2 · **Complexity:** S +- **Principal:** `S` · **Interest:** `MED` · **Basis:** `estimate` +- **Acceptance criteria:** + - [x] Every ISO/IEC 5055 report carries the number of distinct weaknesses observed AND the 138 the standard names, per measure as well as overall. + - [x] The number is labelled a FLOOR in the text that travels with it, and never claims the unobserved weaknesses are absent. + - [x] «Outside the standard» and «the analyser tagged nothing» are counted separately, because they mean different things. + - [x] Measured against this repository's real findings, not a fixture. +- **Status:** `DONE` (2026-08-09) + +**DELIVERED 2026-08-09.** `iso5055CoverageFromSarif` reports **observed / 138**, per measure, alongside the findings; `describeIso5055Coverage` writes the one sentence all surfaces share, so the caveat cannot be dropped by whoever renders it. + +**Measured on this repository's live CodeQL findings — and the result is a product finding, not just a test:** + +| measure | observed / total | findings | +|---|---|---| +| Security | **10 / 74** | 34 | +| Reliability | **0 / 74** | 0 | +| Performance Efficiency | **0 / 18** | 0 | +| Maintainability | **2 / 31** | 2 | + +**Two of the four measures are at zero, and nothing said so before.** CodeQL does not look for Reliability or Performance Efficiency weaknesses at all — which is the concrete, evidenced argument for adding a second analyser, and it was invisible while the report had no denominator. Also surfaced: **28 findings carried a CWE outside the standard and 13 carried none at all**, both counted separately because «not in ISO/IEC 5055» and «the analyser told us nothing we could map» are different facts and only one of them is about the code. + +**The advisory is emitted ONLY on a clean run, and once.** That is when the denominator matters and when the report used to be silent; when there are findings the reader already has something concrete to act on, and repeating the caveat per finding would train them to skip it. Non-blocking by construction — the pack's rules are `blocking: false` — so it informs a verdict and never decides one. + +**What it deliberately refuses to compute:** what the analyser COULD have found. That needs the scanner's own rule catalogue — a vendor call for CodeQL, a registry fetch for semgrep — and a «could have» number derived from anything less would invent the very assurance this row exists to withhold. So `observed` is a floor and is labelled one. + diff --git a/reference/core/control-center/gaps/gap-tracking.es.md b/reference/core/control-center/gaps/gap-tracking.es.md index 2a20cfc07..5472ae560 100644 --- a/reference/core/control-center/gaps/gap-tracking.es.md +++ b/reference/core/control-center/gaps/gap-tracking.es.md @@ -675,9 +675,10 @@ Este tablero es la única fuente de verdad para deuda técnica, gaps, oportunida | [`GT-660`](./gap-reference-catalog.es.md#gt-660) | **El Core propone un menú que nadie puede leer — `--select` nombra un catálogo que ninguna superficie CLI ni MCP publica.** `GT-659` entregó selección en las tres superficies, con cada texto de ayuda apuntando a «el id que publica el catálogo». REST sí publica uno (`GET /api/v1/rulesets`); **el CLI y MCP no publican nada**, así que seleccionar solo era posible para quien ya se supiera de memoria un nombre de fichero — «configurable por tenant» cierto sobre el papel, falso en una terminal. **Y el catálogo de REST es una SEGUNDA derivación**, que construye un `id` desde `metadata.id` / `$id` / la ruta sin extensión mientras el selector compara contra `sourceFile`: medido, **17 de 183 ids publicados no casan con ninguna regla**, así que seguir el consejo del propio endpoint devuelve un `SEL-01` bloqueante. **ARREGLADO el 2026-08-09 con UNA derivación:** `buildRulesetCatalog` lee el mismo `loadAllRulesets` que evalúa el motor, expuesto una sola vez en `RulesetValidatorService.catalog` — sin segunda vía de construcción, y por tanto sin segunda respuesta. `evolith rulesets` (CLI, ADR-0073 bajo `--format json`) y `evolith-ruleset-list` (MCP) reportan ambos **174 packs · 402 reglas · 188 que pueden hacer fallar una corrida**, idéntico. El conteo de bloqueantes se publica POR PACK a propósito: un tenant tiene derecho a saber qué packs ponen su build en rojo antes del primer rojo. REST conserva `id` y gana `ref` — **183 de 183 seleccionables frente a 166 de 183** — aditivo porque `id` está publicado y hay clientes que lo sostienen. **Dos números míos los corrigió la herramienta que esta ficha construyó:** el «corpus de 372» del README de la action era un conteo a mano; el motor dice **402**, y el «235 de 402» de GT-659 era correcto sobre el corpus y se midió en un satélite. **Deliberadamente NO hecho aquí:** el default. No nombrar nada sigue evaluando todo — **85 bloqueantes de 113 issues** en este repositorio — así que un tenant que no adoptó nada sigue bloqueado. Eso es una decisión, no un defecto: [`GT-661`](./gap-reference-catalog.es.md#gt-661). | `Cross` | Cross | P1 | M | `COMPLETADO` | | [`GT-661`](./gap-reference-catalog.es.md#gt-661) | **«Sin selección» significa el corpus completo, así que un tenant que no adoptó nada sigue bloqueado por todo.** El principio del dueño es que el Core PROPONE y el cliente configura. Medido sin selección en este repositorio: **85 issues bloqueantes de 113, status `failed`** — un llamador que no adoptó nada recibe un veredicto que bloquea, de reglas que nunca eligió. SSDF no es el problema (sus 8 reglas son `blocking: false` y solo reportan); las otras **188** bloqueantes sí. `GT-659` hizo expresable una selección y [`GT-660`](./gap-reference-catalog.es.md#gt-660) hizo legible el menú, que es lo que convierte esto de pregunta sin respuesta en decisión. **El riesgo se dice porque es la razón de que esto no se haga sin más:** todo consumidor actual —incluido el CI de este repositorio— depende de que `validate` bloquee sin flags, y un default que dejara de bloquear desarmaría en silencio gates que hoy funcionan, lo que es un fallo peor que el que se arregla. **Principal:** `M` · **Interest:** `MED` · **Basis:** `estimate`. **RESUELTO el 2026-08-09.** **«Sin selección» sigue significando el corpus completo, y el Core lo dice en voz alta.** El principio no exige que el Core deje de bloquear; exige que el Core no sea quien decide qué adoptó un tenant. Sin selección el Core no tiene configuración de tenant que consultar, se niega a inventarla, y etiqueta ese alcance como `core-default` — una PROPUESTA, no una elección del tenant. **La alternativa se midió y se descartó:** un default que dejara de bloquear desarmaría en silencio todos los gates que hoy funcionan, incluido el CI de este repositorio. **Lo que cambia es que el veredicto ya se puede LEER:** `ValidationResult.selection` publica `source` (`caller` o `core-default`), `requested`, `matched`, `unmatched`, `rulesSelected` y `corpusTotal`. Medido en el CLI — sin selección da `core-default`, 402 de 402, `failed`; con `--select standards/ssdf-v1.1.rules.json` da `caller`, 8 de 402, `warning`. Antes ambos renderizaban el mismo rojo. `unmatched` es un CAMPO y no texto de issue, porque cero reglas con cero violaciones es indistinguible de un repositorio limpio. **Faltaba la mitad del cliente y está construida:** `ProfileConfig.select` deja que un tenant configure sus packs adoptados UNA VEZ — una bandera que hay que reescribir en cada invocación no es configuración. `--select` sigue ganando cuando se da, incluso para AMPLIAR un default guardado. Ambos caminos descartan blancos, así que un perfil vacío significa «no configuró nada», nunca «configuró un alcance vacío». Las cuatro superficies del CLI lo llevan. **No se desarmó nada, y eso se verifica en vez de afirmarse:** el reporte ganó un campo, la corrida no perdió una regla. | `Evolith Core` | Cross | P2 | M | `COMPLETADO` | | [`GT-662`](./gap-reference-catalog.es.md#gt-662) | **ISO/IEC 5055 como una medición que Evolith sí puede hacer — CWE→medida, desde los analizadores que el tenant ya corre.** `GT-600` nombró el estándar que sigue a SSDF; ISO/IEC 5055 es el único ISO **automatizado por construcción** — 138 debilidades estructurales, cada una un CWE, en cuatro medidas. El corpus ya envía el índice y un mapeo de 404 filas, y ninguno es un CHECK: `strength` es `none` en **367 de 404** reglas. Un handler nativo no puede cerrarlo (filesystem, sin parser), y 138 reglas inejecutables es lo que `GT-585` midió como **78 de 96 resultados bloqueantes que no podían correr**. **REBANADA 1 ENTREGADA el 2026-08-09:** la capa de traducción `analizador → SARIF → CWE → debilidad → medida`, sin escribir un parser — ya existen cuatro adaptadores de enforcer y CodeQL ya corre aquí. **Medido sobre los hallazgos vivos de este repositorio: de 75 alertas abiertas de CodeQL, 62 llevan un CWE y 34 son debilidades que ISO/IEC 5055 nombra** — diez distintas (22, 23, 36, 78, 79, 88, 99, 434, 570, 571), puntuando 34 en Security y 2 en Maintainability. Una medición ISO real, de un escáner que ya estaba corriendo, sin costo. **Una restricción encontrada midiendo:** el SARIF de `analyses/{id}` de GitHub sirve `tool.driver` con solo `name` y `semanticVersion` — sin `rules[]`, con los resultados referenciando la regla por índice pelado — así que contra ESE endpoint el mapeo encuentra **0 de 75**. Los CWE viven en la API de alerts y en el SARIF que CodeQL sube. Un diseño que hubiera supuesto lo contrario habría enviado un cero permanente y silencioso. **Sigue abierto:** el pack `iso-5055.rules.json` con descriptores `enforce:` y un bloque `notEvaluableHere`, seleccionable por tenant como cualquier otro. **REBANADA 2 ENTREGADA el 2026-08-09.** El pack envía **cuatro reglas, una por medida**, enrutadas por `createIso5055Adapter`. Verificado extremo a extremo contra los mismos hallazgos vivos de CodeQL: **entran 75 alertas, salen 36 violaciones — 34 `ISO5055-SEC`, 2 `ISO5055-MAINT`** — reproduciendo la rebanada 1 por el camino real del adaptador. Publicado por `evolith rulesets` (**175 packs / 406 reglas**, `blocking: 0`) y seleccionable: **4 de 406**. El escáner sigue siendo gratuito: `semgrep --config p/default`, deliberadamente NO `--config auto` (que necesita red y sesión de proveedor), y un tenant que ya emite SARIF apunta `enforce.config.sarif` en vez de escanear dos veces. **Dos defectos sacados a la luz por este pack, ambos arreglados.** (1) Con el analizador ausente, seleccionarlo producía un `GOV-CORE-UNRESOLVED` BLOQUEANTE diciéndole al operador que pasara `--core` cuando ya lo había pasado — el Core había resuelto 406 reglas y 4 estaban seleccionadas. El `corpusTotal` de GT-661 separa «el Core no resolvió nada» de «el llamador acotó a reglas que no corrieron»; un corpus de CERO sigue siendo fallo duro, así que GT-474 se mantiene. (2) Arreglar eso reveló una corrida reportando **`passed` sobre 4 reglas omitidas** — la falsa garantía exacta que este backlog sigue encontrando, por una puerta que yo acababa de abrir. Las reglas ahora son `MUST` + `blocking: false`, un par funcional: MUST hace que GT-569 emita el aviso que convierte el verde en `warning`, blocking:false impide que rompa un build por cobertura nunca reclamada. Medido después: `warning`, 4 omitidas, 4 avisos. **La cobertura es la del analizador, no la del estándar, y `notEvaluableHere` lo dice:** un verde significa que el analizador configurado no encontró ninguna debilidad ISO/IEC 5055, NO que el repositorio esté libre de ellas. | `Evolith Core` | Cross | P2 | L | `COMPLETADO` | +| [`GT-663`](./gap-reference-catalog.es.md#gt-663) | **La medición ISO/IEC 5055 no tenía denominador — «no encontró ninguna» se leía igual que «no busca ninguna».** `GT-662` hizo real la medición (34 violaciones sobre 10 debilidades distintas aquí) pero no podía decir *de cuántas*. El estándar nombra **138**, así que una corrida limpia se lee idéntica tanto si el analizador cubre las 138 como ninguna — **y lo segundo es el caso común**, porque la cobertura es la del ANALIZADOR, nunca la del estándar. Misma forma que `GT-569` arregló un nivel más abajo. **ENTREGADO el 2026-08-09.** Todo reporte lleva ahora observadas / 138, por medida, con el matiz escrito una vez y compartido por todas las superficies para que un renderer no lo pueda quitar. **Medido en vivo, y el resultado es un hallazgo de producto:** Security **10/74** (34 hallazgos), Reliability **0/74**, Performance Efficiency **0/18**, Maintainability **2/31**. **Dos de cuatro medidas están en CERO y nada lo decía antes** — CodeQL no busca Reliability ni Performance Efficiency en absoluto, que es el argumento con evidencia para un segundo analizador y era invisible sin denominador. También sale: 28 hallazgos con un CWE fuera del estándar y 13 sin ninguno, contados aparte porque «no está en el estándar» y «el analizador no dijo nada mapeable» son hechos distintos. Se emite SOLO en corrida limpia y una vez — es cuando el denominador importa y cuando el reporte antes callaba; repetirlo por hallazgo enseñaría al lector a saltárselo. No bloqueante: informa un veredicto y nunca lo decide. **Se niega a calcular qué PODRÍA haber encontrado el analizador:** eso necesita el catálogo del propio escáner, y un «podría» derivado de menos inventaría la garantía que esta ficha existe para no dar. `observed` es un suelo y está etiquetado como tal. | `Evolith Core` | Cross | P2 | S | `COMPLETADO` | -**Progreso:** 648 / 660 completados · 2 en progreso · 0 pendientes · 10 diferidos +**Progreso:** 649 / 661 completados · 2 en progreso · 0 pendientes · 10 diferidos **Oleada 2026-06-23 (auditoría profunda de Winston III):** Añadidos 14 gaps nuevos `GT-212`…`GT-225` del Winston Audit Playbook que cubren: higiene de estado ADR (GT-212), metadata + presupuestos operativos + corpus de guías por topología (GT-213, GT-217, GT-219), observabilidad + OpenAPI en controladores REST (GT-214, GT-215), paridad de input-schemas OPA + densidad de tests por topología (GT-216, GT-222), plantillas de rollback + on-call de Fase 05 (GT-218), cobertura de ramas CLI + paridad de envelope --format + limpieza de skip-list (GT-220, GT-224, GT-225), audit logging HTTP de MCP (GT-221), y tests e2e de paridad cross-surface (GT-223). diff --git a/reference/core/control-center/gaps/gap-tracking.md b/reference/core/control-center/gaps/gap-tracking.md index 3c0749a11..5cc262298 100644 --- a/reference/core/control-center/gaps/gap-tracking.md +++ b/reference/core/control-center/gaps/gap-tracking.md @@ -675,9 +675,10 @@ This board is the single source of truth for technical debt, gaps, opportunities | [`GT-660`](./gap-reference-catalog.md#gt-660) | **The Core proposes a menu nobody can read — `--select` names a catalogue no CLI or MCP surface publishes.** `GT-659` shipped selection on all three surfaces, each help text pointing at «the id the catalogue publishes». REST does publish one (`GET /api/v1/rulesets`); **CLI and MCP publish nothing**, so selecting was possible only for a caller who already knew a ruleset filename by heart — «configurable per tenant» true on paper, false in a terminal. **And REST's catalogue is a SECOND derivation**, building an `id` from `metadata.id` / `$id` / the path with its extension stripped while the selector matches on `sourceFile`: measured, **17 of 183 published ids match no rule**, so following the endpoint's own advice returns a blocking `SEL-01`. **FIXED 2026-08-09 with ONE derivation:** `buildRulesetCatalog` reads the same `loadAllRulesets` the engine evaluates, exposed once on `RulesetValidatorService.catalog` — no second construction path, so no second answer. `evolith rulesets` (CLI, ADR-0073 under `--format json`) and `evolith-ruleset-list` (MCP) both report **174 packs · 402 rules · 188 that can fail a run**, identical. Blocking is published PER PACK on purpose: a tenant is entitled to know which packs turn its build red before the first red build. REST keeps `id` and gains `ref` — **183 of 183 selectable against 166 of 183** — additive because `id` is published and clients hold it. **Two of my own numbers were corrected by the tool this row built:** the action README's «372-rule corpus» was a hand-count; the engine says **402**, and GT-659's «235 of 402» was right about the corpus and measured on a satellite. **Deliberately NOT done here:** the default. Naming nothing still evaluates everything — **85 blocking of 113 issues** on this repository — so a tenant that adopted nothing is still blocked. That is a decision, not a defect: [`GT-661`](./gap-reference-catalog.md#gt-661). | `Cross` | Cross | P1 | M | `DONE` | | [`GT-661`](./gap-reference-catalog.md#gt-661) | **«No selection» means the whole corpus, so a tenant that adopted nothing is still blocked by everything.** The owner's principle is that the Core PROPOSES and the client configures. Measured with no selection on this repository: **85 blocking issues of 113, status `failed`** — a caller that adopted nothing receives a verdict that blocks, from rules it never chose. SSDF is not the problem (its 8 rules are `blocking: false` and can only report); the other **188** blocking rules are. `GT-659` made a selection expressible and [`GT-660`](./gap-reference-catalog.md#gt-660) made the menu readable, which is what turns this from an unanswerable question into a decision. **The risk is stated because it is the reason this is not simply done:** every current consumer — this repository's own CI included — depends on `validate` blocking with no flag, and a default that stopped blocking would silently disarm gates that work today, which is a worse failure than the one being fixed. **Principal:** `M` · **Interest:** `MED` · **Basis:** `estimate`. **RESOLVED 2026-08-09.** **«No selection» keeps meaning the whole corpus, and the Core says so out loud.** The principle does not require the Core to stop blocking; it requires the Core not to be the one deciding what a tenant adopted. With no selection the Core has no tenant configuration to consult, refuses to invent one, and labels that scope `core-default` — a PROPOSAL, not a tenant's choice. **The alternative was measured and rejected:** a default that stopped blocking would silently disarm every gate working today, including this repository's own CI. **What changed is that the verdict can be READ:** `ValidationResult.selection` publishes `source` (`caller` or `core-default`), `requested`, `matched`, `unmatched`, `rulesSelected` and `corpusTotal`. Measured on the CLI — no selection gives `core-default`, 402 of 402, `failed`; `--select standards/ssdf-v1.1.rules.json` gives `caller`, 8 of 402, `warning`. Both rendered the same red before. `unmatched` is a FIELD rather than issue text, because zero rules with zero violations is indistinguishable from a clean repository. **The client's half was missing and is built:** `ProfileConfig.select` lets a tenant configure its adopted packs ONCE — a flag you must retype on every invocation is not configuration. `--select` still wins when given, including to WIDEN a stored default. Blanks are dropped on both paths, so an empty profile means «configured nothing», never «configured an empty scope». All four CLI surfaces carry it. **Nothing was disarmed, and that is asserted rather than claimed:** the report gained a field, the run did not lose a rule. | `Evolith Core` | Cross | P2 | M | `DONE` | | [`GT-662`](./gap-reference-catalog.md#gt-662) | **ISO/IEC 5055 as a measurement Evolith can actually make — CWE→measure, from the analysers a tenant already runs.** `GT-600` named the standard after SSDF; ISO/IEC 5055 is the only ISO **automated by construction** — 138 structural weaknesses, each a CWE, across four measures. The corpus already ships the index and a 404-row mapping, and neither is a CHECK: `strength` is `none` for **367 of 404** rules. A native handler cannot close that (filesystem, no parser), and 138 unexecutable rules is what `GT-585` measured as **78 of 96 blocking outcomes that could not run**. **SLICE 1 DELIVERED 2026-08-09:** the translation layer `analyser → SARIF → CWE → weakness → measure`, with no parser written — four enforcer adapters already exist and CodeQL already runs here. **Measured on this repository's live findings: of 75 open CodeQL alerts, 62 carry a CWE and 34 are weaknesses ISO/IEC 5055 names** — ten distinct (22, 23, 36, 78, 79, 88, 99, 434, 570, 571), scoring 34 Security and 2 Maintainability. A real ISO measurement, from a scanner already running, at no cost. **A constraint found by measuring:** GitHub's `analyses/{id}` SARIF serves `tool.driver` with only `name` and `semanticVersion` — no `rules[]`, results referencing a rule by bare index — so against THAT endpoint the mapping finds **0 of 75**. The CWEs live in the alerts API and in the SARIF CodeQL uploads. A design assuming otherwise would have shipped a permanent silent zero. **Still open:** the `iso-5055.rules.json` pack with `enforce:` descriptors and a `notEvaluableHere` block, selectable per tenant like any other. **REBANADA 2 ENTREGADA / SLICE 2 DELIVERED 2026-08-09.** The pack ships **four rules, one per measure**, routed through `createIso5055Adapter`. Verified end to end against the same live CodeQL findings: **75 alerts in, 36 violations out — 34 `ISO5055-SEC`, 2 `ISO5055-MAINT`** — reproducing slice 1 through the real adapter path. Published by `evolith rulesets` (**175 packs / 406 rules**, `blocking: 0`) and selectable: **4 of 406**. The scanner stays free: `semgrep --config p/default`, deliberately NOT `--config auto` (which needs network and a vendor login), and a tenant already emitting SARIF points `enforce.config.sarif` at it instead of scanning twice. **Two defects surfaced by this pack, both fixed.** (1) With the analyser absent, selecting it produced a BLOCKING `GOV-CORE-UNRESOLVED` telling an operator to pass `--core` when they had — the Core had resolved 406 rules and 4 were selected. GT-661's `corpusTotal` separates «the Core resolved nothing» from «the caller narrowed to rules that did not run»; a corpus of ZERO is still a hard failure, so GT-474 holds. (2) Fixing that revealed a run reporting **`passed` over 4 skipped rules** — the exact false assurance this backlog keeps finding, through a door I had just opened. The rules are now `MUST` + `blocking: false`, a load-bearing pair: MUST makes GT-569 emit the advisory that turns green into `warning`, blocking:false stops it failing a build over coverage never claimed. Measured after: `warning`, 4 skipped, 4 advisories. **Coverage is the analyser's, not the standard's, and `notEvaluableHere` says so:** a green means the configured analyser found no ISO/IEC 5055 weakness, NOT that the repository is free of them. | `Evolith Core` | Cross | P2 | L | `DONE` | +| [`GT-663`](./gap-reference-catalog.md#gt-663) | **The ISO/IEC 5055 measurement had no denominator — «found none» read the same as «looks for none».** `GT-662` made the measurement real (34 violations over 10 distinct weaknesses here) but could not say *of how many*. The standard names **138**, so a clean run reads identically whether the analyser covers all 138 or none — **and the second is the common case**, because coverage is the ANALYSER's, never the standard's. Same shape `GT-569` fixed one level down: a count without its denominator redefines what it counts. **DELIVERED 2026-08-09.** Every report now carries observed / 138, per measure, with the caveat written once and shared by all surfaces so a renderer cannot drop it. **Measured live, and the result is a product finding:** Security **10/74** (34 findings), Reliability **0/74**, Performance Efficiency **0/18**, Maintainability **2/31**. **Two of four measures are at ZERO and nothing said so before** — CodeQL does not look for Reliability or Performance Efficiency at all, which is the evidenced argument for a second analyser and was invisible without a denominator. Also surfaced: 28 findings carried a CWE outside the standard, 13 carried none, counted separately because «not in the standard» and «the analyser told us nothing mappable» are different facts. Emitted ONLY on a clean run and once — that is when the denominator matters and when the report used to be silent; repeating it per finding would train a reader to skip it. Non-blocking, so it informs a verdict and never decides one. **Refuses to compute what the analyser COULD have found:** that needs the scanner's own rule catalogue, and a «could have» derived from less would invent the assurance this row exists to withhold. `observed` is a floor and is labelled one. | `Evolith Core` | Cross | P2 | S | `DONE` | -**Progress:** 648 / 660 done · 2 in progress · 0 pending · 10 deferred +**Progress:** 649 / 661 done · 2 in progress · 0 pending · 10 deferred **Wave 2026-06-23 (Winston deep audit III):** Added 14 new gaps `GT-212`…`GT-225` from the Winston Audit Playbook covering: ADR status hygiene (GT-212), topology manifest metadata + operational budgets + guidance corpus (GT-213, GT-217, GT-219), REST controller observability + OpenAPI (GT-214, GT-215), OPA input-schema parity + per-topology test density (GT-216, GT-222), SDLC Phase 05 rollback + on-call templates (GT-218), CLI branch coverage + envelope format coverage + skip-list cleanup (GT-220, GT-224, GT-225), MCP HTTP audit logging (GT-221), and cross-surface parity e2e tests (GT-223). diff --git a/reference/core/control-center/maturity-reports/executive-summary.es.md b/reference/core/control-center/maturity-reports/executive-summary.es.md index 6f6fc527c..bcc3d0684 100644 --- a/reference/core/control-center/maturity-reports/executive-summary.es.md +++ b/reference/core/control-center/maturity-reports/executive-summary.es.md @@ -42,14 +42,14 @@ La forma correcta de usar este resumen es simple: si necesitas contexto, abre so | Indicador | Valor | |---|---:| | Fecha canónica del tablero | 2026-08-08 | -| Gaps totales | 660 | -| Gaps cerrados | 648 | +| Gaps totales | 661 | +| Gaps cerrados | 649 | | Gaps pendientes | 12 | | P0 abiertos | 1 | | P1 abiertos | 3 | | P2 abiertos | 5 | | Cierre total | 98.2% | -| Registros de evidencia de cierre | 630 | +| Registros de evidencia de cierre | 631 | | Readiness registrado | 4 PASS | | Área | Pendientes | P0 | P1 | Primeros IDs | diff --git a/reference/core/control-center/maturity-reports/executive-summary.md b/reference/core/control-center/maturity-reports/executive-summary.md index 4e98b86fe..33e4aad43 100644 --- a/reference/core/control-center/maturity-reports/executive-summary.md +++ b/reference/core/control-center/maturity-reports/executive-summary.md @@ -42,14 +42,14 @@ Use this summary with a simple rule: if you need context, open only the linked I | Indicator | Value | |---|---:| | Canonical board date | 2026-08-08 | -| Total gaps | 660 | -| Closed gaps | 648 | +| Total gaps | 661 | +| Closed gaps | 649 | | Open gaps | 12 | | Open P0 | 1 | | Open P1 | 3 | | Open P2 | 5 | | Total closure | 98.2% | -| Closure evidence records | 630 | +| Closure evidence records | 631 | | Recorded readiness | 4 PASS | | Area | Open | P0 | P1 | First IDs | diff --git a/reference/core/control-center/maturity-reports/maturity-reconciliation.json b/reference/core/control-center/maturity-reports/maturity-reconciliation.json index 2adf80175..42a5547f9 100644 --- a/reference/core/control-center/maturity-reports/maturity-reconciliation.json +++ b/reference/core/control-center/maturity-reports/maturity-reconciliation.json @@ -3,14 +3,14 @@ "scope": "evolith-core", "asOf": "2026-08-08", "gaps": { - "total": 660, - "done": 648, + "total": 661, + "done": 649, "pending": 0, "inProgress": 2, "deferred": 10 }, "evidence": { - "closureRecords": 630, + "closureRecords": 631, "cliPackage": "@beyondnet/evolith-cli@1.2.2", "adrCount": 140, "rulesetCount": 179, diff --git a/src/packages/core-domain/src/application/validators/enforcement/adapters/iso-5055-adapter.spec.ts b/src/packages/core-domain/src/application/validators/enforcement/adapters/iso-5055-adapter.spec.ts index 7d638f95c..821792309 100644 --- a/src/packages/core-domain/src/application/validators/enforcement/adapters/iso-5055-adapter.spec.ts +++ b/src/packages/core-domain/src/application/validators/enforcement/adapters/iso-5055-adapter.spec.ts @@ -101,11 +101,32 @@ describe('ISO/IEC 5055 pack + adapter · GT-662 slice 2', () => { runs: [{ tool: { driver: { name: 'semgrep' } }, results: [{ ruleId: 'style', properties: { cwe: ['CWE-352'] } }] }], }); expect(index.measuresFor(352)).toEqual([]); // checked, not assumed - expect(iso5055ViolationsFromSarif(log, index)).toEqual([]); + // GT-663: no MEASURE violation is produced. What comes back instead is the + // single coverage advisory below — because a run that found nothing is + // exactly when a reader needs the denominator. + const out = iso5055ViolationsFromSarif(log, index); + expect(out.filter((v) => v.severity !== 'warning')).toEqual([]); }); - it('a clean scan is an empty result, not an error', () => { - expect(iso5055ViolationsFromSarif(JSON.stringify({ runs: [{ tool: { driver: { name: 'semgrep' } }, results: [] }] }), index)).toEqual([]); + it('GT-663: a clean scan returns the DENOMINATOR, not silence', () => { + // It used to return `[]`, and `[]` reads as "nothing wrong" whether the + // analyser looks for all 138 weaknesses or for none of them. One advisory, + // once, naming the floor as a floor. + const out = iso5055ViolationsFromSarif( + JSON.stringify({ runs: [{ tool: { driver: { name: 'semgrep' } }, results: [] }] }), + index, + ); + expect(out).toHaveLength(1); + expect(out[0].severity).toBe('warning'); + expect(out[0].message).toContain('0 of the 138'); + expect(out[0].message).toContain('FLOOR'); + }); + + it('GT-663: when there ARE findings the caveat is NOT repeated per finding', () => { + // Repeating it would train a reader to skip it, and they already have + // something concrete to act on. + const out = iso5055ViolationsFromSarif(codeqlLog, index); + expect(out.filter((v) => v.severity === 'warning')).toHaveLength(0); }); it('AN INDEX THAT DID NOT LOAD THROWS — it never reports zero weaknesses', () => { diff --git a/src/packages/core-domain/src/application/validators/enforcement/adapters/iso-5055-adapter.ts b/src/packages/core-domain/src/application/validators/enforcement/adapters/iso-5055-adapter.ts index 7d6190242..68f1e7f30 100644 --- a/src/packages/core-domain/src/application/validators/enforcement/adapters/iso-5055-adapter.ts +++ b/src/packages/core-domain/src/application/validators/enforcement/adapters/iso-5055-adapter.ts @@ -2,6 +2,7 @@ import { IProcessRunner, ProcessResult, EnforcerAnalysisContext } from '../enfor import { ShellEnforcerAdapter, ShellEnforcerConfig } from '../shell-enforcer-adapter'; import { makeViolation } from '../../../../domain/violation'; import type { Violation } from '../../../../domain/violation'; +import { describeIso5055Coverage, iso5055CoverageFromSarif } from '../../standards/iso-5055-coverage'; import { buildIso5055Index, classifySarifResult, @@ -141,6 +142,29 @@ export function iso5055ViolationsFromSarif(log: string, index: Iso5055Index): Vi } } } + // GT-663 — a run that found nothing is exactly when the denominator matters, + // and exactly when the report used to be silent. Zero ISO/IEC 5055 findings + // reads identically whether the analyser looks for all 138 weaknesses or for + // none of them, and the second is the common case because coverage here is + // the ANALYSER's, never the standard's. + // + // Emitted only on a clean run, and only once: when there ARE findings the + // reader already has something concrete to act on, and repeating the caveat + // per finding would train them to skip it. Non-blocking by construction — the + // pack's rules are `blocking: false` — so this informs a verdict, never + // decides one. + if (out.length === 0) { + out.push( + makeViolation({ + ruleId: MEASURE_RULE_IDS.Security, + tool: ISO_5055_TOOL, + file: '', + severity: 'warning', + message: describeIso5055Coverage(iso5055CoverageFromSarif(log, index)), + }), + ); + } + return out; } diff --git a/src/packages/core-domain/src/application/validators/standards/iso-5055-coverage.spec.ts b/src/packages/core-domain/src/application/validators/standards/iso-5055-coverage.spec.ts new file mode 100644 index 000000000..80bc95f5a --- /dev/null +++ b/src/packages/core-domain/src/application/validators/standards/iso-5055-coverage.spec.ts @@ -0,0 +1,98 @@ +import { describeIso5055Coverage, iso5055CoverageFromSarif } from './iso-5055-coverage'; +import { buildIso5055Index } from './iso-5055-measure'; +import { ISO_5055_WEAKNESS_INDEX } from './iso-5055-index.generated'; + +/** + * GT-663 — a count without its denominator silently redefines what it counts. + * + * GT-662 made the measurement real (34 violations over 10 distinct weaknesses on + * this repository) but the report could not say *of how many*. The standard + * names 138, so "found none" read identically whether the analyser looks for all + * 138 or for none of them — and the second is the common case, because coverage + * here is the ANALYSER's, never the standard's. + * + * This is GT-569's fix one standard over, and the tests below are about the same + * thing: what the number is allowed to claim. + */ +const index = buildIso5055Index(ISO_5055_WEAKNESS_INDEX); + +const sarif = (results: unknown[], rules: unknown[] = []) => + JSON.stringify({ runs: [{ tool: { driver: { name: 'CodeQL', rules } }, results }] }); + +describe('ISO/IEC 5055 coverage · GT-663', () => { + it('THE DENOMINATOR: the standard names 138, and the report says so', () => { + const c = iso5055CoverageFromSarif(sarif([]), index); + expect(c.standardWeaknesses).toBe(138); + expect(c.observedWeaknesses).toBe(0); + }); + + it('counts DISTINCT weaknesses, not findings — ten hits on one CWE is still one weakness', () => { + const results = Array.from({ length: 10 }, () => ({ ruleId: 'r', properties: { cwe: ['CWE-89'] } })); + const c = iso5055CoverageFromSarif(sarif(results), index); + expect(c.observedWeaknesses).toBe(1); + expect(c.observedCwes).toEqual([89]); + // Findings are counted too, separately: they are what a reader acts on. + expect(c.byMeasure.find((m) => m.measure === 'Security')!.findings).toBe(10); + expect(c.byMeasure.find((m) => m.measure === 'Security')!.observed).toBe(1); + }); + + it('per measure, the denominator is that measure\'s own size', () => { + const c = iso5055CoverageFromSarif(sarif([{ ruleId: 'r', properties: { cwe: ['CWE-89'] } }]), index); + const security = c.byMeasure.find((m) => m.measure === 'Security')!; + // 74 is the standard's own Security count, read off the shipped index. + expect(security.total).toBe(74); + expect(security.observed).toBe(1); + // Every measure appears, including the ones with nothing found — a measure + // that vanishes from a report is a measure nobody notices was never checked. + expect(c.byMeasure.map((m) => m.measure)).toEqual([ + 'Security', 'Reliability', 'Performance Efficiency', 'Maintainability', + ]); + }); + + it('separates "outside the standard" from "the analyser tagged nothing"', () => { + // Both are zero-contribution to the measurement and mean different things. + // An untagged finding is not evidence of compliance — it is evidence the + // analyser said nothing that could be mapped. + const c = iso5055CoverageFromSarif( + sarif([ + { ruleId: 'csrf', properties: { cwe: ['CWE-352'] } }, // real CWE, not in 5055 + { ruleId: 'style' }, // no CWE at all + ]), + index, + ); + expect(index.measuresFor(352)).toEqual([]); + expect(c.outOfScopeFindings).toBe(1); + expect(c.untaggedFindings).toBe(1); + expect(c.observedWeaknesses).toBe(0); + }); + + it('reads CWEs off the RULE too, which is where CodeQL puts them', () => { + const c = iso5055CoverageFromSarif( + sarif([{ ruleId: 'js/sqli' }], [{ id: 'js/sqli', properties: { tags: ['external/cwe/cwe-089'] } }]), + index, + ); + expect(c.observedCwes).toEqual([89]); + }); + + it('AN UNLOADED INDEX THROWS — "0 of 138" from nothing is a compliance claim built on nothing', () => { + expect(() => iso5055CoverageFromSarif(sarif([]), buildIso5055Index(undefined))).toThrow(/did not load/i); + }); + + // --- the sentence ------------------------------------------------------- + + it('THE CAVEAT TRAVELS WITH THE NUMBER: the text names the floor as a floor', () => { + const c = iso5055CoverageFromSarif(sarif([{ ruleId: 'r', properties: { cwe: ['CWE-89'] } }]), index); + const text = describeIso5055Coverage(c); + expect(text).toContain('1 of the 138'); + expect(text).toContain('FLOOR'); + // The claim it must never make: that the rest are absent. + expect(text).toMatch(/says nothing about how many/); + expect(text).not.toMatch(/compliant|passes the standard/i); + }); + + it('the sentence survives a run that found nothing, which is when it matters most', () => { + const text = describeIso5055Coverage(iso5055CoverageFromSarif(sarif([]), index)); + expect(text).toContain('0 of the 138'); + expect(text).toContain('FLOOR'); + }); +}); diff --git a/src/packages/core-domain/src/application/validators/standards/iso-5055-coverage.ts b/src/packages/core-domain/src/application/validators/standards/iso-5055-coverage.ts new file mode 100644 index 000000000..82ab018a9 --- /dev/null +++ b/src/packages/core-domain/src/application/validators/standards/iso-5055-coverage.ts @@ -0,0 +1,167 @@ +import { classifySarifResult, type Iso5055Index, type Iso5055Measure } from './iso-5055-measure'; + +/** + * GT-663 — the ISO/IEC 5055 measurement reports its own denominator. + * + * GT-662 made the measurement real: this repository's CodeQL findings map to + * **34 violations across 10 distinct weaknesses**. What the report could not say + * is *of how many*. The standard names **138**, so a run that finds none reads + * identically whether the analyser looks for all 138 or for none of them — and + * the second is the common case, because coverage here is the analyser's, never + * the standard's. + * + * That is the exact shape GT-569 fixed for rule coverage: a count without its + * denominator silently redefines what it counts. `rulesChecked: 0` looked like a + * clean repository until the denominator travelled with it. This is the same + * fix, one standard over. + * + * ## What it can and cannot know + * + * It reports what the analyser **DID** find, against what the standard names. It + * does NOT claim to know what the analyser *could* have found — that would need + * the scanner's own rule catalogue, which is a network call to a vendor for + * CodeQL and a registry fetch for semgrep. Publishing a "could have" number + * derived from anything less would be inventing the very assurance this file + * exists to withhold. + * + * So `observed` is a floor, and it is labelled as one. A tenant reading it + * learns «my analyser surfaced 10 of the 138 weaknesses this standard names», + * which is a true and useful sentence, and never «the other 128 are absent». + */ + +/** Per-measure counts, all of them floors observed from one run. */ +export interface Iso5055MeasureCoverage { + readonly measure: Iso5055Measure; + /** Distinct weaknesses of this measure the analyser reported. */ + readonly observed: number; + /** Weaknesses this measure contains, per the standard. */ + readonly total: number; + /** Individual findings attributed to this measure (one weakness can recur). */ + readonly findings: number; +} + +export interface Iso5055Coverage { + /** Distinct ISO/IEC 5055 weaknesses the analyser reported, across all measures. */ + readonly observedWeaknesses: number; + /** Weaknesses the standard names. The denominator that was missing. */ + readonly standardWeaknesses: number; + /** The observed CWE numbers, ascending — enumerable, not just counted. */ + readonly observedCwes: readonly number[]; + /** Findings that carried a CWE the standard does NOT name. */ + readonly outOfScopeFindings: number; + /** Findings that carried no CWE at all — the analyser said nothing to map. */ + readonly untaggedFindings: number; + readonly byMeasure: readonly Iso5055MeasureCoverage[]; +} + +const MEASURE_ORDER: readonly Iso5055Measure[] = [ + 'Security', + 'Reliability', + 'Performance Efficiency', + 'Maintainability', +]; + +/** + * Measure one SARIF log against the standard's denominator. + * + * Pure. Throws on an unloaded index for the same reason + * `iso5055ViolationsFromSarif` does: reporting «0 of 138» from an index that is + * not there is a compliance claim built on nothing. + */ +export function iso5055CoverageFromSarif(log: string, index: Iso5055Index): Iso5055Coverage { + if (index.size === 0) { + throw new Error( + 'ISO/IEC 5055 weakness index did not load (size 0). Refusing to report coverage ' + + 'against an index that is not there.', + ); + } + + const totals = new Map(MEASURE_ORDER.map((m) => [m, 0])); + for (const weakness of index.weaknesses()) { + for (const measure of weakness.measures) totals.set(measure, (totals.get(measure) ?? 0) + 1); + } + + const observed = new Map>(MEASURE_ORDER.map((m) => [m, new Set()])); + const findings = new Map(MEASURE_ORDER.map((m) => [m, 0])); + const allObserved = new Set(); + let outOfScope = 0; + let untagged = 0; + + let parsed: { runs?: unknown[] }; + try { + parsed = JSON.parse(log || '{}'); + } catch { + parsed = {}; + } + + for (const run of (parsed.runs ?? []) as Array>) { + const driver = (run.tool as { driver?: { rules?: unknown[] } } | undefined)?.driver; + const rules = new Map(); + for (const meta of (driver?.rules ?? []) as Array<{ id?: string }>) { + if (meta?.id) rules.set(meta.id, meta); + } + + for (const result of (run.results ?? []) as Array>) { + const finding = classifySarifResult(index, result, rules.get(String(result.ruleId ?? ''))); + + if (finding.cwes.length === 0) { + // Counted separately on purpose. A finding the analyser never tagged is + // not evidence the standard is satisfied — it is evidence the analyser + // told us nothing we could map, and a reader deciding whether to trust + // this measurement needs to see how much of that there was. + untagged += 1; + continue; + } + if (finding.iso5055Cwes.length === 0) { + outOfScope += 1; + continue; + } + + for (const cwe of finding.iso5055Cwes) allObserved.add(cwe); + for (const measure of finding.measures) { + findings.set(measure, (findings.get(measure) ?? 0) + 1); + for (const cwe of finding.iso5055Cwes) { + if (index.measuresFor(cwe).includes(measure)) observed.get(measure)!.add(cwe); + } + } + } + } + + return { + observedWeaknesses: allObserved.size, + standardWeaknesses: index.size, + observedCwes: [...allObserved].sort((a, b) => a - b), + outOfScopeFindings: outOfScope, + untaggedFindings: untagged, + byMeasure: MEASURE_ORDER.map((measure) => ({ + measure, + observed: observed.get(measure)!.size, + total: totals.get(measure) ?? 0, + findings: findings.get(measure) ?? 0, + })), + }; +} + +/** + * The sentence a report carries next to the verdict. + * + * Written here rather than at each call site so all three surfaces say the same + * thing, and so the caveat cannot be dropped by whoever renders it: the floor is + * named as a floor, every time. + */ +export function describeIso5055Coverage(coverage: Iso5055Coverage): string { + const per = coverage.byMeasure + .filter((m) => m.observed > 0) + .map((m) => `${m.measure} ${m.observed}/${m.total}`) + .join(', '); + + return ( + `ISO/IEC 5055: the configured analyser reported ${coverage.observedWeaknesses} of the ` + + `${coverage.standardWeaknesses} weaknesses this standard names` + + (per ? ` (${per})` : '') + + `. That is a FLOOR, not coverage: it counts what was FOUND, and says nothing about how many ` + + `of the remaining ${coverage.standardWeaknesses - coverage.observedWeaknesses} the analyser ` + + `even looks for. ${coverage.outOfScopeFindings} finding(s) carried a CWE outside the standard ` + + `and ${coverage.untaggedFindings} carried none at all.` + ); +}