ci: fail the benchmark job on a Devup UI regression and keep a WASM size budget

.github/workflows/publish.yml

@@ -64,8 +64,20 @@ jobs:
           node-version: latest
       - run: bun install
       - run: bun run build
+      # Every row is the median of the same number of cold samples taken in
+      # rotating order (benchmark-manifest.json). The run goes to an artifact and
+      # the Devup UI rows are compared with the checked baseline at the end of the
+      # job, so the profile and the Rust measurement still run when it fails.
       - name: Benchmark
         run: bun run benchmark
+      - name: Profile Devup UI Turbopack
+        env:
+          DEVUP_UI_PROFILE: "1"
+        run: |
+          rm -rf benchmark/next-devup-ui-single-turbo/.next \
+            benchmark/next-devup-ui-single-turbo/df \
+            benchmark/next-devup-ui-single-turbo/tsconfig.tsbuildinfo
+          bun run --filter next-devup-ui-single-turbo-benchmark build
       - name: Upload the measured run
         uses: actions/upload-artifact@v7
         if: ${{ !cancelled() }}
@@ -74,14 +86,142 @@ jobs:
           path: benchmark-run.json
           if-no-files-found: ignore
           retention-days: 30
-      - name: Profile Devup UI Turbopack
+      - name: Measure the SemanticBuilder passes of the extractor
+        run: cargo bench -p extractor --bench semantic_pass_benchmark -- --quick
+      - name: Compare the Devup UI rows with the baseline
+        run: bun benchmark-check.js
+
+  wasm-size:
+    runs-on: ubuntu-latest
+    timeout-minutes: 60
+    permissions:
+      contents: read
+    env:
+      CARGO_BUILD_JOBS: "2"
+      RUSTUP_TOOLCHAIN: stable
+    steps:
+      - name: Checkout the tested source and both histories
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+          persist-credentials: false
+      - uses: actions-rust-lang/setup-rust-toolchain@v2
+        with:
+          toolchain: stable
+          target: wasm32-unknown-unknown
+          cache: false
+      - uses: oven-sh/setup-bun@v2
+        with:
+          bun-version: latest
+      - uses: actions/setup-node@v7
+        with:
+          node-version: latest
+      - name: Resolve and report exact WASM comparison sources
+        run: node resolve-wasm-size-sources.js
+      - uses: jetli/wasm-pack-action@v0.4.0
+        with:
+          version: latest
+      - name: Cache registry downloads only
+        uses: actions/cache@v6
+        with:
+          path: |
+            ~/.cargo/registry/index/
+            ~/.cargo/registry/cache/
+            ~/.cargo/git/db/
+          key: cargo-wasm-size-${{ runner.os }}-${{ hashFiles('**/Cargo.lock') }}
+      - name: Install the latest shared installer and optimizer once
         env:
-          DEVUP_UI_PROFILE: "1"
+          GH_TOKEN: ${{ github.token }}
         run: |
-          rm -rf benchmark/next-devup-ui-single-turbo/.next \
-            benchmark/next-devup-ui-single-turbo/df \
-            benchmark/next-devup-ui-single-turbo/tsconfig.tsbuildinfo
-          bun run --filter next-devup-ui-single-turbo-benchmark build
+          set -euo pipefail
+          curl -L --proto '=https' --tlsv1.2 -sSf https://raw.githubusercontent.com/cargo-bins/cargo-binstall/main/install-from-binstall-release.sh | bash
+          mkdir -p "$RUNNER_TEMP/wasm-size-tools/binaryen"
+          gh release download --repo WebAssembly/binaryen --pattern '*-x86_64-linux.tar.gz' --dir "$RUNNER_TEMP/wasm-size-tools"
+          tar -xzf "$RUNNER_TEMP"/wasm-size-tools/*.tar.gz --strip-components=1 -C "$RUNNER_TEMP/wasm-size-tools/binaryen"
+          echo "$RUNNER_TEMP/wasm-size-tools/binaryen/bin" >> "$GITHUB_PATH"
+      - name: Build exact sources with shared tools and lock-matched bindgen
+        run: |
+          set -euo pipefail
+          export SIZE_ROOT="$RUNNER_TEMP/wasm-size"
+          mkdir -p "$SIZE_ROOT"
+          stage=source
+          trap 'echo "## WASM size: FAIL - $stage build or provenance failed; no comparison was skipped or substituted." >> "$GITHUB_STEP_SUMMARY"' ERR
+          for side in base candidate; do
+            if [[ "$side" == base ]]; then sha="$BASE_SHA"; else sha="$CANDIDATE_SHA"; fi
+            git cat-file -e "$sha^{commit}"
+            mkdir -p "$SIZE_ROOT/$side"
+            git archive "$sha" | tar -x -C "$SIZE_ROOT/$side"
+          done
+          cat > "$SIZE_ROOT/provenance.mjs" <<'NODE'
+          import { execFileSync } from 'node:child_process'
+          import { createHash } from 'node:crypto'
+          import { readFileSync, writeFileSync } from 'node:fs'
+          import { join } from 'node:path'
+          const run = (command, args) => execFileSync(command, args, { encoding: 'utf8' }).trim()
+          const tools = Object.fromEntries([
+            ['rustc', 'rustc', ['--version', '--verbose']],
+            ['wasmPack', 'wasm-pack', ['--version']],
+            ['wasmBindgen', 'wasm-bindgen', ['--version']],
+            ['wasmOpt', 'wasm-opt', ['--version']],
+            ['bun', 'bun', ['--version']],
+            ['node', 'node', ['--version']],
+          ].map(([key, command, args]) => {
+            const path = run('which', [command])
+            const hash = createHash('sha256').update(readFileSync(path)).digest('hex')
+            return [key, `${run(command, args)}\n${path}\nsha256:${hash}`]
+          }))
+          const side = process.argv[2]
+          const file = join(process.env.SIZE_ROOT, 'metadata.json')
+          const metadata = side === 'base' ? {
+            jobId: `${process.env.GITHUB_RUN_ID}/${process.env.GITHUB_RUN_ATTEMPT}/${process.env.GITHUB_JOB}`,
+            buildCommand: 'cargo metadata --locked --format-version 1 > /dev/null && wasm-pack build --mode no-install --release --target nodejs --out-dir ./pkg --out-name index --locked && node script.js',
+          } : JSON.parse(readFileSync(file, 'utf8'))
+          const sha = side === 'base' ? process.env.BASE_SHA : process.env.CANDIDATE_SHA
+          metadata[side] = {
+            commit: run('git', ['rev-parse', `${sha}^{commit}`]),
+            tree: run('git', ['rev-parse', `${sha}^{tree}`]),
+            wasmBindgenVersion: process.env.WASM_BINDGEN_VERSION,
+            packageDir: join(process.env.SIZE_ROOT, side, 'bindings/devup-ui-wasm/pkg'), tools,
+          }
+          writeFileSync(file, JSON.stringify(metadata, null, 2) + '\n')
+          NODE
+          shared_path="$PATH"
+          for side in base candidate; do
+            stage="$side"
+            (
+              WASM_BINDGEN_VERSION="$(bun resolve-wasm-bindgen.js "$SIZE_ROOT/$side/Cargo.lock")"
+              export WASM_BINDGEN_VERSION
+              bindgen_root="$SIZE_ROOT/abi-tools/$WASM_BINDGEN_VERSION"
+              if [[ ! -x "$bindgen_root/bin/wasm-bindgen" ]]; then
+                cargo binstall wasm-bindgen-cli --version "=$WASM_BINDGEN_VERSION" --root "$bindgen_root" --no-confirm --force
+              fi
+              export PATH="$bindgen_root/bin:$shared_path"
+              node "$SIZE_ROOT/provenance.mjs" "$side"
+              cd "$SIZE_ROOT/$side/bindings/devup-ui-wasm"
+              export CARGO_TARGET_DIR="$SIZE_ROOT/$side-target"
+              cargo metadata --locked --format-version 1 > /dev/null
+              wasm-pack build --mode no-install --release --target nodejs --out-dir ./pkg --out-name index --locked
+              node script.js
+            ) 2>&1 | tee "$SIZE_ROOT/$side-build.log"
+          done
+      - name: Compare all four relative size metrics
+        run: |
+          node verify-wasm-size.js \
+            --base-pkg "$RUNNER_TEMP/wasm-size/base/bindings/devup-ui-wasm/pkg" \
+            --candidate-pkg "$RUNNER_TEMP/wasm-size/candidate/bindings/devup-ui-wasm/pkg" \
+            --metadata "$RUNNER_TEMP/wasm-size/metadata.json" \
+            --report "$RUNNER_TEMP/wasm-size/wasm-size.json"
+      - name: Upload size measurements and build evidence
+        uses: actions/upload-artifact@v7
+        if: ${{ !cancelled() }}
+        with:
+          name: wasm-size
+          path: |
+            ${{ runner.temp }}/wasm-size/metadata.json
+            ${{ runner.temp }}/wasm-size/wasm-size.json
+            ${{ runner.temp }}/wasm-size/*-build.log
+          if-no-files-found: error
+          retention-days: 30
 
   # The landing app is the only real-scale acceptance target in this repo, and
   # `publish` runs it against the deployed vinext/@devup-ui/vite-plugin build.

.gitignore

@@ -25,3 +25,4 @@ playwright-report
 .omo
 /.changepacks/publish.tgz
 benchmark-run.json
+wasm-size.json

README.md

@@ -80,33 +80,33 @@ npm install @devup-ui/webpack-plugin
 ## Comparison Benchmarks
 
 <!-- benchmark:start -->
-[Latest CI benchmark](https://github.com/dev-five-git/devup-ui/actions/runs/36702867475) (commit `8d6c2f62`, Devup UI 1.0.44) on `ubuntu-24.04` with Next.js 16.3.3. All Next.js builds use the native TypeScript 7 CLI for type checking. The English and Korean tables are generated from the same checked result file, `benchmark-results.json`.
+[Latest CI benchmark](https://github.com/dev-five-git/devup-ui/actions/runs/37758611477) (commit `467a379a`, Devup UI 1.0.44) on `ubuntu-24.04` with Next.js 16.3.3. Every row is the median of 3 cold builds that ran in rotating order, shown with the range of the samples. All Next.js builds use the native TypeScript 7 CLI for type checking. The English and Korean tables are generated from the same checked result file, `benchmark-results.json`.
 
-Build Size is every byte of the build output directory (`.next`, or `dist` for vinext); CSS Size is the emitted `.css` files alone, measured separately.
+Build Size is every byte of the build output directory (`.next`, or `dist` for vinext), including caches and server output. The checked table retains its original historical CSS measurements, which recursively counted `.css` files throughout that output. New runs count only browser-shipped CSS: `.next/static/**/*.css` (Webpack `static/css` and Turbopack `static/chunks`) or vinext `dist/client/**/*.css`, excluding caches and server CSS. A missing browser-assets directory is an error; an existing directory with no CSS counts zero. Approved CSS measurement corrections are limited to explicitly named `devup-ui`, `devup-ui-single` and `vinext-devup-ui` rows. `bun benchmark-check.js --write-css-baseline devup-ui` retains the per-file mode; `bun benchmark-check.js --write-css-baseline devup-ui-single,vinext-devup-ui` selects only the other two rows in one comma-separated argument. Only selected existing rows' CSS bytes change; unlike `--write-baseline`, this preserves sample count, build bytes, ratios, tolerances, all unselected rows and calibration history. Missing or unsupported names and conflicting writer modes are rejected.
 
-Webpack values are one cold build:
+Webpack:
 
-| Library                     | Version | Build Time | Build Size        | CSS Size     |
-| --------------------------- | ------- | ---------- | ----------------- | ------------ |
-| tailwindcss                 | 4.3.3   | 16.67s     | 67,332,642 bytes  | 5,852 bytes  |
-| styleX                      | 0.19.0  | 38.08s     | 96,291,287 bytes  | 425 bytes    |
-| vanilla-extract             | 1.21.2  | 16.09s     | 68,565,036 bytes  | 294 bytes    |
-| kuma-ui                     | 1.6.4   | 17.91s     | 75,627,783 bytes  | 340 bytes    |
-| panda-css                   | 1.12.0  | 18.16s     | 71,849,418 bytes  | 15,615 bytes |
-| chakra-ui                   | 3.37.0  | 27.22s     | 207,426,274 bytes | 0 bytes      |
-| mui                         | 9.4.0   | 18.78s     | 101,473,260 bytes | 0 bytes      |
-| **devup-ui (per-file CSS)** | 1.0.44  | **14.27s** | 67,424,897 bytes  | 562 bytes    |
-| **devup-ui (single CSS)**   | 1.0.44  | **14.17s** | 67,426,959 bytes  | 790 bytes    |
+| Library                     | Version | Median Build Time | Range           | Build Size        | CSS Size     |
+| --------------------------- | ------- | ----------------- | --------------- | ----------------- | ------------ |
+| tailwindcss                 | 4.3.3   | 13.02s            | 12.91s - 13.20s | 67,339,250 bytes  | 5,852 bytes  |
+| styleX                      | 0.19.0  | 29.01s            | 28.72s - 29.50s | 96,290,533 bytes  | 425 bytes    |
+| vanilla-extract             | 1.21.2  | 12.34s            | 12.22s - 12.35s | 68,565,511 bytes  | 294 bytes    |
+| kuma-ui                     | 1.6.4   | 13.55s            | 13.49s - 13.57s | 75,627,429 bytes  | 340 bytes    |
+| panda-css                   | 1.12.0  | 13.44s            | 13.29s - 13.58s | 71,849,243 bytes  | 15,615 bytes |
+| chakra-ui                   | 3.37.0  | 20.29s            | 19.95s - 21.06s | 207,425,028 bytes | 0 bytes      |
+| mui                         | 9.4.0   | 14.12s            | 13.78s - 14.26s | 101,476,098 bytes | 0 bytes      |
+| **devup-ui (per-file CSS)** | 1.0.44  | **10.95s**        | 10.87s - 11.09s | 67,427,695 bytes  | 562 bytes    |
+| **devup-ui (single CSS)**   | 1.0.44  | **11.04s**        | 10.88s - 11.40s | 67,436,336 bytes  | 790 bytes    |
 
-Turbopack values are medians of six cold builds in alternating order:
+Turbopack:
 
-| Library                                | Version | Median Build Time | Build Size       | CSS Size    |
-| -------------------------------------- | ------- | ----------------- | ---------------- | ----------- |
-| tailwindcss                            | 4.3.3   | 6.98s             | 38,422,141 bytes | 6,197 bytes |
-| **devup-ui (direct APIs, single CSS)** | 1.0.44  | **6.93s**         | 36,518,952 bytes | 327 bytes   |
-| **devup-ui (static `.css.ts`)**        | 1.0.44  | **6.92s**         | 36,578,506 bytes | 268 bytes   |
+| Library                                | Version | Median Build Time | Range         | Build Size       | CSS Size    |
+| -------------------------------------- | ------- | ----------------- | ------------- | ---------------- | ----------- |
+| tailwindcss                            | 4.3.3   | 5.68s             | 5.51s - 5.73s | 38,440,013 bytes | 6,197 bytes |
+| **devup-ui (direct APIs, single CSS)** | 1.0.44  | **5.79s**         | 5.77s - 5.87s | 36,538,765 bytes | 327 bytes   |
+| **devup-ui (static `.css.ts`)**        | 1.0.44  | **5.71s**         | 5.67s - 5.99s | 36,580,207 bytes | 268 bytes   |
 
-The Turbopack ranges overlap, so the direct-API median is 0.05s (0.7%) ahead of Tailwind on this fixture and the two are effectively at parity. The six cold samples were Tailwind `7.10s, 6.99s, 6.98s, 6.90s, 6.91s, 6.97s`, direct Devup UI `6.98s, 6.83s, 6.95s, 6.91s, 6.90s, 6.94s`, and static `.css.ts` `6.93s, 6.95s, 6.85s, 6.97s, 6.87s, 6.91s`. The fixtures have comparable app shapes, not pixel-identical styling: Tailwind styles the leading paragraph and button more heavily, while Devup UI exercises typed component/style props. Treat these as build-pipeline results rather than a per-rule microbenchmark. Every `.css.ts` module runs on the full Boa evaluator, which adds about 20 ms for the first module and under 1 ms for each further one.
+The Turbopack ranges do not overlap, so the direct-API median is 0.12s (2.1%) behind Tailwind on this fixture. The 3 cold samples were Tailwind `5.68s, 5.73s, 5.51s`, direct Devup UI `5.77s, 5.79s, 5.87s`, and static `.css.ts` `5.71s, 5.99s, 5.67s`. The fixtures have comparable app shapes, not pixel-identical styling: Tailwind styles the leading paragraph and button more heavily, while Devup UI exercises typed component/style props. Treat these as build-pipeline results rather than a per-rule microbenchmark. Every `.css.ts` module runs on the full Boa evaluator, which adds about 20 ms for the first module and under 1 ms for each further one. CI compares the Devup UI rows with a checked baseline (see `benchmark-gate.js` for the rule) and fails on a regression beyond the measured noise.
 <!-- benchmark:end -->
 
 ## How it works