diff --git a/apps/docs/.gitignore b/apps/docs/.gitignore new file mode 100644 index 00000000000..7c4dc796934 --- /dev/null +++ b/apps/docs/.gitignore @@ -0,0 +1,3 @@ +# Written by `node scripts/build-llms-full.mjs` (the `build` script): a build +# artifact of every docs page, ~20 MB, never committed. +/public/llms-full.txt diff --git a/apps/docs/app/llms-full.txt/route.ts b/apps/docs/app/llms-full.txt/route.ts deleted file mode 100644 index d494d2cbb68..00000000000 --- a/apps/docs/app/llms-full.txt/route.ts +++ /dev/null @@ -1,10 +0,0 @@ -import { getLLMText, source } from '@/lib/source'; - -export const revalidate = false; - -export async function GET() { - const scan = source.getPages().map(getLLMText); - const scanned = await Promise.all(scan); - - return new Response(scanned.join('\n\n')); -} diff --git a/apps/docs/next.config.mjs b/apps/docs/next.config.mjs index 05b9429a7d5..216edb49777 100644 --- a/apps/docs/next.config.mjs +++ b/apps/docs/next.config.mjs @@ -104,7 +104,10 @@ const config = { // // 1. Coverage. Only the per-page markdown has an HTML twin to point a // canonical at. `/llms.txt` is an index of every page and - // `/llms-full.txt` is all 400+ of them concatenated (8 MB today); there + // `/llms-full.txt` is all 400+ of them concatenated (20 MB in 2026-10, a + // static file `scripts/build-llms-full.mjs` writes into `public/` -- + // the header applies there too, since Next checks headers before the + // filesystem); there // is no single HTML URL either one is a duplicate *of*, so a canonical // header cannot say anything honest about them and would leave the two // largest parallel copies on the site undirected. `noindex` states the diff --git a/apps/docs/package.json b/apps/docs/package.json index 8def420d6b7..9258f371dbc 100644 --- a/apps/docs/package.json +++ b/apps/docs/package.json @@ -6,7 +6,7 @@ "license": "Apache-2.0", "scripts": { "dev": "next dev", - "build": "pnpm --filter @objectstack/spec gen:schema && pnpm --filter @objectstack/spec gen:docs && pnpm --filter @objectstack/spec gen:upgrade-guide && NODE_OPTIONS='--max-old-space-size=4096' next build", + "build": "pnpm --filter @objectstack/spec gen:schema && pnpm --filter @objectstack/spec gen:docs && pnpm --filter @objectstack/spec gen:upgrade-guide && node scripts/build-llms-full.mjs && NODE_OPTIONS='--max-old-space-size=4096' next build", "start": "next start", "site:lint": "next lint", "postinstall": "fumadocs-mdx", diff --git a/apps/docs/scripts/build-llms-full.mjs b/apps/docs/scripts/build-llms-full.mjs new file mode 100644 index 00000000000..82e7e0b0ff2 --- /dev/null +++ b/apps/docs/scripts/build-llms-full.mjs @@ -0,0 +1,101 @@ +#!/usr/bin/env node +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Writes `public/llms-full.txt` -- every docs page's `getLLMText()`, joined by + * one blank line, in `source.getPages()` order -- at BUILD time, as a static + * file. Run by this app's `build` script, after the spec generators and before + * `next build`: + * + * node scripts/build-llms-full.mjs # writes public/llms-full.txt + * node scripts/build-llms-full.mjs # writes elsewhere (measuring) + * + * ## Why a build-time file and not a route handler + * + * `/llms-full.txt` used to be `app/llms-full.txt/route.ts` with + * `revalidate = false`. `next build` prerenders such a route, and Vercel stores + * a prerendered route as an ISR page whose body is its fallback; the deploy API + * refuses a fallback body over 20 MB (`FALLBACK_BODY_TOO_LARGE`, at the + * `vercel deploy --prebuilt` upload, after a green build). The concatenation + * measured 20,032,800 bytes over 418 pages at e611e2bda, so every production + * docs deploy failed. A file under `public/` is not a prerender: Vercel's Next + * builder copies it into the build output's `static/` tree and the CDN serves + * it as a file, so that cap does not apply to it at any size. The URL, the body + * and the `X-Robots-Tag: noindex` header (`next.config.mjs` `headers()`, which + * Next applies before the filesystem, `public/` included) are unchanged. + * + * ## One producer + * + * The text is `getLLMText()` over `source.getPages()` from `lib/source.ts` -- + * the module the per-page `/llms.mdx/docs/**` route and `/llms.txt` read -- + * imported here as plain Node: fumadocs-mdx's Node loader compiles each page + * with the same `source.config.ts`, and `llms-full/resolve-hooks.mjs` maps the + * two tsconfig aliases. When this script replaced the route, its output was + * byte-identical (sha256) to the route's prerendered body read off the same + * `.source/`. A second copy of the page list or of `getLLMText` here would fork + * the two. + * + * Page ORDER is `source.getPages()` order, which follows the `.source/` index, + * and fumadocs-mdx writes that index in a different order on every run + * (measured: four runs on one tree, four orders) -- as it did for the route. + * Each build's file holds the same pages and bytes; their order may differ. + * + * ## `.source/` is regenerated first + * + * `.source/` is written by `pnpm install`'s postinstall, BEFORE the spec + * generators run, so it does not list pages they add: on a fresh checkout + * `content/docs/protocol-upgrade/` (gitignored) does not exist until + * `gen:upgrade-guide` writes it. `next build` regenerates `.source/` when it + * loads `next.config.mjs`; this script calls the same `postInstall()` first, + * so both read the same page set. + * + * A failure anywhere here exits non-zero and fails the build: a deploy without + * this file would 404 `/llms-full.txt` with no error anywhere. + */ + +import { register } from 'node:module'; +import { mkdirSync, writeFileSync } from 'node:fs'; +import { dirname, join, relative, resolve, sep } from 'node:path'; +import { fileURLToPath } from 'node:url'; + +const APP_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..'); +const DEFAULT_OUT = join(APP_ROOT, 'public', 'llms-full.txt'); + +if (!process.features.typescript) { + console.error( + `build-llms-full: this Node (${process.version}) cannot load lib/source.ts: it has no ` + + 'TypeScript type stripping. Use Node 22.18 or newer (CI and the docs deploy install the latest 22).', + ); + process.exit(1); +} + +const out = process.argv[2] ? resolve(process.argv[2]) : DEFAULT_OUT; + +// `source.config.ts` resolves the content directory against the cwd, as it +// does under `next build`, whose cwd is this app. +process.chdir(APP_ROOT); + +const { postInstall } = await import('fumadocs-mdx/next'); +await postInstall(); + +register('./llms-full/resolve-hooks.mjs', import.meta.url); +const { register: registerMdx } = await import('fumadocs-mdx/node'); +registerMdx(); + +const { source, getLLMText } = await import('@/lib/source'); + +const pages = source.getPages(); +if (pages.length === 0) { + console.error('build-llms-full: source.getPages() returned no pages; refusing to write an empty llms-full.txt.'); + process.exit(1); +} + +const body = (await Promise.all(pages.map(getLLMText))).join('\n\n'); +mkdirSync(dirname(out), { recursive: true }); +writeFileSync(out, body); + +const bytes = Buffer.byteLength(body); +console.log( + `build-llms-full: ${pages.length} pages, ${bytes.toLocaleString('en-US')} bytes ` + + `(${(bytes / 1e6).toFixed(2)} MB) -> ${out.startsWith(APP_ROOT + sep) ? relative(APP_ROOT, out) : out}`, +); diff --git a/apps/docs/scripts/llms-full/resolve-hooks.mjs b/apps/docs/scripts/llms-full/resolve-hooks.mjs new file mode 100644 index 00000000000..11fd842e404 --- /dev/null +++ b/apps/docs/scripts/llms-full/resolve-hooks.mjs @@ -0,0 +1,75 @@ +// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license. + +/** + * Node resolution hooks that let `scripts/build-llms-full.mjs` import + * `lib/source.ts` as plain Node -- the module `next build` bundles -- instead + * of a second copy of the page source. + * + * Each rule is the Node spelling of a resolution the Next build already does: + * + * `fumadocs-mdx:collections/X` -> `.source/X.ts` tsconfig.json `paths` + * `@/X` -> `X.ts` tsconfig.json `paths` + * a bare import from a file outside this app that does not resolve there + * (`lucide-react`, `react/jsx-runtime` in a compiled `content/docs/**.mdx`) + * -> resolved from this app, which is what + * next.config.mjs's `turbopack.resolveAlias` + * does for the same reason: `content/` has + * no node_modules of its own. + * + * A `.ts` file reached through an alias is loaded as an ES module with its + * types stripped (Node's own type stripping), so nothing here transpiles. A + * specifier no rule claims goes to Node unchanged, and so does one a rule + * cannot find on disk: a typo fails as loudly as anywhere else. + */ + +import { existsSync, statSync } from 'node:fs'; +import { dirname, join } from 'node:path'; +import { fileURLToPath, pathToFileURL } from 'node:url'; + +const APP_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..'); +const APP_MANIFEST_URL = pathToFileURL(join(APP_ROOT, 'package.json')).href; +const APP_DIR_URL = pathToFileURL(`${APP_ROOT}/`).href; + +const ALIASES = [ + ['fumadocs-mdx:collections/', join(APP_ROOT, '.source')], + ['@/', APP_ROOT], +]; + +function isFile(path) { + return existsSync(path) && statSync(path).isFile(); +} + +function aliasTarget(specifier) { + for (const [prefix, dir] of ALIASES) { + if (!specifier.startsWith(prefix)) continue; + const base = join(dir, specifier.slice(prefix.length)); + for (const file of [base, `${base}.ts`, join(base, 'index.ts')]) { + if (isFile(file)) return file; + } + } + return null; +} + +/** A package name, not a path, URL or `#` import. */ +function isBare(specifier) { + return !/^(?:[./#]|[a-z][a-z0-9+.-]*:)/i.test(specifier); +} + +export async function resolve(specifier, context, nextResolve) { + const file = aliasTarget(specifier); + if (file) { + const url = pathToFileURL(file).href; + return file.endsWith('.ts') + ? { url, format: 'module-typescript', shortCircuit: true } + : { url, shortCircuit: true }; + } + const parent = context.parentURL; + if (isBare(specifier) && parent?.startsWith('file:') && !parent.startsWith(APP_DIR_URL)) { + try { + return await nextResolve(specifier, context); + } catch { + return nextResolve(specifier, { ...context, parentURL: APP_MANIFEST_URL }); + } + } + return nextResolve(specifier, context); +} diff --git a/turbo.json b/turbo.json index 341e2ebfd81..7c60860de7a 100644 --- a/turbo.json +++ b/turbo.json @@ -36,7 +36,7 @@ }, "@objectstack/docs#build": { "dependsOn": ["^build"], - "outputs": ["dist/**", "json-schema/**", ".next/**", "!.next/cache/**"], + "outputs": ["dist/**", "json-schema/**", ".next/**", "!.next/cache/**", "public/llms-full.txt"], "inputs": ["$TURBO_DEFAULT$", "$TURBO_ROOT$/content/**"] }, "@objectstack/skills#build": {