Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions apps/docs/.gitignore
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
# Written by `node scripts/build-llms-full.mjs` (the `build` script): a build
# artifact of every docs page, ~20 MB, never committed.
/public/llms-full.txt
10 changes: 0 additions & 10 deletions apps/docs/app/llms-full.txt/route.ts

This file was deleted.

5 changes: 4 additions & 1 deletion apps/docs/next.config.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,10 @@ const config = {
//
// 1. Coverage. Only the per-page markdown has an HTML twin to point a
// canonical at. `/llms.txt` is an index of every page and
// `/llms-full.txt` is all 400+ of them concatenated (8 MB today); there
// `/llms-full.txt` is all 400+ of them concatenated (20 MB in 2026-10, a
// static file `scripts/build-llms-full.mjs` writes into `public/` --
// the header applies there too, since Next checks headers before the
// filesystem); there
// is no single HTML URL either one is a duplicate *of*, so a canonical
// header cannot say anything honest about them and would leave the two
// largest parallel copies on the site undirected. `noindex` states the
Expand Down
2 changes: 1 addition & 1 deletion apps/docs/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@
"license": "Apache-2.0",
"scripts": {
"dev": "next dev",
"build": "pnpm --filter @objectstack/spec gen:schema && pnpm --filter @objectstack/spec gen:docs && pnpm --filter @objectstack/spec gen:upgrade-guide && NODE_OPTIONS='--max-old-space-size=4096' next build",
"build": "pnpm --filter @objectstack/spec gen:schema && pnpm --filter @objectstack/spec gen:docs && pnpm --filter @objectstack/spec gen:upgrade-guide && node scripts/build-llms-full.mjs && NODE_OPTIONS='--max-old-space-size=4096' next build",
"start": "next start",
"site:lint": "next lint",
"postinstall": "fumadocs-mdx",
Expand Down
101 changes: 101 additions & 0 deletions apps/docs/scripts/build-llms-full.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,101 @@
#!/usr/bin/env node
// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.

/**
* Writes `public/llms-full.txt` -- every docs page's `getLLMText()`, joined by
* one blank line, in `source.getPages()` order -- at BUILD time, as a static
* file. Run by this app's `build` script, after the spec generators and before
* `next build`:
*
* node scripts/build-llms-full.mjs # writes public/llms-full.txt
* node scripts/build-llms-full.mjs <out-file> # writes elsewhere (measuring)
*
* ## Why a build-time file and not a route handler
*
* `/llms-full.txt` used to be `app/llms-full.txt/route.ts` with
* `revalidate = false`. `next build` prerenders such a route, and Vercel stores
* a prerendered route as an ISR page whose body is its fallback; the deploy API
* refuses a fallback body over 20 MB (`FALLBACK_BODY_TOO_LARGE`, at the
* `vercel deploy --prebuilt` upload, after a green build). The concatenation
* measured 20,032,800 bytes over 418 pages at e611e2bda, so every production
* docs deploy failed. A file under `public/` is not a prerender: Vercel's Next
* builder copies it into the build output's `static/` tree and the CDN serves
* it as a file, so that cap does not apply to it at any size. The URL, the body
* and the `X-Robots-Tag: noindex` header (`next.config.mjs` `headers()`, which
* Next applies before the filesystem, `public/` included) are unchanged.
*
* ## One producer
*
* The text is `getLLMText()` over `source.getPages()` from `lib/source.ts` --
* the module the per-page `/llms.mdx/docs/**` route and `/llms.txt` read --
* imported here as plain Node: fumadocs-mdx's Node loader compiles each page
* with the same `source.config.ts`, and `llms-full/resolve-hooks.mjs` maps the
* two tsconfig aliases. When this script replaced the route, its output was
* byte-identical (sha256) to the route's prerendered body read off the same
* `.source/`. A second copy of the page list or of `getLLMText` here would fork
* the two.
*
* Page ORDER is `source.getPages()` order, which follows the `.source/` index,
* and fumadocs-mdx writes that index in a different order on every run
* (measured: four runs on one tree, four orders) -- as it did for the route.
* Each build's file holds the same pages and bytes; their order may differ.
*
* ## `.source/` is regenerated first
*
* `.source/` is written by `pnpm install`'s postinstall, BEFORE the spec
* generators run, so it does not list pages they add: on a fresh checkout
* `content/docs/protocol-upgrade/` (gitignored) does not exist until
* `gen:upgrade-guide` writes it. `next build` regenerates `.source/` when it
* loads `next.config.mjs`; this script calls the same `postInstall()` first,
* so both read the same page set.
*
* A failure anywhere here exits non-zero and fails the build: a deploy without
* this file would 404 `/llms-full.txt` with no error anywhere.
*/

import { register } from 'node:module';
import { mkdirSync, writeFileSync } from 'node:fs';
import { dirname, join, relative, resolve, sep } from 'node:path';
import { fileURLToPath } from 'node:url';

const APP_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..');
const DEFAULT_OUT = join(APP_ROOT, 'public', 'llms-full.txt');

if (!process.features.typescript) {
console.error(
`build-llms-full: this Node (${process.version}) cannot load lib/source.ts: it has no ` +
'TypeScript type stripping. Use Node 22.18 or newer (CI and the docs deploy install the latest 22).',
);
process.exit(1);
}

const out = process.argv[2] ? resolve(process.argv[2]) : DEFAULT_OUT;

// `source.config.ts` resolves the content directory against the cwd, as it
// does under `next build`, whose cwd is this app.
process.chdir(APP_ROOT);

const { postInstall } = await import('fumadocs-mdx/next');
await postInstall();

register('./llms-full/resolve-hooks.mjs', import.meta.url);
const { register: registerMdx } = await import('fumadocs-mdx/node');
registerMdx();

const { source, getLLMText } = await import('@/lib/source');

const pages = source.getPages();
if (pages.length === 0) {
console.error('build-llms-full: source.getPages() returned no pages; refusing to write an empty llms-full.txt.');
process.exit(1);
}

const body = (await Promise.all(pages.map(getLLMText))).join('\n\n');
mkdirSync(dirname(out), { recursive: true });
writeFileSync(out, body);

const bytes = Buffer.byteLength(body);
console.log(
`build-llms-full: ${pages.length} pages, ${bytes.toLocaleString('en-US')} bytes ` +
`(${(bytes / 1e6).toFixed(2)} MB) -> ${out.startsWith(APP_ROOT + sep) ? relative(APP_ROOT, out) : out}`,
);
75 changes: 75 additions & 0 deletions apps/docs/scripts/llms-full/resolve-hooks.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,75 @@
// Copyright (c) 2026 ObjectStack. Licensed under the Apache-2.0 license.

/**
* Node resolution hooks that let `scripts/build-llms-full.mjs` import
* `lib/source.ts` as plain Node -- the module `next build` bundles -- instead
* of a second copy of the page source.
*
* Each rule is the Node spelling of a resolution the Next build already does:
*
* `fumadocs-mdx:collections/X` -> `.source/X.ts` tsconfig.json `paths`
* `@/X` -> `X.ts` tsconfig.json `paths`
* a bare import from a file outside this app that does not resolve there
* (`lucide-react`, `react/jsx-runtime` in a compiled `content/docs/**.mdx`)
* -> resolved from this app, which is what
* next.config.mjs's `turbopack.resolveAlias`
* does for the same reason: `content/` has
* no node_modules of its own.
*
* A `.ts` file reached through an alias is loaded as an ES module with its
* types stripped (Node's own type stripping), so nothing here transpiles. A
* specifier no rule claims goes to Node unchanged, and so does one a rule
* cannot find on disk: a typo fails as loudly as anywhere else.
*/

import { existsSync, statSync } from 'node:fs';
import { dirname, join } from 'node:path';
import { fileURLToPath, pathToFileURL } from 'node:url';

const APP_ROOT = join(dirname(fileURLToPath(import.meta.url)), '..', '..');
const APP_MANIFEST_URL = pathToFileURL(join(APP_ROOT, 'package.json')).href;
const APP_DIR_URL = pathToFileURL(`${APP_ROOT}/`).href;

const ALIASES = [
['fumadocs-mdx:collections/', join(APP_ROOT, '.source')],
['@/', APP_ROOT],
];

function isFile(path) {
return existsSync(path) && statSync(path).isFile();
}

function aliasTarget(specifier) {
for (const [prefix, dir] of ALIASES) {
if (!specifier.startsWith(prefix)) continue;
const base = join(dir, specifier.slice(prefix.length));
for (const file of [base, `${base}.ts`, join(base, 'index.ts')]) {
if (isFile(file)) return file;
}
}
return null;
}

/** A package name, not a path, URL or `#` import. */
function isBare(specifier) {
return !/^(?:[./#]|[a-z][a-z0-9+.-]*:)/i.test(specifier);
}

export async function resolve(specifier, context, nextResolve) {
const file = aliasTarget(specifier);
if (file) {
const url = pathToFileURL(file).href;
return file.endsWith('.ts')
? { url, format: 'module-typescript', shortCircuit: true }
: { url, shortCircuit: true };
}
const parent = context.parentURL;
if (isBare(specifier) && parent?.startsWith('file:') && !parent.startsWith(APP_DIR_URL)) {
try {
return await nextResolve(specifier, context);
} catch {
return nextResolve(specifier, { ...context, parentURL: APP_MANIFEST_URL });
}
}
return nextResolve(specifier, context);
}
2 changes: 1 addition & 1 deletion turbo.json
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@
},
"@objectstack/docs#build": {
"dependsOn": ["^build"],
"outputs": ["dist/**", "json-schema/**", ".next/**", "!.next/cache/**"],
"outputs": ["dist/**", "json-schema/**", ".next/**", "!.next/cache/**", "public/llms-full.txt"],
"inputs": ["$TURBO_DEFAULT$", "$TURBO_ROOT$/content/**"]
},
"@objectstack/skills#build": {
Expand Down
Loading