docs: serve markdown to AI agents

middleware.js

@@ -0,0 +1,51 @@
+import { next, rewrite } from '@vercel/functions';
+
+// Vercel Routing Middleware: serves the markdown twin of a docs page (built next to it as /page.md)
+// to AI agents that send `Accept: text/markdown`. The home page has no twin, so it answers with llms.txt.
+export const config = {
+    // Pages (with or without .html) and .md twins only; skips assets and any other file
+    matcher: '/((?!assets/)(?!.*\\.(?!(?:md|html)$)[^/.]+$).*)'
+};
+
+// Suffix of page URLs on this site: '' with cleanUrls, '.html' without
+const PAGE_SUFFIX = '';
+
+const quality = (accept, type) => {
+    for (const part of accept.split(',')) {
+        const [media, ...params] = part.split(';').map(value => value.trim().toLowerCase());
+        if (media !== type) continue;
+        const q = params.find(param => param.startsWith('q='));
+        return q ? Number(q.slice(2)) || 0 : 1;
+    }
+    return 0;
+};
+
+const prefersMarkdown = accept => {
+    const markdown = quality(accept, 'text/markdown');
+    return markdown > 0 && markdown >= quality(accept, 'text/html');
+};
+
+export default async function middleware(request) {
+    const url = new URL(request.url);
+
+    if (url.pathname.endsWith('.md')) {
+        return next({ headers: { Link: `<${url.origin}${url.pathname.slice(0, -3)}${PAGE_SUFFIX}>; rel="canonical"`, Vary: 'Accept' } });
+    }
+    if (!prefersMarkdown(request.headers.get('accept') || '')) {
+        return next({ headers: { Vary: 'Accept' } });
+    }
+
+    const page = url.pathname.replace(/(\/index)?(\.html)?\/?$/, '');
+    const markdown = new URL(page ? `${page}.md` : '/llms.txt', url.origin);
+    // Not every page has a twin (archived versions, for one). Credentials are forwarded and redirects
+    // not followed, so a protected preview's login redirect does not pass for an existing file.
+    const headers = {};
+    for (const name of ['cookie', 'x-vercel-protection-bypass']) {
+        if (request.headers.has(name)) headers[name] = request.headers.get(name);
+    }
+    const twin = await fetch(markdown, { method: 'HEAD', headers, redirect: 'manual' }).catch(() => null);
+    if (!twin?.ok) {
+        return next({ headers: { Vary: 'Accept' } });
+    }
+    return rewrite(markdown, { headers: { Vary: 'Accept' } });
+}

package.json

@@ -25,6 +25,7 @@
     "@salesforce/eslint-plugin-aura": "^3.0.0",
     "@salesforce/eslint-plugin-lightning": "^2.0.0",
     "@salesforce/sfdx-lwc-jest": "^7.1.2",
+    "@vercel/functions": "^3.9.11",
     "eslint": "^9.39.2",
     "eslint-plugin-import": "^2.32.0",
     "eslint-plugin-jest": "^29.5.0",

website/.vitepress/config.mts

@@ -111,6 +111,20 @@ export default defineConfig({
       ['link', { rel: 'canonical', href: canonicalUrl || siteUrl }],
       ['meta', { property: 'og:url', content: canonicalUrl || siteUrl }]
     );
+    // Markdown twin built by vitepress-plugin-llms (llms.txt for the home page),
+    // served by middleware.js on Accept: text/markdown
+    const markdownPath =
+      pageData.relativePath === 'index.md'
+        ? 'llms.txt'
+        : pageData.relativePath.replace(/\/index\.md$/, '.md');
+    pageData.frontmatter.head.push([
+      'link',
+      {
+        rel: 'alternate',
+        type: 'text/markdown',
+        href: `${siteUrl}/${markdownPath}`
+      }
+    ]);
     const pageTitle = pageData.frontmatter.title || pageData.title;
     pageData.frontmatter.head.push(
       [

website/public/robots.txt

@@ -1,4 +1,14 @@
 User-agent: *
+User-agent: GPTBot
+User-agent: OAI-SearchBot
+User-agent: ChatGPT-User
+User-agent: ClaudeBot
+User-agent: Claude-SearchBot
+User-agent: Claude-User
+User-agent: PerplexityBot
+User-agent: Perplexity-User
 Allow: /
+Content-Signal: search=yes, ai-input=yes, ai-train=yes
 
+# Markdown for AI agents: /llms.txt, /llms-full.txt and <page>.md for every page
 Sitemap: https://trigger.beyondthecloud.dev/sitemap.xml