feat: comprehensive agent readiness and protocol perfection

apps/site/baseline/robots.txt

@@ -15,39 +15,42 @@ Disallow: /dashboard/
 
 User-Agent: GPTBot
 Allow: /
-Allow: /api/search
-Allow: /api/mcp
-Allow: /.well-known/
-Allow: /llms.txt
-Allow: /llms-full.txt
-Allow: /auth.md
+
+User-Agent: ChatGPT-User
+Allow: /
+
+User-Agent: OAI-SearchBot
+Allow: /
 
 User-Agent: ClaudeBot
 Allow: /
-Allow: /api/search
-Allow: /api/mcp
-Allow: /.well-known/
-Allow: /llms.txt
-Allow: /llms-full.txt
-Allow: /auth.md
+
+User-Agent: Claude-User
+Allow: /
+
+User-Agent: Claude-SearchBot
+Allow: /
+
+User-Agent: Google-Extended
+Allow: /
+
+User-Agent: Amazonbot
+Allow: /
+
+User-Agent: Bytespider
+Allow: /
+
+User-Agent: CCBot
+Allow: /
 
 User-Agent: PerplexityBot
 Allow: /
-Allow: /api/search
-Allow: /api/mcp
-Allow: /.well-known/
-Allow: /llms.txt
-Allow: /llms-full.txt
-Allow: /auth.md
 
 User-Agent: Applebot-Extended
 Allow: /
-Allow: /api/search
-Allow: /api/mcp
-Allow: /.well-known/
-Allow: /llms.txt
-Allow: /llms-full.txt
-Allow: /auth.md
+
+User-Agent: meta-externalagent
+Allow: /
 
 Host: https://codingcat.dev
 Sitemap: https://codingcat.dev/sitemap.xml

apps/site/src/middleware.ts

@@ -57,6 +57,24 @@ export const onRequest = defineMiddleware(async (context, next) => {
 		response = await next();
 	}
 
+	// API error hygiene: structured JSON error responses (RFC 9457 Problem Details) for all /api/ paths
+	if (pathname.startsWith("/api/") && response.status === 404) {
+		const problemDetails = {
+			type: "https://codingcat.dev/errors/not-found",
+			title: "Not Found",
+			status: 404,
+			detail: `The requested API endpoint ${pathname} was not found on this server.`,
+			instance: pathname,
+		};
+		const headers = new Headers(response.headers);
+		headers.set("content-type", "application/problem+json; charset=utf-8");
+		response = new Response(JSON.stringify(problemDetails, null, 2), {
+			status: 404,
+			statusText: "Not Found",
+			headers,
+		});
+	}
+
 	if (context.locals.sanity.preview.enabled) {
 		// Not cosmetic: without this a response containing unpublished drafts can
 		// be written to the edge cache and served to the public.

apps/site/src/pages/.well-known/agent-card.json.ts

@@ -15,6 +15,18 @@ export const GET: APIRoute = async ({ locals }) => {
 		url: origin,
 		version: "1.0.0",
 		protocol: "a2a",
+		supportedInterfaces: [
+			{
+				url: `${origin}/api/mcp`,
+				protocolBinding: "JSONRPC",
+				protocolVersion: "2.0",
+			},
+			{
+				url: `${origin}/api/search`,
+				protocolBinding: "HTTP+JSON",
+				protocolVersion: "1.0",
+			},
+		],
 		endpoints: {
 			mcp: `${origin}/api/mcp`,
 			search: `${origin}/api/search`,

apps/site/src/pages/.well-known/agent.json.ts

@@ -0,0 +1 @@
+export { GET, prerender } from "./agent-card.json";