diff --git a/apps/site/baseline/robots.txt b/apps/site/baseline/robots.txt index df3b2b4ae..4fdf890a5 100644 --- a/apps/site/baseline/robots.txt +++ b/apps/site/baseline/robots.txt @@ -15,39 +15,42 @@ Disallow: /dashboard/ User-Agent: GPTBot Allow: / -Allow: /api/search -Allow: /api/mcp -Allow: /.well-known/ -Allow: /llms.txt -Allow: /llms-full.txt -Allow: /auth.md + +User-Agent: ChatGPT-User +Allow: / + +User-Agent: OAI-SearchBot +Allow: / User-Agent: ClaudeBot Allow: / -Allow: /api/search -Allow: /api/mcp -Allow: /.well-known/ -Allow: /llms.txt -Allow: /llms-full.txt -Allow: /auth.md + +User-Agent: Claude-User +Allow: / + +User-Agent: Claude-SearchBot +Allow: / + +User-Agent: Google-Extended +Allow: / + +User-Agent: Amazonbot +Allow: / + +User-Agent: Bytespider +Allow: / + +User-Agent: CCBot +Allow: / User-Agent: PerplexityBot Allow: / -Allow: /api/search -Allow: /api/mcp -Allow: /.well-known/ -Allow: /llms.txt -Allow: /llms-full.txt -Allow: /auth.md User-Agent: Applebot-Extended Allow: / -Allow: /api/search -Allow: /api/mcp -Allow: /.well-known/ -Allow: /llms.txt -Allow: /llms-full.txt -Allow: /auth.md + +User-Agent: meta-externalagent +Allow: / Host: https://codingcat.dev Sitemap: https://codingcat.dev/sitemap.xml diff --git a/apps/site/src/middleware.ts b/apps/site/src/middleware.ts index 5fad33024..af6a7a1a8 100644 --- a/apps/site/src/middleware.ts +++ b/apps/site/src/middleware.ts @@ -57,6 +57,24 @@ export const onRequest = defineMiddleware(async (context, next) => { response = await next(); } + // API error hygiene: structured JSON error responses (RFC 9457 Problem Details) for all /api/ paths + if (pathname.startsWith("/api/") && response.status === 404) { + const problemDetails = { + type: "https://codingcat.dev/errors/not-found", + title: "Not Found", + status: 404, + detail: `The requested API endpoint ${pathname} was not found on this server.`, + instance: pathname, + }; + const headers = new Headers(response.headers); + headers.set("content-type", "application/problem+json; charset=utf-8"); + response = new Response(JSON.stringify(problemDetails, null, 2), { + status: 404, + statusText: "Not Found", + headers, + }); + } + if (context.locals.sanity.preview.enabled) { // Not cosmetic: without this a response containing unpublished drafts can // be written to the edge cache and served to the public. diff --git a/apps/site/src/pages/.well-known/agent-card.json.ts b/apps/site/src/pages/.well-known/agent-card.json.ts index 8d5ab6682..c83ae8573 100644 --- a/apps/site/src/pages/.well-known/agent-card.json.ts +++ b/apps/site/src/pages/.well-known/agent-card.json.ts @@ -15,6 +15,18 @@ export const GET: APIRoute = async ({ locals }) => { url: origin, version: "1.0.0", protocol: "a2a", + supportedInterfaces: [ + { + url: `${origin}/api/mcp`, + protocolBinding: "JSONRPC", + protocolVersion: "2.0", + }, + { + url: `${origin}/api/search`, + protocolBinding: "HTTP+JSON", + protocolVersion: "1.0", + }, + ], endpoints: { mcp: `${origin}/api/mcp`, search: `${origin}/api/search`, diff --git a/apps/site/src/pages/.well-known/agent.json.ts b/apps/site/src/pages/.well-known/agent.json.ts new file mode 100644 index 000000000..d360ee2dc --- /dev/null +++ b/apps/site/src/pages/.well-known/agent.json.ts @@ -0,0 +1 @@ +export { GET, prerender } from "./agent-card.json"; diff --git a/apps/site/src/pages/.well-known/openapi.json.ts b/apps/site/src/pages/.well-known/openapi.json.ts index 858db5480..0a062c044 100644 --- a/apps/site/src/pages/.well-known/openapi.json.ts +++ b/apps/site/src/pages/.well-known/openapi.json.ts @@ -43,84 +43,104 @@ export const GET: APIRoute = async ({ locals }) => { description: "Search keyword or natural language query", schema: { type: "string", + minLength: 1, + description: + "Search term or question to find in tutorials and podcasts", + }, + }, + { + name: "type", + in: "query", + required: false, + description: "Filter results by specific content type", + schema: { + type: "string", + enum: ["all", "post", "podcast"], + default: "all", + description: "Content type filter", }, }, ], responses: { "200": { - description: "Successful search results", + description: + "Successful search results with matching content items", content: { "application/json": { schema: { - type: "object", - properties: { - query: { type: "string" }, - total: { type: "integer" }, - results: { - type: "array", - items: { - type: "object", - properties: { - title: { type: "string" }, - type: { type: "string" }, - url: { type: "string" }, - markdownUrl: { type: "string" }, - excerpt: { type: "string" }, - date: { type: "string" }, - }, - }, - }, - }, + $ref: "#/components/schemas/SearchResponse", }, }, }, }, "400": { - description: "Missing required search query", + description: "Missing or invalid search query parameter", + content: { + "application/problem+json": { + schema: { + $ref: "#/components/schemas/ProblemDetails", + }, + }, + }, + }, + "500": { + description: + "Internal server error during search query execution", + content: { + "application/problem+json": { + schema: { + $ref: "#/components/schemas/ProblemDetails", + }, + }, + }, }, }, }, }, "/api/mcp": { post: { - summary: "Streamable HTTP Model Context Protocol (MCP) Server", + summary: "Model Context Protocol (MCP) Server", description: - "JSON-RPC 2.0 MCP endpoint implementing agent tools for searching and discovering CodingCat.dev technical content.", + "JSON-RPC 2.0 MCP endpoint supporting tools/list, tools/call, server/discover, and initialize for AI agent interaction.", operationId: "mcpEndpoint", requestBody: { required: true, content: { "application/json": { schema: { - type: "object", - properties: { - jsonrpc: { type: "string", example: "2.0" }, - id: { - oneOf: [{ type: "string" }, { type: "integer" }], - }, - method: { type: "string" }, - params: { type: "object" }, - }, - required: ["jsonrpc", "method"], + $ref: "#/components/schemas/McpJsonRpcRequest", }, }, }, }, responses: { "200": { - description: "JSON-RPC 2.0 Response", + description: + "JSON-RPC 2.0 response containing execution result or protocol error", + content: { + "application/json": { + schema: { + $ref: "#/components/schemas/McpJsonRpcResponse", + }, + }, + }, + }, + "400": { + description: "Invalid JSON-RPC request syntax or parse failure", content: { "application/json": { schema: { - type: "object", - properties: { - jsonrpc: { type: "string", example: "2.0" }, - id: { - oneOf: [{ type: "string" }, { type: "integer" }], - }, - result: { type: "object" }, - error: { type: "object" }, - }, + $ref: "#/components/schemas/McpJsonRpcResponse", + }, + }, + }, + }, + "500": { + description: "Unexpected JSON-RPC execution failure", + content: { + "application/json": { + schema: { + $ref: "#/components/schemas/McpJsonRpcResponse", }, }, }, @@ -139,27 +159,283 @@ export const GET: APIRoute = async ({ locals }) => { content: { "application/json": { schema: { - type: "object", - properties: { - name: { type: "string" }, - email: { type: "string", format: "email" }, - company: { type: "string" }, - tier: { type: "string" }, - message: { type: "string" }, - }, - required: ["name", "email", "tier"], + $ref: "#/components/schemas/SponsorshipRequest", }, }, }, }, responses: { "200": { - description: "Inquiry received successfully", + description: "Inquiry received and acknowledged successfully", + content: { + "application/json": { + schema: { + $ref: "#/components/schemas/SponsorshipResponse", + }, + }, + }, }, "400": { - description: "Missing required fields or invalid format", + description: + "Missing required fields or invalid submission payload", + content: { + "application/problem+json": { + schema: { + $ref: "#/components/schemas/ProblemDetails", + }, + }, + }, + }, + "500": { + description: "Failed to process sponsorship inquiry", + content: { + "application/problem+json": { + schema: { + $ref: "#/components/schemas/ProblemDetails", + }, + }, + }, + }, + }, + }, + }, + }, + components: { + schemas: { + ProblemDetails: { + type: "object", + description: "RFC 9457 standard problem details object for HTTP APIs", + properties: { + type: { + type: "string", + format: "uri", + description: "URI reference identifying the problem type", + example: "https://codingcat.dev/errors/not-found", + }, + title: { + type: "string", + description: "Short human-readable summary of the problem type", + example: "Not Found", + }, + status: { + type: "integer", + description: "HTTP status code generated by origin server", + example: 404, + }, + detail: { + type: "string", + description: + "Human-readable explanation specific to this occurrence of the problem", + example: + "The requested API endpoint was not found on this server.", + }, + instance: { + type: "string", + format: "uri-reference", + description: + "URI reference identifying specific occurrence of the problem", + example: "/api/missing", + }, + }, + required: ["type", "title", "status", "detail"], + }, + SearchResult: { + type: "object", + description: + "Individual tutorial, podcast, or article search hit with metadata", + properties: { + title: { + type: "string", + description: "Title of the tutorial or podcast episode", + example: "Building Modern Web Applications", + }, + type: { + type: "string", + enum: ["post", "podcast", "page"], + description: "Content document type in Sanity CMS", + example: "post", + }, + url: { + type: "string", + format: "uri", + description: "Full canonical web URL of the content page", + example: "https://codingcat.dev/post/modern-web", + }, + markdownUrl: { + type: "string", + format: "uri", + description: + "Direct token-efficient Markdown representation of the content", + example: "https://codingcat.dev/post/modern-web.md", + }, + excerpt: { + type: "string", + description: "Brief summary or meta description of the entry", + example: "Learn how to build modern web apps...", + }, + date: { + type: "string", + format: "date-time", + description: "Publication date in ISO 8601 format", + example: "2026-01-15T00:00:00.000Z", + }, + }, + required: ["title", "type", "url", "markdownUrl"], + }, + SearchResponse: { + type: "object", + description: + "Search results container including original query and hit list", + properties: { + query: { + type: "string", + description: + "Normalized query term executed against the search index", + example: "next.js vs astro", + }, + total: { + type: "integer", + description: "Total number of matching content records returned", + example: 12, + }, + results: { + type: "array", + description: "List of matched content items sorted by relevance", + items: { + $ref: "#/components/schemas/SearchResult", + }, + }, + }, + required: ["query", "total", "results"], + }, + McpJsonRpcRequest: { + type: "object", + description: "Model Context Protocol JSON-RPC 2.0 request payload", + properties: { + jsonrpc: { + type: "string", + enum: ["2.0"], + description: "JSON-RPC specification version", + example: "2.0", + }, + id: { + oneOf: [{ type: "string" }, { type: "integer" }], + description: "Unique request identifier echoed in response", + example: 1, + }, + method: { + type: "string", + description: + "MCP method name (e.g. tools/list, tools/call, server/discover, initialize)", + example: "tools/list", + }, + params: { + type: "object", + description: "Method arguments object", + }, + }, + required: ["jsonrpc", "method"], + }, + McpJsonRpcError: { + type: "object", + description: "JSON-RPC 2.0 error descriptor", + properties: { + code: { + type: "integer", + description: "Standard JSON-RPC error code", + example: -32601, + }, + message: { + type: "string", + description: "Short human-readable error description", + example: "Method not found", + }, + data: { + type: "object", + description: "Additional structured error diagnostic details", + }, + }, + required: ["code", "message"], + }, + McpJsonRpcResponse: { + type: "object", + description: "Model Context Protocol JSON-RPC 2.0 response payload", + properties: { + jsonrpc: { + type: "string", + enum: ["2.0"], + example: "2.0", + }, + id: { + oneOf: [ + { type: "string" }, + { type: "integer" }, + { type: "null" }, + ], + example: 1, + }, + result: { + type: "object", + description: "Success payload for the executed method", + }, + error: { + $ref: "#/components/schemas/McpJsonRpcError", + }, + }, + required: ["jsonrpc"], + }, + SponsorshipRequest: { + type: "object", + description: "Sponsorship inquiry submission parameters", + properties: { + name: { + type: "string", + description: "Contact name of the sponsor or marketer", + example: "Jane Doe", + }, + email: { + type: "string", + format: "email", + description: "Direct email address for sponsorship follow-up", + example: "sponsor@example.com", + }, + company: { + type: "string", + description: "Company or brand name seeking sponsorship", + example: "Acme Cloud", + }, + tier: { + type: "string", + description: "Desired sponsorship package or campaign format", + example: "podcast-episode", + }, + message: { + type: "string", + description: + "Campaign goals, target audience, and preferred dates", + example: "We would like to sponsor 3 episodes in Q2.", + }, + }, + required: ["name", "email", "tier"], + }, + SponsorshipResponse: { + type: "object", + description: "Confirmation of sponsorship submission", + properties: { + success: { + type: "boolean", + description: + "Indicates whether the inquiry was successfully recorded", + example: true, + }, + message: { + type: "string", + description: "User-facing acknowledgement of submission status", + example: + "Inquiry received. We will respond within 2 business days.", }, }, + required: ["success", "message"], }, }, }, diff --git a/apps/site/src/pages/api/mcp.ts b/apps/site/src/pages/api/mcp.ts index 84040040c..c88ad5784 100644 --- a/apps/site/src/pages/api/mcp.ts +++ b/apps/site/src/pages/api/mcp.ts @@ -14,17 +14,39 @@ interface JsonRpcRequest { params?: any; } +const MCP_HEADERS = { + "content-type": "application/json; charset=utf-8", + "access-control-allow-origin": "*", + "access-control-allow-methods": "GET, POST, OPTIONS", + "access-control-allow-headers": + "Content-Type, Authorization, Mcp-Session-Id, Mcp-Protocol-Version", + "mcp-protocol-version": "2024-11-05", +}; + +export const OPTIONS: APIRoute = async () => { + return new Response(null, { + status: 204, + headers: MCP_HEADERS, + }); +}; + export const GET: APIRoute = async ({ locals }) => { const origin = locals.siteUrl.origin; return new Response( JSON.stringify( { status: "ok", - server: "CodingCat.dev Search MCP Server", + server: "codingcatdev-search-mcp", version: "1.0.0", - protocolVersion: "2025-06-18", + protocolVersion: "2024-11-05", + supportedVersions: ["2026-07-28", "2024-11-05"], transport: "streamable-http", serverCard: `${origin}/.well-known/mcp/server-card.json`, + capabilities: { + tools: { listChanged: false }, + resources: { subscribe: false, listChanged: false }, + prompts: { listChanged: false }, + }, tools: ["search_content"], }, null, @@ -32,7 +54,7 @@ export const GET: APIRoute = async ({ locals }) => { ), { headers: { - "content-type": "application/json; charset=utf-8", + ...MCP_HEADERS, "cache-control": "public, max-age=3600", }, }, @@ -52,23 +74,60 @@ export const POST: APIRoute = async ({ request, locals }) => { id: null, error: { code: -32700, message: "Parse error" }, }), - { status: 400, headers: { "content-type": "application/json" } }, + { status: 400, headers: MCP_HEADERS }, ); } const id = body.id ?? null; const method = body.method; - // 1. Initialize + // 1. Stateless discovery (MCP 2026-07-28+) + if (method === "server/discover") { + return new Response( + JSON.stringify({ + jsonrpc: "2.0", + id, + result: { + supportedVersions: ["2026-07-28", "2024-11-05"], + capabilities: { + tools: { listChanged: false }, + resources: { subscribe: false, listChanged: false }, + prompts: { listChanged: false }, + }, + serverInfo: { + name: "codingcatdev-search-mcp", + version: "1.0.0", + }, + _meta: { + "io.modelcontextprotocol/serverInfo": { + name: "codingcatdev-search-mcp", + version: "1.0.0", + }, + }, + instructions: + "Search technical tutorials, web development guides, and podcasts on CodingCat.dev.", + }, + }), + { headers: MCP_HEADERS }, + ); + } + + // 2. Initialize (Legacy / standard MCP handshake) if (method === "initialize") { + const clientVersion = body.params?.protocolVersion; + const protocolVersion = + clientVersion === "2026-07-28" ? "2026-07-28" : "2024-11-05"; + return new Response( JSON.stringify({ jsonrpc: "2.0", id, result: { - protocolVersion: "2025-06-18", + protocolVersion, capabilities: { - tools: {}, + tools: { listChanged: false }, + resources: { subscribe: false, listChanged: false }, + prompts: { listChanged: false }, }, serverInfo: { name: "codingcatdev-search-mcp", @@ -76,11 +135,11 @@ export const POST: APIRoute = async ({ request, locals }) => { }, }, }), - { headers: { "content-type": "application/json" } }, + { headers: MCP_HEADERS }, ); } - // 2. Notifications (initialized acknowledgement) + // 3. Notifications (initialized acknowledgement) if (method === "notifications/initialized") { return new Response( JSON.stringify({ @@ -88,11 +147,11 @@ export const POST: APIRoute = async ({ request, locals }) => { id, result: {}, }), - { headers: { "content-type": "application/json" } }, + { headers: MCP_HEADERS }, ); } - // 3. Ping + // 4. Ping if (method === "ping") { return new Response( JSON.stringify({ @@ -100,11 +159,11 @@ export const POST: APIRoute = async ({ request, locals }) => { id, result: {}, }), - { headers: { "content-type": "application/json" } }, + { headers: MCP_HEADERS }, ); } - // 4. Tools list + // 5. Tools list if (method === "tools/list") { return new Response( JSON.stringify({ @@ -137,11 +196,39 @@ export const POST: APIRoute = async ({ request, locals }) => { ], }, }), - { headers: { "content-type": "application/json" } }, + { headers: MCP_HEADERS }, + ); + } + + // 6. Resources list + if (method === "resources/list") { + return new Response( + JSON.stringify({ + jsonrpc: "2.0", + id, + result: { + resources: [], + }, + }), + { headers: MCP_HEADERS }, + ); + } + + // 7. Prompts list + if (method === "prompts/list") { + return new Response( + JSON.stringify({ + jsonrpc: "2.0", + id, + result: { + prompts: [], + }, + }), + { headers: MCP_HEADERS }, ); } - // 5. Tools call + // 8. Tools call if (method === "tools/call") { const toolName = body.params?.name; const args = body.params?.arguments || {}; @@ -165,7 +252,7 @@ export const POST: APIRoute = async ({ request, locals }) => { isError: true, }, }), - { headers: { "content-type": "application/json" } }, + { headers: MCP_HEADERS }, ); } @@ -230,7 +317,7 @@ export const POST: APIRoute = async ({ request, locals }) => { ], }, }), - { headers: { "content-type": "application/json" } }, + { headers: MCP_HEADERS }, ); } @@ -240,16 +327,17 @@ export const POST: APIRoute = async ({ request, locals }) => { id, error: { code: -32601, message: `Tool not found: ${toolName}` }, }), - { status: 404, headers: { "content-type": "application/json" } }, + { headers: MCP_HEADERS }, ); } + // Default fallback: standard JSON-RPC 2.0 Method Not Found return new Response( JSON.stringify({ jsonrpc: "2.0", id, error: { code: -32601, message: `Method not found: ${method}` }, }), - { status: 404, headers: { "content-type": "application/json" } }, + { headers: MCP_HEADERS }, ); }; diff --git a/apps/site/src/pages/robots.txt.ts b/apps/site/src/pages/robots.txt.ts index b21a87e3c..367c6e1db 100644 --- a/apps/site/src/pages/robots.txt.ts +++ b/apps/site/src/pages/robots.txt.ts @@ -36,39 +36,42 @@ Disallow: /dashboard/ User-Agent: GPTBot Allow: / -Allow: /api/search -Allow: /api/mcp -Allow: /.well-known/ -Allow: /llms.txt -Allow: /llms-full.txt -Allow: /auth.md + +User-Agent: ChatGPT-User +Allow: / + +User-Agent: OAI-SearchBot +Allow: / User-Agent: ClaudeBot Allow: / -Allow: /api/search -Allow: /api/mcp -Allow: /.well-known/ -Allow: /llms.txt -Allow: /llms-full.txt -Allow: /auth.md + +User-Agent: Claude-User +Allow: / + +User-Agent: Claude-SearchBot +Allow: / + +User-Agent: Google-Extended +Allow: / + +User-Agent: Amazonbot +Allow: / + +User-Agent: Bytespider +Allow: / + +User-Agent: CCBot +Allow: / User-Agent: PerplexityBot Allow: / -Allow: /api/search -Allow: /api/mcp -Allow: /.well-known/ -Allow: /llms.txt -Allow: /llms-full.txt -Allow: /auth.md User-Agent: Applebot-Extended Allow: / -Allow: /api/search -Allow: /api/mcp -Allow: /.well-known/ -Allow: /llms.txt -Allow: /llms-full.txt -Allow: /auth.md + +User-Agent: meta-externalagent +Allow: / Host: ${origin} Sitemap: ${origin}/sitemap.xml diff --git a/apps/site/src/pages/sitemap-index.xml.ts b/apps/site/src/pages/sitemap-index.xml.ts new file mode 100644 index 000000000..478d34368 --- /dev/null +++ b/apps/site/src/pages/sitemap-index.xml.ts @@ -0,0 +1,28 @@ +import type { APIRoute } from "astro"; + +export const prerender = false; + +/** + * Standard XML Sitemap Index (RFC / sitemaps.org 0.9) + * Directs search and agent crawlers to the main content sitemap. + */ +export const GET: APIRoute = async ({ locals }) => { + const origin = locals.siteUrl.origin; + const now = new Date().toISOString(); + + const body = ` + + + ${origin}/sitemap.xml + ${now} + + +`; + + return new Response(body, { + headers: { + "content-type": "application/xml; charset=utf-8", + "cache-control": "max-age=0, s-maxage=3600", + }, + }); +};