feat: agent discovery (Link headers, markdown negotiation, Content-Signal) (#263)

* docs(spec): add agent discovery design

Brainstormed spec for three agent-discovery fixes: Content-Signal in
robots.txt, Link response headers, and markdown content negotiation.
Ephemeral — to be removed after implementation.

* docs(plan): add agent discovery implementation plan

Task-by-task implementation plan for the three fixes in the
companion spec. Ephemeral — removed as the final task.

* feat(robots): declare Content-Signal AI usage preferences

* feat(vercel): add Link response headers for agent discovery

* feat(build): copy source .md files to dist for agent markdown negotiation

* feat(vercel): rewrite to .md when Accept: text/markdown

* style: format with prettier

* chore: remove ephemeral agent-discovery spec and plan

Author: sriramveeraghanta

docs/.vitepress/config.mts

@@ -1,8 +1,8 @@
 import { defineConfig, type HeadConfig } from "vitepress";
 import { tabsMarkdownPlugin } from "vitepress-plugin-tabs";
 import { withMermaid } from "vitepress-plugin-mermaid";
-import { readFileSync } from "node:fs";
-import { resolve } from "node:path";
+import { readFileSync, readdirSync, statSync, mkdirSync, copyFileSync } from "node:fs";
+import { resolve, join, relative, dirname } from "node:path";
 
 function loadEnvVar(key: string): string | undefined {
   // process.env takes precedence (CI/hosting platforms set vars here)
@@ -75,6 +75,29 @@ export default withMermaid(
         ],
       },
     },
+    buildEnd(siteConfig) {
+      // Copy source .md files into dist/ for Accept: text/markdown negotiation.
+      const srcDir = siteConfig.srcDir;
+      const outDir = siteConfig.outDir;
+
+      function walk(dir: string): void {
+        for (const entry of readdirSync(dir)) {
+          if (entry === ".vitepress" || entry === "public" || entry === "node_modules") continue;
+          const abs = join(dir, entry);
+          const stat = statSync(abs);
+          if (stat.isDirectory()) {
+            walk(abs);
+          } else if (stat.isFile() && abs.endsWith(".md")) {
+            const rel = relative(srcDir, abs);
+            const dest = join(outDir, rel);
+            mkdirSync(dirname(dest), { recursive: true });
+            copyFileSync(abs, dest);
+          }
+        }
+      }
+
+      walk(srcDir);
+    },
     title: "Plane developer documentation",
     description:
       "Self-host Plane, integrate with our API, configure webhooks, and extend your project management platform. Complete guides for developers building on Plane.",

docs/public/robots.txt

@@ -8,6 +8,10 @@ Allow: /
 # Disallow crawling of search results (if any)
 Disallow: /search
 
+# Content Signals — AI content usage preferences
+# https://contentsignals.org/
+Content-Signal: search=yes, ai-train=yes, ai-input=yes
+
 # Disallow crawling of any internal/private paths (add as needed)
 # Disallow: /private/
 # Disallow: /admin/

vercel.json

@@ -1,5 +1,16 @@
 {
   "cleanUrls": true,
+  "headers": [
+    {
+      "source": "/(.*)",
+      "headers": [
+        {
+          "key": "Link",
+          "value": "</llms.txt>; rel=\"describedby\"; type=\"text/plain\", </api-reference/introduction>; rel=\"service-doc\"; type=\"text/html\", </sitemap.xml>; rel=\"sitemap\"; type=\"application/xml\""
+        }
+      ]
+    }
+  ],
   "redirects": [
     {
       "source": "/api-reference",
@@ -93,5 +104,12 @@
       "source": "/self-hosting",
       "destination": "/self-hosting/overview"
     }
+  ],
+  "rewrites": [
+    {
+      "source": "/:path*",
+      "has": [{ "type": "header", "key": "accept", "value": ".*text/markdown.*" }],
+      "destination": "/:path*.md"
+    }
   ]
 }