feat(shell-docs): add sitemap, robots, and per-framework self-canonical metadata (CopilotKit#4703)

## Summary

Phase 3 of the docs.copilotkit.ai cutover. Surfaces a complete sitemap,
basic robots config, and per-framework self-canonical metadata so every
URL variant is indexed under its own canonical instead of collapsing
onto a single root.

- **sitemap.ts** — emits one entry per (root URL × framework variant)
pair plus reference, AG-UI, and per-framework override pages.
`lastModified` resolves from MDX frontmatter \`lastmod\` first, then
file mtime, then \`new Date()\`. Strips Next.js route-group \`(name)\`
segments and trailing \`/index\` so URLs match what the routers actually
serve. ~2,250 entries.
- **robots.ts** — allow all, disallow \`/api/\`, sitemap pointer at
\`\${NEXT_PUBLIC_BASE_URL}/sitemap.xml\`.
- **sitemap-helpers.ts** — shared MDX walking + base-URL resolution.
- **generateMetadata()** added to the four catch-all docs routes
(\`[[...slug]]\`, \`[framework]/[[...slug]]\`, \`reference/[...slug]\`,
\`ag-ui/[[...slug]]\`). Each sets \`alternates.canonical\` to the page's
own full URL — per-framework self-canonical, not root canonical. So
\`/langgraph-python/quickstart\` declares itself canonical,
\`/agno/quickstart\` declares itself canonical, and the bare
\`/quickstart\` declares itself canonical too.
- **.env.example** — documents \`NEXT_PUBLIC_BASE_URL\` and
\`NEXT_PUBLIC_SHELL_URL\`. Extended the existing comment in
\`next.config.ts\` with the new consumers.

## Test plan

- [x] \`npm run build\` from \`showcase/shell-docs/\` passes; route
table shows \`/sitemap.xml\` and \`/robots.txt\` as static.
- [x] \`curl http://localhost:3099/sitemap.xml\` returns valid XML;
2,254 \`<url>\` entries covering bare unscoped, framework-scoped,
reference, and AG-UI URLs; no \`(other)\` route-group leakage; no
trailing \`/index\` artifacts.
- [x] \`curl http://localhost:3099/robots.txt\` returns the expected
User-Agent / Allow / Disallow / Sitemap config.
- [x] \`/shared-state\` HTML contains \`<link rel="canonical"
href=".../shared-state">\`.
- [x] \`/langgraph-python/quickstart\` HTML contains \`<link
rel="canonical" href=".../langgraph-python/quickstart">\` (NOT pointing
at the bare \`/quickstart\`).
- [x] \`/agno/quickstart\` and \`/reference/components/CopilotChat\` and
\`/ag-ui/concepts/architecture\` all self-canonical.

Author: AlemTuzlak

showcase/shell-docs/.env.example

@@ -0,0 +1,16 @@
+# shell-docs environment variables.
+#
+# Both NEXT_PUBLIC_* values below are required for `next build`
+# (next.config.ts throws when either is missing during a production
+# build). In dev, missing values fall back to localhost defaults so
+# you can run `next dev` without a .env.local.
+
+# Canonical base URL used by sitemap.ts, robots.ts, and the per-page
+# canonical metadata. In production this points at the docs host so
+# crawlers index every framework variant at its self-canonical URL.
+NEXT_PUBLIC_BASE_URL=https://docs.copilotkit.ai
+
+# URL of the showcase shell host, which owns /integrations and /matrix.
+# Used by the top-nav cross-host links and InlineDemo. Replace with your
+# deployment's shell host.
+NEXT_PUBLIC_SHELL_URL=https://www.copilotkit.ai

showcase/shell-docs/next.config.ts

@@ -4,6 +4,13 @@ import type { NextConfig } from "next";
 // because of the NEXT_PUBLIC_ prefix. Do NOT re-declare it in an `env` block —
 // doing so bakes the build-time value into server code and overrides runtime env.
 //
+// Consumers in production: src/app/sitemap.ts, src/app/robots.ts, and the
+// per-page `generateMetadata()` canonical URLs in the catch-all routes
+// (src/app/[[...slug]]/page.tsx, src/app/[framework]/[[...slug]]/page.tsx,
+// src/app/reference/[...slug]/page.tsx, src/app/ag-ui/[[...slug]]/page.tsx).
+// All read it through `getBaseUrl()` in src/lib/sitemap-helpers.ts, which
+// falls back to https://docs.copilotkit.ai when unset.
+//
 // Fail fast during an actual `next build` if the variable is missing, so we
 // never ship broken absolute URLs. Other invocations that also load this
 // config (e.g. `next lint`, `next dev`) only warn, because failing them on a

showcase/shell-docs/src/app/[[...slug]]/page.tsx

@@ -7,6 +7,7 @@
 // the user chooses one — code without a backend context is incomplete.
 
 import React from "react";
+import type { Metadata } from "next";
 import Link from "next/link";
 import { DocsLandingNext } from "@/components/docs-landing-next";
 import { SidebarFrameworkSelector } from "@/components/sidebar-framework-selector";
@@ -15,6 +16,25 @@ import { SidebarNav } from "@/components/sidebar-nav";
 import { UnscopedDocsPage } from "@/components/unscoped-docs-page";
 import { CONTENT_DIR, buildNavTree } from "@/lib/docs-render";
 import type { NavNode } from "@/lib/docs-render";
+import { getBaseUrl } from "@/lib/sitemap-helpers";
+
+// Per-framework self-canonical: each variant of a doc page declares
+// itself canonical so search engines index every framework's quickstart
+// (etc.) at its own URL rather than collapsing them all onto the bare
+// /quickstart. Done at the page level so the metadata depends on params.
+export async function generateMetadata({
+  params,
+}: {
+  params: Promise<{ slug?: string[] }>;
+}): Promise<Metadata> {
+  const { slug } = await params;
+  const slugPath = slug && slug.length > 0 ? `/${slug.join("/")}` : "";
+  return {
+    alternates: {
+      canonical: `${getBaseUrl()}${slugPath}`,
+    },
+  };
+}
 
 function DocsOverview() {
   const navTree = buildNavTree(CONTENT_DIR);

showcase/shell-docs/src/app/[framework]/[[...slug]]/page.tsx

@@ -15,6 +15,7 @@
 // correctly even though Next.js routes them here before [[...slug]].
 
 import React from "react";
+import type { Metadata } from "next";
 import { notFound, redirect } from "next/navigation";
 import Link from "next/link";
 import { DocsLandingNext } from "@/components/docs-landing-next";
@@ -32,9 +33,31 @@ import {
 import type { NavNode } from "@/lib/docs-render";
 import { getDocsFolder, getIntegration, getIntegrations } from "@/lib/registry";
 import type { Integration } from "@/lib/registry";
+import { getBaseUrl } from "@/lib/sitemap-helpers";
 import { RESERVED_ROUTE_SLUGS } from "@/app/layout";
 import demoContent from "@/data/demo-content.json";
 
+// Per-framework self-canonical: /<framework>/<slug> declares itself
+// canonical (NOT the bare /<slug>) so search engines index each
+// framework variant at its own URL. When the URL's first segment
+// doesn't match a registered integration, the route falls through to
+// UnscopedDocsPage but the canonical still points at the same URL —
+// the page's identity is defined by its URL, not the resolution
+// strategy used to render it.
+export async function generateMetadata({
+  params,
+}: {
+  params: Promise<{ framework: string; slug?: string[] }>;
+}): Promise<Metadata> {
+  const { framework, slug } = await params;
+  const slugTail = slug && slug.length > 0 ? `/${slug.join("/")}` : "";
+  return {
+    alternates: {
+      canonical: `${getBaseUrl()}/${framework}${slugTail}`,
+    },
+  };
+}
+
 export async function generateStaticParams() {
   // Rely on the catch-all's dynamic behaviour at runtime; returning an
   // empty array keeps build times short since there are ~17 frameworks

showcase/shell-docs/src/app/ag-ui/[[...slug]]/page.tsx

@@ -2,6 +2,7 @@ import React from "react";
 import fs from "fs";
 import path from "path";
 import matter from "gray-matter";
+import type { Metadata } from "next";
 import { notFound } from "next/navigation";
 import { MDXRemote } from "next-mdx-remote/rsc";
 import remarkGfm from "remark-gfm";
@@ -11,6 +12,23 @@ import { SidebarNav } from "@/components/sidebar-nav";
 import { docsComponents } from "@/lib/mdx-registry";
 import { stripLeadingImports } from "@/lib/docs-render";
 import { resolveWithinDir, safeReadFileSync } from "@/lib/safe-fs";
+import { getBaseUrl } from "@/lib/sitemap-helpers";
+
+// Self-canonical for /ag-ui[/<slug>]. AG-UI pages aren't per-framework
+// but get a canonical for parity with the rest of the docs surface.
+export async function generateMetadata({
+  params,
+}: {
+  params: Promise<{ slug?: string[] }>;
+}): Promise<Metadata> {
+  const { slug } = await params;
+  const slugTail = slug && slug.length > 0 ? `/${slug.join("/")}` : "";
+  return {
+    alternates: {
+      canonical: `${getBaseUrl()}/ag-ui${slugTail}`,
+    },
+  };
+}
 
 const CONTENT_DIR = path.join(process.cwd(), "src/content/ag-ui");
 

showcase/shell-docs/src/app/reference/[...slug]/page.tsx

@@ -1,3 +1,4 @@
+import type { Metadata } from "next";
 import Link from "next/link";
 import { notFound } from "next/navigation";
 import { MDXRemote } from "next-mdx-remote/rsc";
@@ -19,6 +20,23 @@ import {
 } from "@/lib/reference-items";
 import { stripLeadingImports } from "@/lib/docs-render";
 import { safeReadFileSync } from "@/lib/safe-fs";
+import { getBaseUrl } from "@/lib/sitemap-helpers";
+
+// Self-canonical for /reference/<slug>. Reference pages are not
+// per-framework, but we still emit a canonical so the production URL
+// is unambiguous and any future host aliases can't fragment indexing.
+export async function generateMetadata({
+  params,
+}: {
+  params: Promise<{ slug: string[] }>;
+}): Promise<Metadata> {
+  const { slug } = await params;
+  return {
+    alternates: {
+      canonical: `${getBaseUrl()}/reference/${slug.join("/")}`,
+    },
+  };
+}
 
 // next-mdx-remote components map
 const mdxComponents = {

showcase/shell-docs/src/app/robots.ts

@@ -0,0 +1,20 @@
+// Next.js App Router robots config. Allows all user-agents across the
+// public docs surface, blocks the internal `/api/` routes, and points
+// crawlers at sitemap.xml so they can discover every framework variant.
+
+import type { MetadataRoute } from "next";
+import { getBaseUrl } from "@/lib/sitemap-helpers";
+
+export default function robots(): MetadataRoute.Robots {
+  const baseUrl = getBaseUrl();
+  return {
+    rules: [
+      {
+        userAgent: "*",
+        allow: "/",
+        disallow: "/api/",
+      },
+    ],
+    sitemap: `${baseUrl}/sitemap.xml`,
+  };
+}

showcase/shell-docs/src/app/sitemap.ts

@@ -0,0 +1,108 @@
+// Next.js App Router sitemap. Emits one entry per (root URL, framework
+// variant) pair so search engines can crawl every framework-scoped doc
+// at its self-canonical URL.
+//
+// The expansion is:
+//   - Root URL (/)
+//   - Bare unscoped pages (/<slug>)            from src/content/docs/**.mdx
+//   - Framework-scoped pages (/<fw>/<slug>)    bare slugs × every registered
+//                                              integration, plus per-framework
+//                                              override pages under
+//                                              src/content/docs/integrations/
+//   - Reference (/reference/<slug>)            from src/content/reference/
+//   - AG-UI (/ag-ui/<slug>)                    from src/content/ag-ui/
+//
+// Each entry's `lastModified` is resolved via resolveLastModified —
+// frontmatter `lastmod` first, then file mtime, then `new Date()`.
+
+import type { MetadataRoute } from "next";
+import {
+  getAgUiPages,
+  getBareDocsPages,
+  getBaseUrl,
+  getFrameworkOverridePages,
+  getReferencePages,
+  resolveLastModified,
+} from "@/lib/sitemap-helpers";
+import { getDocsFolder, getIntegrations } from "@/lib/registry";
+
+export default function sitemap(): MetadataRoute.Sitemap {
+  const baseUrl = getBaseUrl();
+  const now = new Date();
+  const entries: MetadataRoute.Sitemap = [];
+
+  // 1. Root / overview.
+  entries.push({
+    url: `${baseUrl}/`,
+    lastModified: now,
+  });
+
+  // 2. Bare unscoped docs and 3. framework-scoped variants. Each bare
+  // slug generates one bare entry plus N framework-scoped entries — one
+  // per registered integration. Per-framework override pages are added
+  // alongside (deduped against the bare × framework cross-product).
+  const bareDocs = getBareDocsPages();
+  const integrations = getIntegrations();
+
+  // Track every framework-scoped URL we've already emitted so the
+  // override loop below can skip duplicates without scanning the array.
+  const seenFrameworkUrls = new Set<string>();
+
+  for (const { slug, filePath } of bareDocs) {
+    const lastModified = resolveLastModified(filePath);
+    entries.push({
+      url: `${baseUrl}/${slug}`,
+      lastModified,
+    });
+    for (const integration of integrations) {
+      const url = `${baseUrl}/${integration.slug}/${slug}`;
+      seenFrameworkUrls.add(url);
+      entries.push({ url, lastModified });
+    }
+  }
+
+  // Framework landing pages: /<framework> on its own.
+  for (const integration of integrations) {
+    const url = `${baseUrl}/${integration.slug}`;
+    seenFrameworkUrls.add(url);
+    entries.push({ url, lastModified: now });
+  }
+
+  // Per-framework override pages — topics that only exist under
+  // integrations/<folder>/. These are addressable as /<framework>/<slug>
+  // and aren't covered by the bare × framework cross-product above.
+  for (const integration of integrations) {
+    const folder = getDocsFolder(integration.slug);
+    for (const { slug, filePath } of getFrameworkOverridePages(folder)) {
+      const url = `${baseUrl}/${integration.slug}/${slug}`;
+      if (seenFrameworkUrls.has(url)) continue;
+      seenFrameworkUrls.add(url);
+      entries.push({
+        url,
+        lastModified: resolveLastModified(filePath),
+      });
+    }
+  }
+
+  // 4. Reference docs.
+  for (const { slug, filePath } of getReferencePages()) {
+    entries.push({
+      url: `${baseUrl}/reference/${slug}`,
+      lastModified: resolveLastModified(filePath),
+    });
+  }
+  // Reference index.
+  entries.push({ url: `${baseUrl}/reference`, lastModified: now });
+
+  // 5. AG-UI.
+  for (const { slug, filePath } of getAgUiPages()) {
+    entries.push({
+      url: `${baseUrl}/ag-ui/${slug}`,
+      lastModified: resolveLastModified(filePath),
+    });
+  }
+  // AG-UI overview landing.
+  entries.push({ url: `${baseUrl}/ag-ui`, lastModified: now });
+
+  return entries;
+}