refactor(showcase/shell-docs): reference pages share loadItems helper + recursive static params + guarded reads

- Extract duplicated loadItems/getAllItems into @/lib/reference-items so the
  /reference index page and the /reference/[...slug] page read the same
  tree the same way (same subdirs, same walker, same gray-matter path,
  same caching).
- Walker is now recursive: subfolder files like components/inputs/textarea.mdx
  are indexed and statically generated. Previously only top-level .mdx
  files under components/ and hooks/ were picked up.
- Wrap gray-matter and fs reads in try/catch per file: a single malformed
  frontmatter block no longer crashes the whole static-generation pass —
  we log the offending path and skip that file.
- Fence-aware import stripper in the slug page so code samples containing
  import ... lines inside fences are not corrupted. (Duplicated inline
  here and in ag-ui page with a TODO(dedup) marker pointing at a future
  shared helper in @/lib/docs-render.)
- loadReferenceItems memoizes in production (module-scope cache keyed by
  subdir). Dev bypasses the cache so edits show up without a server
  restart.

Author: jpr5

showcase/shell-docs/src/app/reference/[...slug]/page.tsx

@@ -13,30 +13,11 @@ import {
   Accordion,
 } from "@/components/mdx-components";
 import { SidebarNav } from "@/components/sidebar-nav";
-
-const CONTENT_DIR = path.join(process.cwd(), "src/content/reference");
-
-type NavItem = { slug: string; title: string; category: string };
-
-function getAllItems(): NavItem[] {
-  const items: NavItem[] = [];
-
-  for (const subdir of ["components", "hooks"]) {
-    const dir = path.join(CONTENT_DIR, subdir);
-    if (!fs.existsSync(dir)) continue;
-    for (const f of fs.readdirSync(dir).filter((f) => f.endsWith(".mdx"))) {
-      const raw = fs.readFileSync(path.join(dir, f), "utf-8");
-      const { data } = matter(raw);
-      items.push({
-        slug: `${subdir}/${f.replace(/\.mdx$/, "")}`,
-        title: (data.title as string) || f.replace(/\.mdx$/, ""),
-        category: subdir === "components" ? "Components" : "Hooks",
-      });
-    }
-  }
-
-  return items;
-}
+import {
+  REFERENCE_CONTENT_DIR,
+  loadAllReferenceItems,
+  referenceStaticParams,
+} from "@/lib/reference-items";
 
 // next-mdx-remote components map
 const mdxComponents = {
@@ -49,18 +30,38 @@ const mdxComponents = {
   // Strip unknown imports — MDX import statements become no-ops in next-mdx-remote
 };
 
-export function generateStaticParams() {
-  const params: { slug: string[] }[] = [];
-
-  for (const subdir of ["components", "hooks"]) {
-    const dir = path.join(CONTENT_DIR, subdir);
-    if (!fs.existsSync(dir)) continue;
-    for (const f of fs.readdirSync(dir).filter((f) => f.endsWith(".mdx"))) {
-      params.push({ slug: [subdir, f.replace(/\.mdx$/, "")] });
+// Strip leading `import …` lines from the top of an MDX source without
+// touching `import …` lines that appear inside fenced code blocks. The
+// previous implementation filtered any line matching /^import\s+/, which
+// silently mangled doc code samples like `import os` inside Python fences.
+// TODO(dedup): hoist into a shared helper once the ag-ui page and this
+// one both import it (both already have near-identical copies).
+function stripImportsFenceAware(source: string): string {
+  const lines = source.split("\n");
+  const out: string[] = [];
+  let inFence = false;
+  let fenceMarker = "";
+  for (const line of lines) {
+    const fenceMatch = line.match(/^\s*(```+|~~~+)/);
+    if (fenceMatch) {
+      if (!inFence) {
+        inFence = true;
+        fenceMarker = fenceMatch[1];
+      } else if (line.trim().startsWith(fenceMarker)) {
+        inFence = false;
+        fenceMarker = "";
+      }
+      out.push(line);
+      continue;
     }
+    if (!inFence && /^import\s+/.test(line)) continue;
+    out.push(line);
   }
+  return out.join("\n");
+}
 
-  return params;
+export function generateStaticParams() {
+  return referenceStaticParams();
 }
 
 export default async function ReferenceSlugPage({
@@ -70,24 +71,43 @@ export default async function ReferenceSlugPage({
 }) {
   const { slug } = await params;
   const slugPath = slug.join("/");
-  const filePath = path.join(CONTENT_DIR, `${slugPath}.mdx`);
+  const filePath = path.join(REFERENCE_CONTENT_DIR, `${slugPath}.mdx`);
 
   if (!fs.existsSync(filePath)) {
     notFound();
   }
 
-  const raw = fs.readFileSync(filePath, "utf-8");
-  const { content, data } = matter(raw);
+  let raw: string;
+  try {
+    raw = fs.readFileSync(filePath, "utf-8");
+  } catch (err) {
+    console.error(`[reference] Failed to read ${filePath}:`, err);
+    notFound();
+  }
+
+  let content = "";
+  let data: Record<string, unknown> = {};
+  try {
+    const parsed = matter(raw);
+    content = parsed.content;
+    data = parsed.data;
+  } catch (err) {
+    console.error(
+      `[reference] Failed to parse frontmatter in ${filePath}:`,
+      err,
+    );
+    notFound();
+  }
 
-  // Strip import lines — next-mdx-remote doesn't support them
-  const cleanedContent = content
-    .split("\n")
-    .filter((line) => !line.match(/^import\s+/))
-    .join("\n");
+  const cleanedContent = stripImportsFenceAware(content);
 
-  const allItems = getAllItems();
-  const title = (data.title as string) || slug[slug.length - 1];
-  const description = data.description as string | undefined;
+  const allItems = loadAllReferenceItems();
+  const title =
+    typeof data.title === "string" && data.title.length > 0
+      ? data.title
+      : slug[slug.length - 1];
+  const description =
+    typeof data.description === "string" ? data.description : undefined;
 
   return (
     <div className="flex min-h-[calc(100vh-53px)]">

showcase/shell-docs/src/app/reference/page.tsx

@@ -2,40 +2,32 @@ import Link from "next/link";
 import fs from "fs";
 import path from "path";
 import matter from "gray-matter";
-
-const CONTENT_DIR = path.join(process.cwd(), "src/content/reference");
-
-type RefItem = { slug: string; title: string; description?: string };
-
-function loadItems(subdir: string): RefItem[] {
-  const dir = path.join(CONTENT_DIR, subdir);
-  if (!fs.existsSync(dir)) return [];
-  return fs
-    .readdirSync(dir)
-    .filter((f) => f.endsWith(".mdx"))
-    .map((f) => {
-      const raw = fs.readFileSync(path.join(dir, f), "utf-8");
-      const { data } = matter(raw);
-      return {
-        slug: `${subdir}/${f.replace(/\.mdx$/, "")}`,
-        title: (data.title as string) || f.replace(/\.mdx$/, ""),
-        description: data.description as string | undefined,
-      };
-    });
-}
+import {
+  REFERENCE_CONTENT_DIR,
+  loadReferenceItems,
+} from "@/lib/reference-items";
 
 export default function ReferencePage() {
-  const components = loadItems("components");
-  const hooks = loadItems("hooks");
+  const components = loadReferenceItems("components");
+  const hooks = loadReferenceItems("hooks");
 
-  // Also load the index page frontmatter for the intro
-  let intro = "";
-  const indexPath = path.join(CONTENT_DIR, "index.mdx");
+  // Also load the index page frontmatter for the intro. Guarded so a
+  // malformed frontmatter block falls back to a default rather than
+  // crashing the whole index page.
+  let intro = "API Reference for the next-generation CopilotKit React API.";
+  const indexPath = path.join(REFERENCE_CONTENT_DIR, "index.mdx");
   if (fs.existsSync(indexPath)) {
-    const { data } = matter(fs.readFileSync(indexPath, "utf-8"));
-    intro =
-      (data.description as string) ||
-      "API Reference for the next-generation CopilotKit React API.";
+    try {
+      const { data } = matter(fs.readFileSync(indexPath, "utf-8"));
+      if (typeof data.description === "string" && data.description.length > 0) {
+        intro = data.description;
+      }
+    } catch (err) {
+      console.error(
+        `[reference] Failed to parse frontmatter in ${indexPath}:`,
+        err,
+      );
+    }
   }
 
   return (

showcase/shell-docs/src/lib/reference-items.ts

@@ -0,0 +1,132 @@
+// Shared helpers for walking the `src/content/reference/` tree. Used by
+// both /reference (index page) and /reference/[...slug] so the two stay
+// in sync: same subdirs, same recursive traversal, same gray-matter
+// handling, same caching behavior.
+
+import fs from "fs";
+import path from "path";
+import matter from "gray-matter";
+
+export const REFERENCE_CONTENT_DIR = path.join(
+  process.cwd(),
+  "src/content/reference",
+);
+
+// Top-level reference categories we index. Anything outside this list is
+// ignored (e.g. a stray snippet file at the root).
+export const REFERENCE_SUBDIRS = ["components", "hooks"] as const;
+export type ReferenceSubdir = (typeof REFERENCE_SUBDIRS)[number];
+
+export type ReferenceItem = {
+  /** subdir-relative slug, e.g. `components/chat` or `components/inputs/textarea`. */
+  slug: string;
+  title: string;
+  description?: string;
+  category: "Components" | "Hooks";
+};
+
+function categoryFor(subdir: ReferenceSubdir): "Components" | "Hooks" {
+  return subdir === "components" ? "Components" : "Hooks";
+}
+
+/**
+ * Recursively collect all `.mdx` files under `dir` and return their paths
+ * relative to `dir` (without the `.mdx` extension). Silently skips
+ * unreadable subdirectories so a single EACCES doesn't break the build.
+ */
+function walkMdx(dir: string, prefix: string = ""): string[] {
+  let entries: fs.Dirent[];
+  try {
+    entries = fs.readdirSync(dir, { withFileTypes: true });
+  } catch (err) {
+    console.error(`[reference-items] Failed to read dir ${dir}:`, err);
+    return [];
+  }
+  const out: string[] = [];
+  for (const entry of entries) {
+    if (entry.name.startsWith(".")) continue;
+    const childAbs = path.join(dir, entry.name);
+    const childRel = prefix ? `${prefix}/${entry.name}` : entry.name;
+    if (entry.isDirectory()) {
+      out.push(...walkMdx(childAbs, childRel));
+    } else if (entry.isFile() && entry.name.endsWith(".mdx")) {
+      out.push(childRel.replace(/\.mdx$/, ""));
+    }
+  }
+  return out;
+}
+
+/**
+ * Load items from a single reference subdir, recursing into subfolders.
+ * Malformed frontmatter on any file is logged and skipped — we never
+ * crash the whole index just because one page has a bad YAML block.
+ */
+function loadSubdirItems(subdir: ReferenceSubdir): ReferenceItem[] {
+  const dir = path.join(REFERENCE_CONTENT_DIR, subdir);
+  if (!fs.existsSync(dir)) return [];
+
+  const items: ReferenceItem[] = [];
+  for (const relSlug of walkMdx(dir)) {
+    const filePath = path.join(dir, `${relSlug}.mdx`);
+    let raw: string;
+    try {
+      raw = fs.readFileSync(filePath, "utf-8");
+    } catch (err) {
+      console.error(`[reference-items] Failed to read ${filePath}:`, err);
+      continue;
+    }
+    let data: Record<string, unknown> = {};
+    try {
+      ({ data } = matter(raw));
+    } catch (err) {
+      console.error(
+        `[reference-items] Failed to parse frontmatter in ${filePath}:`,
+        err,
+      );
+      continue;
+    }
+    const fallbackTitle = relSlug.split("/").pop() ?? relSlug;
+    items.push({
+      slug: `${subdir}/${relSlug}`,
+      title:
+        typeof data.title === "string" && data.title.length > 0
+          ? data.title
+          : fallbackTitle,
+      description:
+        typeof data.description === "string" ? data.description : undefined,
+      category: categoryFor(subdir),
+    });
+  }
+  return items;
+}
+
+// In-memory cache — keyed by subdir, rebuilt once per process in prod. In
+// dev we skip the cache so MDX edits show up without a server restart.
+const __itemsCache = new Map<ReferenceSubdir, ReferenceItem[]>();
+function isProd(): boolean {
+  return process.env.NODE_ENV === "production";
+}
+
+export function loadReferenceItems(subdir: ReferenceSubdir): ReferenceItem[] {
+  if (isProd()) {
+    const cached = __itemsCache.get(subdir);
+    if (cached) return cached;
+  }
+  const items = loadSubdirItems(subdir);
+  if (isProd()) __itemsCache.set(subdir, items);
+  return items;
+}
+
+export function loadAllReferenceItems(): ReferenceItem[] {
+  return REFERENCE_SUBDIRS.flatMap((s) => loadReferenceItems(s));
+}
+
+/**
+ * For `generateStaticParams`: return every reference page as its Next.js
+ * catch-all slug array. Recursive (unlike the previous one-level-only
+ * implementation), so subfolder docs like `components/inputs/textarea`
+ * are statically generated too.
+ */
+export function referenceStaticParams(): { slug: string[] }[] {
+  return loadAllReferenceItems().map((item) => ({ slug: item.slug.split("/") }));
+}