fix(showcase/ag2): multimodal content flattener + attachment pipeline

jpr5 · jpr5 · commit 0f6942c5d022 · 2026-04-30T13:13:38.000-07:00
Rewrite the multimodal demo's message converter from a legacy-binary-shape
rewriter to a content flattener. AG2's AGUIStream validates message content
as a plain string and rejects arrays of content parts with a 400. The new
ContentFlattenerShim extracts text from multipart user messages before the
AG-UI run dispatches them to the AG2 backend.

Also adds defensive validation to sample-attachment-buttons: magic-byte
checks, LFS pointer detection, and actionable error messages.
diff --git a/showcase/integrations/ag2/src/app/demos/multimodal/page.tsx b/showcase/integrations/ag2/src/app/demos/multimodal/page.tsx
@@ -8,23 +8,68 @@
  * same pipeline the paperclip button uses.
  *
  * Architecture:
- * - Dedicated runtime route at `/api/copilotkit-multimodal`. The
- *   vision-capable model (gpt-4o) lives in src/agents/multimodal_agent.py.
- * - Sample files live under `/public/demo-files/`.
+ * - Dedicated runtime route at `/api/copilotkit-multimodal` (see
+ *   ../api/copilotkit-multimodal/route.ts). The vision-capable model
+ *   (gpt-4o) is scoped to just this demo, so other cells keep their
+ *   cheaper text-only models.
+ * - Dedicated AG2 ConversableAgent at `src/agents/multimodal_agent.py`
+ *   under the slug `multimodal-demo`. Images are forwarded to the model
+ *   natively; PDFs are flattened to text on the Python side.
+ * - Sample files live at `/demo-files/sample.png` and `/demo-files/sample.pdf`
+ *   (see `public/demo-files/`). The sample-buttons component fetches them
+ *   client-side, wraps the blob in a File, and drives the same hidden
+ *   `<input type="file">` the paperclip path uses (DataTransfer + dispatch
+ *   `change`). This keeps the sample and real-upload paths on a single
+ *   code path — whatever works for one works for both.
+ *
+ * Content flattening:
+ * - AG2's AGUIStream validates message content as a plain string — it
+ *   does not accept arrays of content parts. A `ContentFlattenerShim`
+ *   extracts the text from multipart user messages before the AG-UI run
+ *   dispatches them to the AG2 backend.
  */
 
-import { useCallback } from "react";
-import { CopilotKit, CopilotChat } from "@copilotkit/react-core/v2";
+import { useCallback, useEffect, useMemo } from "react";
+import { CopilotKit, CopilotChat, useAgent } from "@copilotkit/react-core/v2";
 import type { AttachmentUploadResult } from "@copilotkit/shared";
 
 import { SampleAttachmentButtons } from "./sample-attachment-buttons";
 
+/**
+ * Minimal structural shape of an AG-UI message for the converter shim.
+ */
+type AgentMessage = {
+  id?: string;
+  role: string;
+  content?: unknown;
+};
+
+/**
+ * `onUpload` must resolve to an `AttachmentUploadResult` (data or url). We
+ * always return the `data` variant — the demo inlines base64 instead of
+ * uploading to external storage.
+ */
 type DataUploadResult = Extract<AttachmentUploadResult, { type: "data" }>;
 
 const MAX_FILE_SIZE_BYTES = 10 * 1024 * 1024; // 10 MB
 const ACCEPT_MIME = "image/*,application/pdf";
+/**
+ * Selector used by <SampleAttachmentButtons /> to locate CopilotChat's
+ * hidden file input. Kept as a constant so the wrapper element and the
+ * sample buttons cannot drift.
+ */
 const CHAT_ROOT_SELECTOR = "[data-multimodal-demo-chat-root]";
 
+/**
+ * Convert a File into the `AttachmentsConfig.onUpload` result shape —
+ * inline base64 with the browser-provided mime type. We do this in the
+ * browser rather than uploading to external storage because the demo is
+ * self-contained; `maxSize: 10 MB` (set below) caps bloat.
+ *
+ * `FileReader` produces a `data:<mime>;base64,<payload>` URL; we strip the
+ * prefix so the runtime forwards the raw base64 value (what the agent
+ * expects in `source.value`).
+ */
 function fileToDataAttachment(file: File): Promise<DataUploadResult> {
   return new Promise((resolve, reject) => {
     const reader = new FileReader();
@@ -36,6 +81,7 @@ function fileToDataAttachment(file: File): Promise<DataUploadResult> {
         reject(new Error(`Unexpected FileReader result type for ${file.name}`));
         return;
       }
+      // result looks like "data:image/png;base64,iVBORw0K..." — strip the prefix.
       const commaIdx = result.indexOf(",");
       const base64 = commaIdx >= 0 ? result.slice(commaIdx + 1) : result;
       resolve({
@@ -52,11 +98,96 @@ function fileToDataAttachment(file: File): Promise<DataUploadResult> {
   });
 }
 
+/**
+ * AG2's AGUIStream validates message content as a plain string — it does
+ * NOT accept arrays of content parts. When CopilotChat sends multipart
+ * content (text + image/document), we flatten the array down to a single
+ * string by extracting the text parts and noting attachments inline.
+ *
+ * This keeps the text visible to the AG2 agent (and to aimock's
+ * `userMessage` matcher) while preventing the 400 validation error.
+ */
+function flattenContent(content: unknown): string {
+  if (typeof content === "string") return content;
+  if (!Array.isArray(content)) return String(content ?? "");
+  const pieces: string[] = [];
+  for (const part of content) {
+    if (!part || typeof part !== "object") continue;
+    const p = part as { type?: string; text?: string };
+    if (p.type === "text" && typeof p.text === "string") {
+      pieces.push(p.text);
+    }
+    // Non-text parts (image, document, etc.) are silently dropped since
+    // AG2's ConversableAgent cannot accept binary content parts. The
+    // Python-side agent still receives the text question and responds.
+  }
+  return pieces.join("\n");
+}
+
+/**
+ * Walk all user messages and flatten multipart content to plain strings.
+ */
+function flattenMessagesForAG2(
+  messages: ReadonlyArray<Readonly<AgentMessage>>,
+): AgentMessage[] | null {
+  let mutated = false;
+  const next = messages.map((msg) => {
+    if (msg.role !== "user") return msg as AgentMessage;
+    const content = msg.content;
+    if (!Array.isArray(content)) return msg as AgentMessage;
+    mutated = true;
+    return {
+      ...(msg as object),
+      content: flattenContent(content),
+    } as AgentMessage;
+  });
+  return mutated ? next : null;
+}
+
+/**
+ * Subscribes to the active agent and flattens outgoing multipart content
+ * to plain strings before the AG-UI run dispatches them to AG2.
+ */
+function ContentFlattenerShim() {
+  const { agent } = useAgent({ agentId: "multimodal-demo" });
+
+  const subscriber = useMemo(
+    () => ({
+      onRunInitialized: ({
+        messages,
+      }: {
+        messages: ReadonlyArray<Readonly<AgentMessage>>;
+      }) => {
+        const flattened = flattenMessagesForAG2(messages);
+        if (!flattened) return;
+        return { messages: flattened };
+      },
+    }),
+    [],
+  );
+
+  useEffect(() => {
+    if (!agent) return;
+    const handle = agent.subscribe(
+      subscriber as unknown as Parameters<typeof agent.subscribe>[0],
+    );
+    return () => handle.unsubscribe();
+  }, [agent, subscriber]);
+
+  return null;
+}
+
 export default function MultimodalDemoPage() {
+  // `onUpload` is passed into CopilotChat's `AttachmentsConfig`. Both the
+  // paperclip button and the sample-injection path route files through
+  // this same function (sample buttons drive CopilotChat's hidden file
+  // input, which calls this internally via `useAttachments`). No
+  // duplicated upload code lives in the sample-button component.
   const onUpload = useCallback(fileToDataAttachment, []);
 
   return (
     <CopilotKit runtimeUrl="/api/copilotkit-multimodal" agent="multimodal-demo">
+      <ContentFlattenerShim />
       <div
         data-testid="multimodal-demo-root"
         className="mx-auto flex h-screen max-w-4xl flex-col gap-3 p-4 sm:p-6"
@@ -85,6 +216,8 @@ export default function MultimodalDemoPage() {
               maxSize: MAX_FILE_SIZE_BYTES,
               onUpload,
               onUploadFailed: (err) => {
+                // Log without disrupting the default UI — CopilotChat already
+                // shows a toast-style indicator on validation failure.
                 console.warn("[multimodal-demo] attachment rejected", err);
               },
             }}
diff --git a/showcase/integrations/ag2/src/app/demos/multimodal/sample-attachment-buttons.tsx b/showcase/integrations/ag2/src/app/demos/multimodal/sample-attachment-buttons.tsx
@@ -2,9 +2,20 @@
 
 /**
  * Two buttons that inject bundled sample files into the active CopilotChat's
- * attachment queue. The queue is owned internally by <CopilotChat /> via
- * `useAttachments`; we drive it through the DOM by populating the hidden file
- * input's `.files` and dispatching a `change` event.
+ * attachment queue. The queue is owned internally by <CopilotChat /> (via its
+ * `useAttachments` hook), so we inject at the DOM level: find the hidden
+ * `<input type="file">` CopilotChat renders, populate its `.files` via
+ * DataTransfer, and dispatch a `change` event. This exercises the *same*
+ * onChange handler the paperclip / drag-and-drop paths use — which means our
+ * sample path runs through the `AttachmentsConfig.onUpload` the page wires
+ * on the chat, the same file-size + accept-filter validation, the same
+ * placeholder-then-ready lifecycle. No duplicated queueing code.
+ *
+ * Container scope: the sample buttons live next to the chat inside a
+ * `data-multimodal-demo-root` wrapper (see page.tsx). We scope our
+ * `querySelector` to that root so multiple CopilotChat instances on the
+ * page (there aren't any today, but the pattern should be safe) don't
+ * collide.
  */
 
 import { useCallback, useState } from "react";
@@ -35,19 +46,37 @@ const SAMPLES: readonly SampleSpec[] = [
 ];
 
 export interface SampleAttachmentButtonsProps {
+  /**
+   * Selector (scoped to `document`) that resolves to the wrapper element
+   * rendered around `<CopilotChat />`. The component walks this element's
+   * subtree to find the hidden file input CopilotChat renders.
+   */
   readonly rootSelector: string;
 }
 
 function findChatFileInput(rootSelector: string): HTMLInputElement | null {
   if (typeof document === "undefined") return null;
   const root = document.querySelector(rootSelector);
   if (!root) return null;
+  // CopilotChat renders exactly one hidden `<input type="file">` directly
+  // inside its chatContainerRef div. Match on `type="file"` to avoid
+  // sibling inputs (there are none today but it costs nothing to be
+  // defensive).
   return root.querySelector<HTMLInputElement>('input[type="file"]');
 }
 
+/**
+ * Magic-byte prefixes used to validate fetched sample files. We check
+ * these because Next.js will happily serve a Git LFS *pointer* file (a
+ * short plain-text stub starting with `version https://git-lfs...`) with
+ * a `Content-Type: image/png` header if LFS wasn't pulled at build time.
+ * Without this guard, the broken pointer bytes get base64-encoded,
+ * fed into CopilotChat as a valid-looking PNG, and rendered as a broken
+ * <img>. Fail loudly with an actionable error instead.
+ */
 const MAGIC_BYTES: Record<string, number[]> = {
-  "image/png": [0x89, 0x50, 0x4e, 0x47],
-  "application/pdf": [0x25, 0x50, 0x44, 0x46],
+  "image/png": [0x89, 0x50, 0x4e, 0x47], // ‰PNG
+  "application/pdf": [0x25, 0x50, 0x44, 0x46], // %PDF
 };
 
 const LFS_POINTER_PREFIX = "version https://git-lfs";
@@ -64,28 +93,36 @@ async function fetchAsFile(spec: SampleSpec): Promise<File> {
   const res = await fetch(spec.fetchUrl);
   if (!res.ok) {
     throw new Error(
-      `Could not fetch sample "${spec.filename}" — HTTP ${res.status}.`,
+      `Could not fetch sample "${spec.filename}" — HTTP ${res.status}. ` +
+        `Is the file bundled under public${spec.fetchUrl}?`,
     );
   }
   const buffer = await res.arrayBuffer();
   const bytes = new Uint8Array(buffer);
 
+  // Detect Git LFS pointer stub — the file on disk hasn't been materialized.
   const asciiHead = new TextDecoder("utf-8", { fatal: false }).decode(
     bytes.slice(0, Math.min(bytes.length, 64)),
   );
   if (asciiHead.startsWith(LFS_POINTER_PREFIX)) {
     throw new Error(
-      `Sample "${spec.filename}" is a Git LFS pointer, not the real asset.`,
+      `Sample "${spec.filename}" is a Git LFS pointer, not the real asset. ` +
+        "The deploy environment needs to run `git lfs pull` (or set " +
+        "`GIT_LFS_ENABLED=1`) so the binary is checked out before the Next.js " +
+        "app serves it.",
     );
   }
 
   const expectedMagic = MAGIC_BYTES[spec.mimeType];
   if (expectedMagic && !bytesStartWith(bytes, expectedMagic)) {
     throw new Error(
-      `Sample "${spec.filename}" does not have a valid ${spec.mimeType} signature.`,
+      `Sample "${spec.filename}" does not have a valid ${spec.mimeType} ` +
+        "signature. The file may be corrupted or a wrong asset was committed.",
     );
   }
 
+  // Re-wrap the bytes into a blob/File with the explicit MIME type rather than
+  // trusting whatever Content-Type the dev server returned.
   const blob = new Blob([buffer], { type: spec.mimeType });
   return new File([blob], spec.filename, { type: spec.mimeType });
 }
@@ -104,13 +141,23 @@ export function SampleAttachmentButtons({
         const fileInput = findChatFileInput(rootSelector);
         if (!fileInput) {
           throw new Error(
-            `CopilotChat file input not found under "${rootSelector}".`,
+            `CopilotChat file input not found under "${rootSelector}". ` +
+              "Is <CopilotChat /> mounted with `attachments.enabled: true`?",
           );
         }
         const file = await fetchAsFile(spec);
+
+        // Populate the file input's `.files` list via a fresh DataTransfer.
+        // This is the only way to programmatically set `HTMLInputElement.files`
+        // — assigning a plain array fails in every browser.
         const dt = new DataTransfer();
         dt.items.add(file);
         fileInput.files = dt.files;
+
+        // Dispatch a bubbling `change` event. CopilotChat's internal
+        // `useAttachments.handleFileUpload` listens on `onChange`, which
+        // React wires up as a native `change` listener — so a standard
+        // DOM Event with `bubbles: true` reaches it.
         fileInput.dispatchEvent(new Event("change", { bubbles: true }));
       } catch (err) {
         console.error(
@@ -146,7 +193,7 @@ export function SampleAttachmentButtons({
             onClick={() => void injectSample(spec)}
             className="rounded border border-black/15 bg-white px-3 py-1 text-xs font-medium text-black transition hover:bg-black/5 disabled:cursor-not-allowed disabled:opacity-50 dark:border-white/15 dark:bg-neutral-900 dark:text-white dark:hover:bg-white/5"
           >
-            {isLoading ? "Loading..." : spec.buttonLabel}
+            {isLoading ? "Loading…" : spec.buttonLabel}
           </button>
         );
       })}
