Skip to content

Commit ff236cf

Browse files
AlemTuzlakAlemTuzlak
committed
feat(react-ui): add AttachmentRenderer and update UserMessage for multimodal content
1 parent c472f45 commit ff236cf

3 files changed

Lines changed: 161 additions & 21 deletions

File tree

‎packages/react-ui/src/components/chat/AttachmentRenderer.tsx‎

Lines changed: 100 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,100 @@
1+
import React, { useState } from "react";
2+
import type { InputContentSource } from "@copilotkit/shared";
3+
4+
interface AttachmentRendererProps {
5+
type: "image" | "audio" | "video" | "document";
6+
source: InputContentSource;
7+
content?: string;
8+
className?: string;
9+
}
10+
11+
export const AttachmentRenderer: React.FC<AttachmentRendererProps> = ({
12+
type,
13+
source,
14+
content,
15+
className = "",
16+
}) => {
17+
const src = getSourceUrl(source);
18+
19+
switch (type) {
20+
case "image":
21+
return <ImageAttachment src={src} content={content} className={className} />;
22+
case "audio":
23+
return (
24+
<div className={`copilotKitAttachment copilotKitAttachmentAudio ${className}`}>
25+
<audio src={src} controls preload="metadata" />
26+
</div>
27+
);
28+
case "video":
29+
return (
30+
<div className={`copilotKitAttachment copilotKitAttachmentVideo ${className}`}>
31+
<video src={src} controls preload="metadata" />
32+
</div>
33+
);
34+
case "document":
35+
return (
36+
<div className={`copilotKitAttachment copilotKitAttachmentDocument ${className}`}>
37+
<div className="copilotKitAttachmentDocIcon">
38+
{getDocumentIcon(source.mimeType ?? "")}
39+
</div>
40+
<div className="copilotKitAttachmentDocInfo">
41+
<span className="copilotKitAttachmentDocName">
42+
{content || "Document"}
43+
</span>
44+
<span className="copilotKitAttachmentDocMeta">
45+
{source.mimeType || "Unknown type"}
46+
</span>
47+
</div>
48+
</div>
49+
);
50+
}
51+
};
52+
53+
function ImageAttachment({
54+
src,
55+
content,
56+
className,
57+
}: {
58+
src: string;
59+
content?: string;
60+
className: string;
61+
}) {
62+
const [error, setError] = useState(false);
63+
64+
if (error) {
65+
return (
66+
<div className={`copilotKitImageRendering copilotKitImageRenderingError ${className}`}>
67+
<div className="copilotKitImageRenderingErrorMessage">Failed to load image</div>
68+
{content && <div className="copilotKitImageRenderingContent">{content}</div>}
69+
</div>
70+
);
71+
}
72+
73+
return (
74+
<div className={`copilotKitImageRendering ${className}`}>
75+
<img
76+
src={src}
77+
alt={content || "Image attachment"}
78+
className="copilotKitImageRenderingImage"
79+
onError={() => setError(true)}
80+
/>
81+
{content && <div className="copilotKitImageRenderingContent">{content}</div>}
82+
</div>
83+
);
84+
}
85+
86+
function getSourceUrl(source: InputContentSource): string {
87+
if (source.type === "url") {
88+
return source.value;
89+
}
90+
return `data:${source.mimeType};base64,${source.value}`;
91+
}
92+
93+
function getDocumentIcon(mimeType: string): string {
94+
if (mimeType.includes("pdf")) return "PDF";
95+
if (mimeType.includes("word") || mimeType.includes("document")) return "DOC";
96+
if (mimeType.includes("sheet") || mimeType.includes("excel")) return "XLS";
97+
if (mimeType.includes("presentation") || mimeType.includes("powerpoint")) return "PPT";
98+
if (mimeType.includes("text/")) return "TXT";
99+
return "FILE";
100+
}

‎packages/react-ui/src/components/chat/messages/ImageRenderer.tsx‎

Lines changed: 17 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,32 @@
11
import React, { useState } from "react";
22
import { ImageRendererProps } from "../props";
33

4-
/**
5-
* Default image rendering component that can be customized by users.
6-
* Uses CSS classes for styling so users can override styles.
7-
*/
84
export const ImageRenderer: React.FC<ImageRendererProps> = ({
95
image,
6+
source,
107
content,
118
className = "",
129
}) => {
1310
const [imageError, setImageError] = useState(false);
14-
const imageSrc = `data:image/${image.format};base64,${image.bytes}`;
15-
const altText = content || "User uploaded image";
1611

17-
const handleImageError = () => {
18-
setImageError(true);
19-
};
12+
// Determine image src from either legacy ImageData or new InputContentSource
13+
let imageSrc: string;
14+
if (source) {
15+
imageSrc =
16+
source.type === "url"
17+
? source.value
18+
: `data:${source.mimeType};base64,${source.value}`;
19+
} else if (image) {
20+
imageSrc = `data:image/${image.format};base64,${image.bytes}`;
21+
} else {
22+
return null;
23+
}
24+
25+
const altText = content || "User uploaded image";
2026

2127
if (imageError) {
2228
return (
23-
<div
24-
className={`copilotKitImageRendering copilotKitImageRenderingError ${className}`}
25-
>
29+
<div className={`copilotKitImageRendering copilotKitImageRenderingError ${className}`}>
2630
<div className="copilotKitImageRenderingErrorMessage">
2731
Failed to load image
2832
</div>
@@ -39,7 +43,7 @@ export const ImageRenderer: React.FC<ImageRendererProps> = ({
3943
src={imageSrc}
4044
alt={altText}
4145
className="copilotKitImageRenderingImage"
42-
onError={handleImageError}
46+
onError={() => setImageError(true)}
4347
/>
4448
{content && (
4549
<div className="copilotKitImageRenderingContent">{content}</div>

‎packages/react-ui/src/components/chat/messages/UserMessage.tsx‎

Lines changed: 44 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
import { UserMessageProps } from "../props";
2+
import { AttachmentRenderer } from "../AttachmentRenderer";
23

34
type UserMessageContent = NonNullable<UserMessageProps["message"]>["content"];
45

@@ -30,25 +31,60 @@ const getTextContent = (
3031
);
3132
};
3233

34+
const getMediaParts = (content: UserMessageContent | undefined) => {
35+
if (!content || typeof content === "string") return [];
36+
37+
return content.filter(
38+
(part) =>
39+
part.type === "image" ||
40+
part.type === "audio" ||
41+
part.type === "video" ||
42+
part.type === "document",
43+
) as Array<{
44+
type: "image" | "audio" | "video" | "document";
45+
source: { type: "data"; value: string; mimeType: string } | { type: "url"; value: string; mimeType?: string };
46+
}>;
47+
};
48+
3349
export const UserMessage = (props: UserMessageProps) => {
3450
const { message, ImageRenderer } = props;
35-
const isImageMessage =
36-
message && "image" in message && Boolean(message.image);
51+
const content = message?.content;
3752

38-
if (isImageMessage) {
39-
const imageMessage = message!;
40-
const content = getTextContent(imageMessage?.content);
53+
// Legacy path: old-style image field on message
54+
const isLegacyImageMessage =
55+
message && "image" in message && Boolean((message as any).image);
4156

57+
if (isLegacyImageMessage) {
58+
const legacyImage = (message as any).image;
59+
const textContent = getTextContent(content);
4260
return (
4361
<div className="copilotKitMessage copilotKitUserMessage">
44-
<ImageRenderer image={imageMessage.image!} content={content} />
62+
<ImageRenderer image={legacyImage} content={textContent} />
4563
</div>
4664
);
4765
}
4866

49-
const content = getTextContent(message?.content);
67+
const textContent = getTextContent(content);
68+
const mediaParts = getMediaParts(content);
69+
70+
if (mediaParts.length === 0) {
71+
return (
72+
<div className="copilotKitMessage copilotKitUserMessage">
73+
{textContent}
74+
</div>
75+
);
76+
}
5077

5178
return (
52-
<div className="copilotKitMessage copilotKitUserMessage">{content}</div>
79+
<div className="copilotKitMessage copilotKitUserMessage">
80+
{textContent && <div>{textContent}</div>}
81+
{mediaParts.map((part, index) => (
82+
<AttachmentRenderer
83+
key={index}
84+
type={part.type}
85+
source={part.source}
86+
/>
87+
))}
88+
</div>
5389
);
5490
};

0 commit comments

Comments
 (0)