Skip to content

Commit c64606d

Browse files
mmemme
committed
Merge branch 'main' into feat/runtime-remote-actions
2 parents a7baaa8 + fdbd2b8 commit c64606d

12 files changed

Lines changed: 671 additions & 68 deletions

File tree

‎CopilotKit/examples/next-openai/src/app/api/copilotkit/anthropic/route.ts‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,20 @@
1+
import {
2+
CopilotRuntime,
3+
AnthropicAdapter,
4+
copilotRuntimeNextJSAppRouterEndpoint,
5+
} from "@copilotkit/runtime";
6+
import { NextRequest } from "next/server";
7+
8+
export const POST = async (req: NextRequest) => {
9+
const runtime = new CopilotRuntime({});
10+
11+
const serviceAdapter = new AnthropicAdapter();
12+
13+
const { handleRequest } = copilotRuntimeNextJSAppRouterEndpoint({
14+
runtime,
15+
serviceAdapter,
16+
endpoint: req.nextUrl.pathname,
17+
});
18+
19+
return handleRequest(req);
20+
};

‎CopilotKit/examples/next-openai/src/app/presentation/page.tsx‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ export default function AIPresentation() {
1010

1111
return (
1212
<CopilotKit
13-
runtimeUrl="/api/copilotkit"
13+
runtimeUrl="/api/copilotkit/anthropic"
1414
transcribeAudioUrl="/api/transcribe"
1515
textToSpeechUrl="/api/tts"
1616
>

‎CopilotKit/packages/runtime/package.json‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@
4343
"typescript": "^5.2.3"
4444
},
4545
"dependencies": {
46+
"@anthropic-ai/sdk": "^0.27.3",
4647
"@copilotkit/shared": "workspace:*",
4748
"@google/generative-ai": "^0.11.2",
4849
"@graphql-yoga/plugin-defer-stream": "^3.3.1",

‎CopilotKit/packages/runtime/src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
11
import "reflect-metadata";
22
export * from "./lib";
33
export * from "./utils";
4+
export * from "./service-adapters";

‎CopilotKit/packages/runtime/src/service-adapters/anthropic/anthropic-adapter.ts‎

Lines changed: 197 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,197 @@
1+
/**
2+
* Copilot Runtime adapter for Anthropic.
3+
*
4+
* ## Example
5+
*
6+
* ```ts
7+
* import { CopilotRuntime, AnthropicAdapter } from "@copilotkit/runtime";
8+
* import Anthropic from "@anthropic-ai/sdk";
9+
*
10+
* const copilotKit = new CopilotRuntime();
11+
*
12+
* const anthropic = new Anthropic({
13+
* apiKey: "<your-api-key>",
14+
* });
15+
*
16+
* const serviceAdapter = new AnthropicAdapter({ anthropic });
17+
*
18+
* return copilotKit.streamHttpServerResponse(req, res, serviceAdapter);
19+
* ```
20+
*/
21+
import Anthropic from "@anthropic-ai/sdk";
22+
import {
23+
CopilotServiceAdapter,
24+
CopilotRuntimeChatCompletionRequest,
25+
CopilotRuntimeChatCompletionResponse,
26+
} from "../service-adapter";
27+
import {
28+
convertActionInputToAnthropicTool,
29+
convertMessageToAnthropicMessage,
30+
groupAnthropicMessagesByRole,
31+
limitMessagesToTokenCount,
32+
} from "./utils";
33+
34+
import { randomId } from "@copilotkit/shared";
35+
import { TextMessage } from "../../graphql/types/converted";
36+
37+
const DEFAULT_MODEL = "claude-3-opus-20240229";
38+
39+
export interface AnthropicAdapterParams {
40+
/**
41+
* An optional Anthropic instance to use. If not provided, a new instance will be
42+
* created.
43+
*/
44+
anthropic?: Anthropic;
45+
46+
/**
47+
* The model to use.
48+
*/
49+
model?: string;
50+
}
51+
52+
export class AnthropicAdapter implements CopilotServiceAdapter {
53+
private model: string = DEFAULT_MODEL;
54+
55+
private _anthropic: Anthropic;
56+
public get anthropic(): Anthropic {
57+
return this._anthropic;
58+
}
59+
60+
constructor(params?: AnthropicAdapterParams) {
61+
this._anthropic = params?.anthropic || new Anthropic({});
62+
if (params?.model) {
63+
this.model = params.model;
64+
}
65+
}
66+
67+
async process(
68+
request: CopilotRuntimeChatCompletionRequest,
69+
): Promise<CopilotRuntimeChatCompletionResponse> {
70+
const {
71+
threadId,
72+
model = this.model,
73+
messages: rawMessages,
74+
actions,
75+
eventSource,
76+
forwardedParameters,
77+
} = request;
78+
const tools = actions.map(convertActionInputToAnthropicTool);
79+
80+
const messages = [...rawMessages];
81+
82+
// get the instruction message
83+
const instructionsMessage = messages.shift();
84+
const instructions =
85+
instructionsMessage instanceof TextMessage ? instructionsMessage.content : "";
86+
87+
let anthropicMessages = messages.map(convertMessageToAnthropicMessage);
88+
anthropicMessages = limitMessagesToTokenCount(anthropicMessages, tools, model);
89+
anthropicMessages = groupAnthropicMessagesByRole(anthropicMessages);
90+
91+
let toolChoice: any = forwardedParameters?.toolChoice;
92+
if (forwardedParameters?.toolChoice === "function") {
93+
toolChoice = {
94+
type: "tool",
95+
name: forwardedParameters.toolChoiceFunctionName,
96+
};
97+
}
98+
99+
const stream = this.anthropic.messages.create({
100+
system: instructions,
101+
model: this.model,
102+
messages: anthropicMessages,
103+
max_tokens: forwardedParameters?.maxTokens || 1024,
104+
...(tools.length > 0 && { tools }),
105+
...(toolChoice && { tool_choice: toolChoice }),
106+
stream: true,
107+
});
108+
109+
eventSource.stream(async (eventStream$) => {
110+
let mode: "function" | "message" | null = null;
111+
let didOutputText = false;
112+
let currentMessageId = randomId();
113+
let currentToolCallId = randomId();
114+
let filterThinkingTextBuffer = new FilterThinkingTextBuffer();
115+
116+
for await (const chunk of await stream) {
117+
if (chunk.type === "message_start") {
118+
currentMessageId = chunk.message.id;
119+
} else if (chunk.type === "content_block_start") {
120+
if (chunk.content_block.type === "text") {
121+
didOutputText = false;
122+
filterThinkingTextBuffer.reset();
123+
mode = "message";
124+
} else if (chunk.content_block.type === "tool_use") {
125+
currentToolCallId = chunk.content_block.id;
126+
eventStream$.sendActionExecutionStart(currentToolCallId, chunk.content_block.name);
127+
mode = "function";
128+
}
129+
} else if (chunk.type === "content_block_delta") {
130+
if (chunk.delta.type === "text_delta") {
131+
const text = filterThinkingTextBuffer.onTextChunk(chunk.delta.text);
132+
if (text.length > 0) {
133+
if (!didOutputText) {
134+
eventStream$.sendTextMessageStart(currentMessageId);
135+
didOutputText = true;
136+
}
137+
eventStream$.sendTextMessageContent(text);
138+
}
139+
} else if (chunk.delta.type === "input_json_delta") {
140+
eventStream$.sendActionExecutionArgs(chunk.delta.partial_json);
141+
}
142+
} else if (chunk.type === "content_block_stop") {
143+
if (mode === "message") {
144+
if (didOutputText) {
145+
eventStream$.sendTextMessageEnd();
146+
}
147+
} else if (mode === "function") {
148+
eventStream$.sendActionExecutionEnd();
149+
}
150+
}
151+
}
152+
153+
eventStream$.complete();
154+
});
155+
156+
return {
157+
threadId: threadId || randomId(),
158+
};
159+
}
160+
}
161+
162+
const THINKING_TAG = "<thinking>";
163+
const THINKING_TAG_END = "</thinking>";
164+
165+
class FilterThinkingTextBuffer {
166+
private buffer: string;
167+
private didFilterThinkingTag: boolean = false;
168+
169+
constructor() {
170+
this.buffer = "";
171+
}
172+
173+
onTextChunk(text: string): string {
174+
this.buffer += text;
175+
if (this.didFilterThinkingTag) {
176+
return text;
177+
}
178+
const potentialTag = this.buffer.slice(0, THINKING_TAG.length);
179+
if (THINKING_TAG.startsWith(potentialTag)) {
180+
if (this.buffer.includes(THINKING_TAG_END)) {
181+
const end = this.buffer.indexOf(THINKING_TAG_END);
182+
const filteredText = this.buffer.slice(end + THINKING_TAG_END.length);
183+
this.buffer = filteredText;
184+
this.didFilterThinkingTag = true;
185+
return filteredText;
186+
} else {
187+
return "";
188+
}
189+
}
190+
return text;
191+
}
192+
193+
reset() {
194+
this.buffer = "";
195+
this.didFilterThinkingTag = false;
196+
}
197+
}

‎CopilotKit/packages/runtime/src/service-adapters/anthropic/utils.ts‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
import {
2+
ActionExecutionMessage,
3+
Message,
4+
ResultMessage,
5+
TextMessage,
6+
} from "../../graphql/types/converted";
7+
import { ActionInput } from "../../graphql/inputs/action.input";
8+
import { Anthropic } from "@anthropic-ai/sdk";
9+
10+
export function limitMessagesToTokenCount(
11+
messages: any[],
12+
tools: any[],
13+
model: string,
14+
maxTokens?: number,
15+
): any[] {
16+
maxTokens ||= MAX_TOKENS;
17+
18+
const result: any[] = [];
19+
const toolsNumTokens = countToolsTokens(model, tools);
20+
if (toolsNumTokens > maxTokens) {
21+
throw new Error(`Too many tokens in function definitions: ${toolsNumTokens} > ${maxTokens}`);
22+
}
23+
maxTokens -= toolsNumTokens;
24+
25+
for (const message of messages) {
26+
if (message.role === "system") {
27+
const numTokens = countMessageTokens(model, message);
28+
maxTokens -= numTokens;
29+
30+
if (maxTokens < 0) {
31+
throw new Error("Not enough tokens for system message.");
32+
}
33+
}
34+
}
35+
36+
let cutoff: boolean = false;
37+
38+
const reversedMessages = [...messages].reverse();
39+
for (const message of reversedMessages) {
40+
if (message.role === "system") {
41+
result.unshift(message);
42+
continue;
43+
} else if (cutoff) {
44+
continue;
45+
}
46+
let numTokens = countMessageTokens(model, message);
47+
if (maxTokens < numTokens) {
48+
cutoff = true;
49+
continue;
50+
}
51+
result.unshift(message);
52+
maxTokens -= numTokens;
53+
}
54+
55+
return result;
56+
}
57+
58+
const MAX_TOKENS = 128000;
59+
60+
function countToolsTokens(model: string, tools: any[]): number {
61+
if (tools.length === 0) {
62+
return 0;
63+
}
64+
const json = JSON.stringify(tools);
65+
return countTokens(model, json);
66+
}
67+
68+
function countMessageTokens(model: string, message: any): number {
69+
return countTokens(model, JSON.stringify(message.content) || "");
70+
}
71+
72+
function countTokens(model: string, text: string): number {
73+
return text.length / 3;
74+
}
75+
76+
export function convertActionInputToAnthropicTool(action: ActionInput): Anthropic.Messages.Tool {
77+
return {
78+
name: action.name,
79+
description: action.description,
80+
input_schema: JSON.parse(action.jsonSchema),
81+
};
82+
}
83+
84+
export function convertMessageToAnthropicMessage(
85+
message: Message,
86+
): Anthropic.Messages.MessageParam {
87+
if (message instanceof TextMessage) {
88+
if (message.role === "system") {
89+
return {
90+
role: "assistant",
91+
content: [
92+
{ type: "text", text: "THE FOLLOWING MESSAGE IS A SYSTEM MESSAGE: " + message.content },
93+
],
94+
};
95+
} else {
96+
return {
97+
role: message.role === "user" ? "user" : "assistant",
98+
content: [{ type: "text", text: message.content }],
99+
};
100+
}
101+
} else if (message instanceof ActionExecutionMessage) {
102+
return {
103+
role: "assistant",
104+
content: [
105+
{
106+
id: message.id,
107+
type: "tool_use",
108+
input: message.arguments,
109+
name: message.name,
110+
},
111+
],
112+
};
113+
} else if (message instanceof ResultMessage) {
114+
return {
115+
role: "user",
116+
content: [
117+
{
118+
type: "tool_result",
119+
content: message.result,
120+
tool_use_id: message.actionExecutionId,
121+
},
122+
],
123+
};
124+
}
125+
}
126+
127+
export function groupAnthropicMessagesByRole(
128+
messageParams: Anthropic.Messages.MessageParam[],
129+
): Anthropic.Messages.MessageParam[] {
130+
return messageParams.reduce((acc, message) => {
131+
const lastGroup = acc[acc.length - 1];
132+
133+
if (lastGroup && lastGroup.role === message.role) {
134+
lastGroup.content = lastGroup.content.concat(message.content as any);
135+
} else {
136+
acc.push({
137+
role: message.role,
138+
content: [...(message.content as any)],
139+
});
140+
}
141+
142+
return acc;
143+
}, [] as Anthropic.Messages.MessageParam[]);
144+
}

0 commit comments

Comments
 (0)