Lab/main dev

CopilotKit/examples/next-openai/.env.local.example

@@ -1 +1 @@
-OPENAI_API_KEY=xxxxxxx
+OPENAI_API_KEY=sk-proj...

CopilotKit/packages/react-core/src/components/copilot-provider/copilot-messages.tsx

@@ -117,10 +117,28 @@ export function MessagesTapProvider({ children }: { children: React.ReactNode })
  */
 
 export function CopilotMessages({ children }: { children: ReactNode }) {
-  const [messages, setMessages] = useState<Message[]>([]);
+  const [messages, setMessagesRaw] = useState<Message[]>([]);
   const lastLoadedThreadId = useRef<string>();
   const lastLoadedAgentName = useRef<string>();
   const lastLoadedMessages = useRef<string>();
+
+  // Wrapper to log all setMessages calls
+  const setMessages = useCallback((newMessages: Message[] | ((prev: Message[]) => Message[])) => {
+    if (typeof newMessages === 'function') {
+      setMessagesRaw((prev) => {
+        const updated = newMessages(prev);
+        console.log(`📝 [MESSAGES UPDATE] Function update: ${prev.length} -> ${updated.length} messages`);
+        return updated;
+      });
+    } else {
+      console.log(`📝 [MESSAGES SET] Direct set: ${newMessages.length} messages`);
+      const stack = new Error().stack;
+      if (stack?.includes('fetchMessages')) {
+        console.log(`   ⚠️ Called from fetchMessages - this resets conversation!`);
+      }
+      setMessagesRaw(newMessages);
+    }
+  }, []);
 
   const { updateTapMessages } = useMessagesTap();
 
@@ -262,6 +280,11 @@ export function CopilotMessages({ children }: { children: ReactNode }) {
       return;
     }
 
+    console.log("⚠️ [MESSAGES CONTEXT] Loading agent state - this might reset messages!");
+    console.log(`   ThreadId: ${threadId} (was: ${lastLoadedThreadId.current})`);
+    console.log(`   Agent: ${agentSession?.agentName} (was: ${lastLoadedAgentName.current})`);
+    console.log(`   Current messages count: ${messages.length}`);
+
     const fetchMessages = async () => {
       if (!agentSession?.agentName) return;
 
@@ -287,8 +310,9 @@ export function CopilotMessages({ children }: { children: ReactNode }) {
         lastLoadedThreadId.current = threadId;
         lastLoadedAgentName.current = agentSession?.agentName;
 
-        const messages = loadMessagesFromJsonRepresentation(JSON.parse(newMessages || "[]"));
-        setMessages(messages);
+        const loadedMessages = loadMessagesFromJsonRepresentation(JSON.parse(newMessages || "[]"));
+        console.log(`🔄 [MESSAGES RESET] Replacing ${messages.length} messages with ${loadedMessages.length} from agent state`);
+        setMessages(loadedMessages);
       }
     };
     void fetchMessages();

CopilotKit/packages/react-core/src/hooks/index.ts

@@ -19,3 +19,7 @@ export { useLangGraphInterrupt } from "./use-langgraph-interrupt";
 export { useLangGraphInterruptRender } from "./use-langgraph-interrupt-render";
 export { useCopilotAdditionalInstructions } from "./use-copilot-additional-instructions";
 export type { Tree, TreeNode } from "./use-tree";
+export { useRealtimeChat } from "./use-realtime-chat";
+export type { RealtimeConfig, RealtimeToolDefinition, UseRealtimeChatReturn } from "./use-realtime-chat";
+export { useRealtimeActionHandler } from "./use-realtime-action-handler";
+export type { UseRealtimeActionHandlerReturn } from "./use-realtime-action-handler";

CopilotKit/packages/react-core/src/hooks/use-chat.ts

@@ -934,7 +934,10 @@ export function useChat(options: UseChatOptions): UseChatHelpers {
 
   const append = useAsyncCallback(
     async (message: Message, options?: AppendMessageOptions): Promise<void> => {
-      const followUp = options?.followUp ?? true;
+      // Check if this specific message should skip inference
+      const skipInference = (message as any).metadata?.skipInference;
+      const followUp = skipInference ? false : (options?.followUp ?? true);
+
       if (isLoading) {
         pendingAppendsRef.current.push({ message, followUp });
         return;

CopilotKit/packages/react-core/src/hooks/use-realtime-action-handler.ts

@@ -0,0 +1,102 @@
+/**
+ * Realtime Action Handler for CopilotKit
+ * 
+ * This hook provides integration between voice/realtime commands and CopilotKit actions,
+ * using the unified ActionExecutionMessage with metadata for voice context.
+ */
+
+import { useCallback } from "react";
+import { useCopilotContext, useCopilotMessagesContext } from "../context";
+import { FrontendAction } from "../types/frontend-action";
+import { ActionExecutionMessage } from "@copilotkit/runtime-client-gql";
+
+export interface UseRealtimeActionHandlerReturn {
+  /**
+   * Execute an action with source context (voice/text/api)
+   */
+  executeAction: (actionName: string, args: Record<string, any>, source?: 'voice' | 'text' | 'api') => Promise<any>;
+
+  /**
+   * Get all available actions
+   */
+  getAvailableActions: () => FrontendAction<any>[];
+
+  /**
+   * Backward compatibility for voice actions
+   */
+  executeVoiceAction?: (actionName: string, args: Record<string, any>) => Promise<any>;
+}
+
+export function useRealtimeActionHandler(): UseRealtimeActionHandlerReturn {
+  const { actions } = useCopilotContext();
+  const { messages, setMessages } = useCopilotMessagesContext();
+
+  const executeAction = useCallback(async (
+    actionName: string, 
+    args: Record<string, any>,
+    source: 'voice' | 'text' | 'api' = 'text'
+  ) => {
+
+    // Find the specific action
+    const action = Object.values(actions).find((a: any) => a.name === actionName);
+
+    if (!action) {
+      console.warn(`[RealtimeActionHandler] Action not found: ${actionName}`);
+      throw new Error(`Action not found: ${actionName}`);
+    }
+
+
+    // Extract metadata if present
+    const { __metadata, ...cleanArgs } = args;
+    const callId = __metadata?.callId;
+
+    // Create ActionExecutionMessage with metadata for source
+    const actionMessage = new ActionExecutionMessage({
+      id: callId || `${source}-action-${Date.now()}`,
+      name: actionName,
+      arguments: cleanArgs,
+      parentMessageId: messages.length > 0 ? messages[messages.length - 1].id : null,
+      metadata: {
+        source,
+        skipInference: source === 'voice', // Voice actions skip inference by default
+        ...(source === 'voice' && { voiceData: { timestamp: Date.now() } })
+      }
+    } as any);
+
+    // Add the action message to trigger rendering
+    // Use function form to ensure we get the latest messages
+    if (source === 'voice') {
+      // For voice actions, add a longer delay to ensure transcript arrives first
+      // This is more reliable than complex reordering which breaks message types
+      setTimeout(() => {
+        setMessages((prevMessages) => [...prevMessages, actionMessage]);
+      }, 500); // 500ms should handle most transcript delays
+    } else {
+      setMessages((prevMessages) => [...prevMessages, actionMessage]);
+    }
+
+    // Execute the action handler if it exists
+    if (action.handler) {
+      try {
+        const result = await action.handler(cleanArgs);
+        return result;
+      } catch (error) {
+        console.error(`[RealtimeActionHandler] Action handler failed:`, error);
+        throw error;
+      }
+    }
+
+    return { success: true };
+  }, [actions, messages, setMessages]);
+
+  const getAvailableActions = useCallback(() => {
+    return Object.values(actions) as FrontendAction<any>[];
+  }, [actions]);
+
+  return {
+    executeAction,
+    getAvailableActions,
+    // Backwards compatibility aliases
+    executeVoiceAction: (name: string, args: Record<string, any>) => executeAction(name, args, 'voice'),
+  };
+}