From 2791a5301e308d5358c948c74616d7f71806aa75 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Tue, 9 Jun 2026 10:22:02 -0700 Subject: [PATCH 1/3] feat: emit capability-gated reasoning on tool-call-only response paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Extend the model-capability reasoning gate (resolveReasoningForModel) to the tool-call-only response path of every provider — OpenAI chat + Responses, Cohere, Bedrock (invoke + Converse), Ollama (incl. content+tool), and WebSocket Responses — plus the Gemini chat tool-only path and the Gemini audio companion. A tool-only fixture carrying reasoning now emits the provider-native reasoning channel for reasoning-capable models and is suppressed under strict mode / warns otherwise, matching the text and content+tool paths. Leading reasoning blocks shift streaming tool/output indices by one where applicable. --- src/bedrock-converse.ts | 93 +++++++++++++++++++++++++-------- src/bedrock.ts | 105 +++++++++++++++++++++++++++++-------- src/cohere.ts | 38 +++++++++++++- src/gemini.ts | 111 +++++++++++++++++++++++++++++++--------- src/helpers.ts | 20 ++++++++ src/ollama.ts | 67 +++++++++++++++++++++++- src/responses.ts | 21 +++++++- src/server.ts | 16 +++++- src/ws-responses.ts | 9 ++++ 9 files changed, 408 insertions(+), 72 deletions(-) diff --git a/src/bedrock-converse.ts b/src/bedrock-converse.ts index 35da99c7..4e769820 100644 --- a/src/bedrock-converse.ts +++ b/src/bedrock-converse.ts @@ -231,19 +231,46 @@ function buildBedrockStreamToolCallEvents( toolCalls: ToolCall[], chunkSize: number, logger: Logger, + reasoning?: string, overrides?: ResponseOverrides, ): Array<{ eventType: string; payload: object }> { const events: Array<{ eventType: string; payload: object }> = [ { eventType: "messageStart", payload: { role: "assistant" } }, ]; + // A leading reasoning block occupies contentBlockIndex 0, shifting the + // toolUse blocks by +1 (mirrors the content+tool builder's sequencing). + if (reasoning) { + const reasoningBlockIndex = 0; + events.push({ + eventType: "contentBlockStart", + payload: { contentBlockIndex: reasoningBlockIndex, start: { reasoningContent: {} } }, + }); + for (let i = 0; i < reasoning.length; i += chunkSize) { + events.push({ + eventType: "contentBlockDelta", + payload: { + contentBlockIndex: reasoningBlockIndex, + delta: { reasoningContent: { text: reasoning.slice(i, i + chunkSize) } }, + }, + }); + } + events.push({ + eventType: "contentBlockStop", + payload: { contentBlockIndex: reasoningBlockIndex }, + }); + } + + const toolBlockOffset = reasoning ? 1 : 0; + for (let tcIdx = 0; tcIdx < toolCalls.length; tcIdx++) { + const blockIndex = tcIdx + toolBlockOffset; const tc = toolCalls[tcIdx]; const toolUseId = tc.id || generateToolUseId(); events.push({ eventType: "contentBlockStart", payload: { - contentBlockIndex: tcIdx, + contentBlockIndex: blockIndex, start: { toolUse: { toolUseId, name: tc.name } }, }, }); @@ -252,14 +279,14 @@ function buildBedrockStreamToolCallEvents( events.push({ eventType: "contentBlockDelta", payload: { - contentBlockIndex: tcIdx, + contentBlockIndex: blockIndex, delta: { toolUse: { input: argsStr.slice(i, i + chunkSize) } }, }, }); } events.push({ eventType: "contentBlockStop", - payload: { contentBlockIndex: tcIdx }, + payload: { contentBlockIndex: blockIndex }, }); } events.push({ @@ -419,30 +446,39 @@ function buildConverseTextResponse( function buildConverseToolCallResponse( toolCalls: ToolCall[], logger: Logger, + reasoning?: string, overrides?: ResponseOverrides, ): object { + const contentBlocks: object[] = []; + if (reasoning) { + contentBlocks.push({ + reasoningContent: { reasoningText: { text: reasoning } }, + }); + } + for (const tc of toolCalls) { + let argsObj: unknown; + try { + argsObj = JSON.parse(tc.arguments || "{}"); + } catch { + logger.warn( + `Malformed JSON in fixture tool call arguments for "${tc.name}": ${tc.arguments}`, + ); + argsObj = {}; + } + contentBlocks.push({ + toolUse: { + toolUseId: tc.id || generateToolUseId(), + name: tc.name, + input: argsObj, + }, + }); + } + return { output: { message: { role: "assistant", - content: toolCalls.map((tc) => { - let argsObj: unknown; - try { - argsObj = JSON.parse(tc.arguments || "{}"); - } catch { - logger.warn( - `Malformed JSON in fixture tool call arguments for "${tc.name}": ${tc.arguments}`, - ); - argsObj = {}; - } - return { - toolUse: { - toolUseId: tc.id || generateToolUseId(), - name: tc.name, - input: argsObj, - }, - }; - }), + content: contentBlocks, }, }, stopReason: converseStopReason(overrides?.finishReason, "tool_use"), @@ -771,6 +807,12 @@ export async function handleConverse( ); } const overrides = extractOverrides(response); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + resolveStrictMode(defaults.strict, req.headers), + logger, + ); journal.add({ method: req.method ?? "POST", path: urlPath, @@ -778,7 +820,7 @@ export async function handleConverse( body: completionReq, response: { status: 200, fixture }, }); - const body = buildConverseToolCallResponse(response.toolCalls, logger, overrides); + const body = buildConverseToolCallResponse(response.toolCalls, logger, effReasoning, overrides); res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); return; @@ -1111,6 +1153,12 @@ export async function handleConverseStream( ); } const overrides = extractOverrides(response); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + resolveStrictMode(defaults.strict, req.headers), + logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path: urlPath, @@ -1122,6 +1170,7 @@ export async function handleConverseStream( response.toolCalls, chunkSize, logger, + effReasoning, overrides, ); const interruption = createInterruptionSignal(fixture); diff --git a/src/bedrock.ts b/src/bedrock.ts index 282b524d..70a57596 100644 --- a/src/bedrock.ts +++ b/src/bedrock.ts @@ -294,29 +294,36 @@ function buildBedrockToolCallResponse( toolCalls: ToolCall[], model: string, logger: Logger, + reasoning?: string, overrides?: ResponseOverrides, ): object { + const contentBlocks: object[] = []; + // Thinking block (emitted before tool_use blocks when reasoning is present) + if (reasoning) { + contentBlocks.push({ type: "thinking", thinking: reasoning, signature: "" }); + } + for (const tc of toolCalls) { + let argsObj: unknown; + try { + argsObj = JSON.parse(tc.arguments || "{}"); + } catch { + logger.warn( + `Malformed JSON in fixture tool call arguments for "${tc.name}": ${tc.arguments}`, + ); + argsObj = {}; + } + contentBlocks.push({ + type: "tool_use", + id: tc.id || generateToolUseId(), + name: tc.name, + input: argsObj, + }); + } return { id: overrides?.id ?? generateMessageId(), type: "message", role: "assistant", - content: toolCalls.map((tc) => { - let argsObj: unknown; - try { - argsObj = JSON.parse(tc.arguments || "{}"); - } catch { - logger.warn( - `Malformed JSON in fixture tool call arguments for "${tc.name}": ${tc.arguments}`, - ); - argsObj = {}; - } - return { - type: "tool_use", - id: tc.id || generateToolUseId(), - name: tc.name, - input: argsObj, - }; - }), + content: contentBlocks, model: overrides?.model ?? model, stop_reason: bedrockStopReason(overrides?.finishReason, "tool_use"), stop_sequence: null, @@ -562,6 +569,9 @@ export async function handleBedrock( response.toolCalls, completionReq.model, logger, + // Reasoning is rendered by the text response in this merged path; pass + // undefined here to avoid emitting a duplicate thinking block. + undefined, overrides, ); // Merge: take the text response as base, append tool_use blocks, set stop_reason to tool_use @@ -614,6 +624,12 @@ export async function handleBedrock( logger.warn("webSearches in fixture response are not supported for Bedrock API — ignoring"); } const overrides = extractOverrides(response); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + resolveStrictMode(defaults.strict, req.headers), + logger, + ); journal.add({ method: req.method ?? "POST", path: urlPath, @@ -625,6 +641,7 @@ export async function handleBedrock( response.toolCalls, completionReq.model, logger, + effReasoning, overrides, ); res.writeHead(200, { "Content-Type": "application/json" }); @@ -928,21 +945,62 @@ export function buildBedrockStreamToolCallEvents( model: string, chunkSize: number, logger: Logger, + reasoning?: string, overrides?: ResponseOverrides, ): Array<{ eventType: string; payload: object }> { const events: Array<{ eventType: string; payload: object }> = []; events.push(buildBedrockInvokeMessageStart(model, overrides)); + // Leading reasoning block shifts every subsequent tool_use block index by 1. + let blockIndex = 0; + + // Thinking block (emitted before tool_use blocks when reasoning is present) + if (reasoning) { + events.push({ + eventType: BEDROCK_INVOKE_STREAM_EVENT_TYPE, + payload: { + type: "content_block_start", + index: blockIndex, + content_block: { type: "thinking", thinking: "", signature: "" }, + }, + }); + for (let i = 0; i < reasoning.length; i += chunkSize) { + const slice = reasoning.slice(i, i + chunkSize); + events.push({ + eventType: BEDROCK_INVOKE_STREAM_EVENT_TYPE, + payload: { + type: "content_block_delta", + index: blockIndex, + delta: { type: "thinking_delta", thinking: slice }, + }, + }); + } + events.push({ + eventType: BEDROCK_INVOKE_STREAM_EVENT_TYPE, + payload: { + type: "content_block_delta", + index: blockIndex, + delta: { type: "signature_delta", signature: "" }, + }, + }); + events.push({ + eventType: BEDROCK_INVOKE_STREAM_EVENT_TYPE, + payload: { type: "content_block_stop", index: blockIndex }, + }); + blockIndex++; + } + for (let tcIdx = 0; tcIdx < toolCalls.length; tcIdx++) { const tc = toolCalls[tcIdx]; const toolUseId = tc.id || generateToolUseId(); + const currentBlock = blockIndex + tcIdx; events.push({ eventType: BEDROCK_INVOKE_STREAM_EVENT_TYPE, payload: { type: "content_block_start", - index: tcIdx, + index: currentBlock, content_block: { type: "tool_use", id: toolUseId, @@ -960,7 +1018,7 @@ export function buildBedrockStreamToolCallEvents( eventType: BEDROCK_INVOKE_STREAM_EVENT_TYPE, payload: { type: "content_block_delta", - index: tcIdx, + index: currentBlock, delta: { type: "input_json_delta", partial_json: slice }, }, }); @@ -968,7 +1026,7 @@ export function buildBedrockStreamToolCallEvents( events.push({ eventType: BEDROCK_INVOKE_STREAM_EVENT_TYPE, - payload: { type: "content_block_stop", index: tcIdx }, + payload: { type: "content_block_stop", index: currentBlock }, }); } @@ -1287,6 +1345,12 @@ export async function handleBedrockStream( logger.warn("webSearches in fixture response are not supported for Bedrock API — ignoring"); } const overrides = extractOverrides(response); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + resolveStrictMode(defaults.strict, req.headers), + logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path: urlPath, @@ -1299,6 +1363,7 @@ export async function handleBedrockStream( completionReq.model, chunkSize, logger, + effReasoning, overrides, ); const interruption = createInterruptionSignal(fixture); diff --git a/src/cohere.ts b/src/cohere.ts index 946234d6..f28f7a1e 100644 --- a/src/cohere.ts +++ b/src/cohere.ts @@ -262,6 +262,7 @@ function buildCohereTextResponse( function buildCohereToolCallResponse( toolCalls: ToolCall[], logger: Logger, + reasoning?: string, overrides?: ResponseOverrides, ): object { const cohereCalls = toolCalls.map((tc) => { @@ -286,12 +287,18 @@ function buildCohereToolCallResponse( }; }); + // Reasoning as a leading text block (Cohere has no native reasoning type) + const contentBlocks: { type: string; text: string }[] = []; + if (reasoning) { + contentBlocks.push({ type: "text", text: reasoning }); + } + return { id: overrides?.id ?? generateMessageId(), finish_reason: cohereFinishReason(overrides?.finishReason, "TOOL_CALL"), message: { role: "assistant", - content: [], + content: contentBlocks, tool_calls: cohereCalls, tool_plan: "", citations: [], @@ -443,6 +450,7 @@ function buildCohereToolCallStreamEvents( toolCalls: ToolCall[], chunkSize: number, logger: Logger, + reasoning?: string, overrides?: ResponseOverrides, ): CohereSSEEvent[] { const msgId = overrides?.id ?? generateMessageId(); @@ -463,6 +471,24 @@ function buildCohereToolCallStreamEvents( }, }); + // Reasoning as a text block before the tool plan (Cohere has no native reasoning type) + if (reasoning) { + events.push({ + type: "content-start", + index: 0, + delta: { message: { content: { type: "text" } } }, + }); + for (let i = 0; i < reasoning.length; i += chunkSize) { + const slice = reasoning.slice(i, i + chunkSize); + events.push({ + type: "content-delta", + index: 0, + delta: { message: { content: { type: "text", text: slice } } }, + }); + } + events.push({ type: "content-end", index: 0 }); + } + // tool-plan-delta events.push({ type: "tool-plan-delta", @@ -1090,6 +1116,13 @@ export async function handleCohere( ); } const overrides = extractOverrides(response); + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + cohereReq.model, + effectiveStrict, + logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path: req.url ?? "/v2/chat", @@ -1098,7 +1131,7 @@ export async function handleCohere( response: { status: 200, fixture }, }); if (cohereReq.stream !== true) { - const body = buildCohereToolCallResponse(response.toolCalls, logger, overrides); + const body = buildCohereToolCallResponse(response.toolCalls, logger, effReasoning, overrides); res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); } else { @@ -1106,6 +1139,7 @@ export async function handleCohere( response.toolCalls, chunkSize, logger, + effReasoning, overrides, ); const interruption = createInterruptionSignal(fixture); diff --git a/src/gemini.ts b/src/gemini.ts index 9fd5d780..70c95f8e 100644 --- a/src/gemini.ts +++ b/src/gemini.ts @@ -344,24 +344,44 @@ function parseToolCallPart(tc: ToolCall, logger: Logger): GeminiPart { function buildGeminiToolCallStreamChunks( toolCalls: ToolCall[], + chunkSize: number, logger: Logger, + reasoning?: string, overrides?: ResponseOverrides, ): GeminiResponseChunk[] { + const chunks: GeminiResponseChunk[] = []; + + // Reasoning chunks (thought: true) — mirror the content+tool / text paths so a + // tool-only fixture on a reasoning-capable model still emits its thought channel. + if (reasoning) { + for (let i = 0; i < reasoning.length; i += chunkSize) { + const slice = reasoning.slice(i, i + chunkSize); + chunks.push({ + candidates: [ + { + content: { role: "model", parts: [{ text: slice, thought: true }] }, + index: 0, + }, + ], + }); + } + } + const parts: GeminiPart[] = toolCalls.map((tc) => parseToolCallPart(tc, logger)); // Gemini sends all tool calls in a single response chunk - return [ - { - candidates: [ - { - content: { role: "model", parts }, - finishReason: geminiFinishReason(overrides?.finishReason, "FUNCTION_CALL"), - index: 0, - }, - ], - usageMetadata: geminiUsageMetadata(overrides), - }, - ]; + chunks.push({ + candidates: [ + { + content: { role: "model", parts }, + finishReason: geminiFinishReason(overrides?.finishReason, "FUNCTION_CALL"), + index: 0, + }, + ], + usageMetadata: geminiUsageMetadata(overrides), + }); + + return chunks; } // Non-streaming response builders @@ -392,9 +412,14 @@ function buildGeminiTextResponse( function buildGeminiToolCallResponse( toolCalls: ToolCall[], logger: Logger, + reasoning?: string, overrides?: ResponseOverrides, ): GeminiResponseChunk { - const parts: GeminiPart[] = toolCalls.map((tc) => parseToolCallPart(tc, logger)); + const parts: GeminiPart[] = []; + if (reasoning) { + parts.push({ text: reasoning, thought: true }); + } + parts.push(...toolCalls.map((tc) => parseToolCallPart(tc, logger))); return { candidates: [ @@ -518,11 +543,21 @@ function resolveAudioInlineData(audio: AudioResponse): { mimeType: string; data: // NOTE: audio companions are only re-emitted on this Gemini replay path because // `audioB64` collapse is currently Gemini-only — a cross-provider audio fixture // would not replay its companions. -function buildGeminiAudioParts(audio: AudioResponse, logger: Logger): GeminiPart[] { +// `effReasoning` is the capability-gated reasoning string (already passed +// through resolveReasoningForModel at the dispatch). When the requested model +// is not reasoning-capable (and strict mode is on), it is undefined and the +// companion `thought` part is suppressed — mirroring the text/content+tool +// branches so the audio path does not replay reasoning a real Gemini model +// would never emit. +function buildGeminiAudioParts( + audio: AudioResponse, + logger: Logger, + effReasoning: string | undefined, +): GeminiPart[] { const inlineData = resolveAudioInlineData(audio); const parts: GeminiPart[] = [{ inlineData }]; - if (audio.reasoning) { - parts.push({ text: audio.reasoning, thought: true }); + if (effReasoning) { + parts.push({ text: effReasoning, thought: true }); } if (audio.content) { parts.push({ text: audio.content }); @@ -533,11 +568,15 @@ function buildGeminiAudioParts(audio: AudioResponse, logger: Logger): GeminiPart return parts; } -function buildGeminiAudioResponse(audio: AudioResponse, logger: Logger): GeminiResponseChunk { +function buildGeminiAudioResponse( + audio: AudioResponse, + logger: Logger, + effReasoning: string | undefined, +): GeminiResponseChunk { return { candidates: [ { - content: { role: "model", parts: buildGeminiAudioParts(audio, logger) }, + content: { role: "model", parts: buildGeminiAudioParts(audio, logger, effReasoning) }, finishReason: audio.toolCalls?.length ? "FUNCTION_CALL" : "STOP", index: 0, }, @@ -546,12 +585,16 @@ function buildGeminiAudioResponse(audio: AudioResponse, logger: Logger): GeminiR }; } -function buildGeminiAudioStreamChunks(audio: AudioResponse, logger: Logger): GeminiResponseChunk[] { +function buildGeminiAudioStreamChunks( + audio: AudioResponse, + logger: Logger, + effReasoning: string | undefined, +): GeminiResponseChunk[] { return [ { candidates: [ { - content: { role: "model", parts: buildGeminiAudioParts(audio, logger) }, + content: { role: "model", parts: buildGeminiAudioParts(audio, logger, effReasoning) }, finishReason: audio.toolCalls?.length ? "FUNCTION_CALL" : "STOP", index: 0, }, @@ -804,6 +847,13 @@ export async function handleGemini( // Audio response if (isAudioResponse(response)) { + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + model, + effectiveStrict, + defaults.logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path, @@ -812,11 +862,11 @@ export async function handleGemini( response: { status: 200, fixture }, }); if (!streaming) { - const body = buildGeminiAudioResponse(response, logger); + const body = buildGeminiAudioResponse(response, logger, effReasoning); res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); } else { - const chunks = buildGeminiAudioStreamChunks(response, logger); + const chunks = buildGeminiAudioStreamChunks(response, logger, effReasoning); const interruption = createInterruptionSignal(fixture); const completed = await writeGeminiSSEStream(res, chunks, { latency, @@ -950,6 +1000,13 @@ export async function handleGemini( logger.warn("webSearches in fixture response are not supported for Gemini API — ignoring"); } const overrides = extractOverrides(response); + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + model, + effectiveStrict, + defaults.logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path, @@ -958,11 +1015,17 @@ export async function handleGemini( response: { status: 200, fixture }, }); if (!streaming) { - const body = buildGeminiToolCallResponse(response.toolCalls, logger, overrides); + const body = buildGeminiToolCallResponse(response.toolCalls, logger, effReasoning, overrides); res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); } else { - const chunks = buildGeminiToolCallStreamChunks(response.toolCalls, logger, overrides); + const chunks = buildGeminiToolCallStreamChunks( + response.toolCalls, + chunkSize, + logger, + effReasoning, + overrides, + ); const interruption = createInterruptionSignal(fixture); const completed = await writeGeminiSSEStream(res, chunks, { latency, diff --git a/src/helpers.ts b/src/helpers.ts index 7e08e2a7..c258434b 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -447,6 +447,7 @@ export function buildToolCallChunks( toolCalls: ToolCall[], model: string, chunkSize: number, + reasoning?: string, overrides?: ResponseOverrides, ): SSEChunk[] { const id = overrides?.id ?? generateId(); @@ -455,6 +456,23 @@ export function buildToolCallChunks( const chunks: SSEChunk[] = []; const fingerprint = overrides?.systemFingerprint; + // Reasoning chunks (emitted before tool calls, OpenRouter format) + if (reasoning) { + for (let i = 0; i < reasoning.length; i += chunkSize) { + const slice = reasoning.slice(i, i + chunkSize); + chunks.push({ + id, + object: "chat.completion.chunk", + created, + model: effectiveModel, + choices: [ + { index: 0, delta: { reasoning_content: slice }, logprobs: null, finish_reason: null }, + ], + ...(fingerprint !== undefined && { system_fingerprint: fingerprint }), + }); + } + } + // Role chunk chunks.push({ id, @@ -595,6 +613,7 @@ export function buildTextCompletion( export function buildToolCallCompletion( toolCalls: ToolCall[], model: string, + reasoning?: string, overrides?: ResponseOverrides, requestMessages?: ChatCompletionRequest["messages"], ): ChatCompletion { @@ -622,6 +641,7 @@ export function buildToolCallCompletion( role: overrides?.role ?? "assistant", content: null, refusal: null, + ...(reasoning ? { reasoning_content: reasoning } : {}), tool_calls: toolCalls.map((tc) => ({ id: tc.id || generateToolCallId(), type: "function" as const, diff --git a/src/ollama.ts b/src/ollama.ts index c73f969c..01d6b931 100644 --- a/src/ollama.ts +++ b/src/ollama.ts @@ -228,7 +228,9 @@ function buildOllamaChatTextResponse(content: string, model: string, reasoning?: function buildOllamaChatToolCallChunks( toolCalls: ToolCall[], model: string, + chunkSize: number, logger: Logger, + reasoning?: string, ): object[] { const ollamaToolCalls = toolCalls.map((tc) => { let argsObj: unknown; @@ -251,6 +253,20 @@ function buildOllamaChatToolCallChunks( // Tool calls are sent in a single chunk (no streaming of individual args) const chunks: object[] = []; const createdAt = new Date().toISOString(); + + // Reasoning chunks (before tool calls) + if (reasoning) { + for (let i = 0; i < reasoning.length; i += chunkSize) { + const slice = reasoning.slice(i, i + chunkSize); + chunks.push({ + model, + created_at: createdAt, + message: { role: "assistant", content: "", reasoning_content: slice }, + done: false, + }); + } + } + chunks.push({ model, created_at: createdAt, @@ -278,6 +294,7 @@ function buildOllamaChatToolCallResponse( toolCalls: ToolCall[], model: string, logger: Logger, + reasoning?: string, ): object { const ollamaToolCalls = toolCalls.map((tc) => { let argsObj: unknown; @@ -303,6 +320,7 @@ function buildOllamaChatToolCallResponse( message: { role: "assistant", content: "", + ...(reasoning ? { reasoning_content: reasoning } : {}), tool_calls: ollamaToolCalls, }, done: true, @@ -318,10 +336,24 @@ function buildOllamaChatContentWithToolCallsChunks( model: string, chunkSize: number, logger: Logger, + reasoning?: string, ): object[] { const chunks: object[] = []; const createdAt = new Date().toISOString(); + // Reasoning chunks (before content) + if (reasoning) { + for (let i = 0; i < reasoning.length; i += chunkSize) { + const slice = reasoning.slice(i, i + chunkSize); + chunks.push({ + model, + created_at: createdAt, + message: { role: "assistant", content: "", reasoning_content: slice }, + done: false, + }); + } + } + // Content chunks first for (let i = 0; i < content.length; i += chunkSize) { const slice = content.slice(i, i + chunkSize); @@ -380,6 +412,7 @@ function buildOllamaChatContentWithToolCallsResponse( toolCalls: ToolCall[], model: string, logger: Logger, + reasoning?: string, ): object { const ollamaToolCalls = toolCalls.map((tc) => { let argsObj: unknown; @@ -405,6 +438,7 @@ function buildOllamaChatContentWithToolCallsResponse( message: { role: "assistant", content, + ...(reasoning ? { reasoning_content: reasoning } : {}), tool_calls: ollamaToolCalls, }, done: true, @@ -689,12 +723,21 @@ export async function handleOllama( body: completionReq, response: { status: 200, fixture }, }); + // Gate reasoning emission on the requested model's capability (aimock#254). + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + effectiveStrict, + logger, + ); if (!streaming) { const body = buildOllamaChatContentWithToolCallsResponse( response.content, response.toolCalls, completionReq.model, logger, + effReasoning, ); res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); @@ -705,6 +748,7 @@ export async function handleOllama( completionReq.model, chunkSize, logger, + effReasoning, ); const interruption = createInterruptionSignal(fixture); const completed = await writeNDJSONStream(res, chunks, { @@ -787,12 +831,31 @@ export async function handleOllama( body: completionReq, response: { status: 200, fixture }, }); + // Gate reasoning emission on the requested model's capability (aimock#254). + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + effectiveStrict, + logger, + ); if (!streaming) { - const body = buildOllamaChatToolCallResponse(response.toolCalls, completionReq.model, logger); + const body = buildOllamaChatToolCallResponse( + response.toolCalls, + completionReq.model, + logger, + effReasoning, + ); res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); } else { - const chunks = buildOllamaChatToolCallChunks(response.toolCalls, completionReq.model, logger); + const chunks = buildOllamaChatToolCallChunks( + response.toolCalls, + completionReq.model, + chunkSize, + logger, + effReasoning, + ); const interruption = createInterruptionSignal(fixture); const completed = await writeNDJSONStream(res, chunks, { latency, diff --git a/src/responses.ts b/src/responses.ts index 3febd700..c522b7cc 100644 --- a/src/responses.ts +++ b/src/responses.ts @@ -324,13 +324,14 @@ export function buildToolCallStreamEvents( toolCalls: ToolCall[], model: string, chunkSize: number, + reasoning?: string, webSearches?: string[], overrides?: ResponseOverrides, ): ResponsesSSEEvent[] { const { respId, created, events, prefixOutputItems, nextOutputIndex } = buildResponsePreamble( model, chunkSize, - undefined, + reasoning, webSearches, overrides, ); @@ -721,10 +722,18 @@ function buildTextResponse( function buildToolCallResponse( toolCalls: ToolCall[], model: string, + reasoning?: string, webSearches?: string[], overrides?: ResponseOverrides, ): object { const output: object[] = []; + if (reasoning) { + output.push({ + type: "reasoning", + id: generateId("rs"), + summary: [{ type: "summary_text", text: reasoning }], + }); + } if (webSearches && webSearches.length > 0) { for (const query of webSearches) { output.push({ @@ -1208,6 +1217,14 @@ export async function handleResponses( // Tool call response if (isToolCallResponse(response)) { const overrides = extractOverrides(response); + // Gate reasoning emission on the requested model's capability (aimock#254). + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + effectiveStrict, + defaults.logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path: req.url ?? "/v1/responses", @@ -1219,6 +1236,7 @@ export async function handleResponses( const body = buildToolCallResponse( response.toolCalls, completionReq.model, + effReasoning, response.webSearches, overrides, ); @@ -1229,6 +1247,7 @@ export async function handleResponses( response.toolCalls, completionReq.model, chunkSize, + effReasoning, response.webSearches, overrides, ); diff --git a/src/server.ts b/src/server.ts index 5f67a436..17652245 100644 --- a/src/server.ts +++ b/src/server.ts @@ -936,6 +936,13 @@ async function handleCompletions( ); } const overrides = extractOverrides(response); + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + body.model, + effectiveStrict, + defaults.logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path: req.url ?? COMPLETIONS_PATH, @@ -947,13 +954,20 @@ async function handleCompletions( const completion = buildToolCallCompletion( response.toolCalls, body.model, + effReasoning, overrides, body.messages, ); res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(completion)); } else { - const chunks = buildToolCallChunks(response.toolCalls, body.model, chunkSize, overrides); + const chunks = buildToolCallChunks( + response.toolCalls, + body.model, + chunkSize, + effReasoning, + overrides, + ); const completionText = response.toolCalls.map((tc) => tc.name + tc.arguments).join(""); const usageChunk = includeUsage ? buildUsageChunk( diff --git a/src/ws-responses.ts b/src/ws-responses.ts index 56820516..2052ccab 100644 --- a/src/ws-responses.ts +++ b/src/ws-responses.ts @@ -355,6 +355,15 @@ async function processMessage( response.toolCalls, completionReq.model, chunkSize, + // Gate the synthesized reasoning channel on the requested model's + // capability, matching the WS text / content+tool branches and the HTTP + // tool-only path so reasoning emission is transport-independent. + resolveReasoningForModel( + response.reasoning, + completionReq.model, + effectiveStrict, + defaults.logger, + ), response.webSearches, extractOverrides(response), ); From c29c6eb8af1387bed981064c8c1a9e806f58b220 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Tue, 9 Jun 2026 10:22:09 -0700 Subject: [PATCH 2/3] test: cover tool-only reasoning capability across providers Per-provider stream + non-stream tests for the tool-only reasoning gate (capable emits / non-reasoning strict-OFF warns+emits / strict-ON suppresses / no-reasoning no-op), including streaming block-index ordering assertions for the leading reasoning block. --- ...edrock-converse-toolonly-reasoning.test.ts | 360 ++++++++++++++++ .../bedrock-toolonly-reasoning.test.ts | 365 +++++++++++++++++ .../cohere-toolonly-reasoning.test.ts | 249 +++++++++++ .../gemini-audio-reasoning-capability.test.ts | 273 +++++++++++++ ...mini-toolonly-reasoning-capability.test.ts | 260 ++++++++++++ ...chat-toolonly-reasoning-capability.test.ts | 250 ++++++++++++ .../reasoning-capability-ollama-tools.test.ts | 386 ++++++++++++++++++ ...nses-toolonly-reasoning-capability.test.ts | 282 +++++++++++++ src/__tests__/token-estimation.test.ts | 2 + src/__tests__/ws-responses.test.ts | 78 ++++ 10 files changed, 2505 insertions(+) create mode 100644 src/__tests__/bedrock-converse-toolonly-reasoning.test.ts create mode 100644 src/__tests__/bedrock-toolonly-reasoning.test.ts create mode 100644 src/__tests__/cohere-toolonly-reasoning.test.ts create mode 100644 src/__tests__/gemini-audio-reasoning-capability.test.ts create mode 100644 src/__tests__/gemini-toolonly-reasoning-capability.test.ts create mode 100644 src/__tests__/openai-chat-toolonly-reasoning-capability.test.ts create mode 100644 src/__tests__/reasoning-capability-ollama-tools.test.ts create mode 100644 src/__tests__/responses-toolonly-reasoning-capability.test.ts diff --git a/src/__tests__/bedrock-converse-toolonly-reasoning.test.ts b/src/__tests__/bedrock-converse-toolonly-reasoning.test.ts new file mode 100644 index 00000000..775d5027 --- /dev/null +++ b/src/__tests__/bedrock-converse-toolonly-reasoning.test.ts @@ -0,0 +1,360 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import * as http from "node:http"; +import { crc32 } from "node:zlib"; +import type { Fixture } from "../types.js"; +import { createServer, type ServerInstance } from "../server.js"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +let instance: ServerInstance | null = null; +let baseUrl: string; + +function post( + path: string, + body: unknown, + headers?: Record, +): Promise<{ status: number; headers: http.IncomingHttpHeaders; body: string }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify(body); + const parsed = new URL(baseUrl); + const req = http.request( + { + hostname: parsed.hostname, + port: parsed.port, + path, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => { + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + body: Buffer.concat(chunks).toString(), + }); + }); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +function postRaw( + path: string, + body: unknown, + headers?: Record, +): Promise<{ status: number; headers: http.IncomingHttpHeaders; body: Buffer }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify(body); + const parsed = new URL(baseUrl); + const req = http.request( + { + hostname: parsed.hostname, + port: parsed.port, + path, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => { + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + body: Buffer.concat(chunks), + }); + }); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +/** + * Decode AWS Event Stream binary frames from a Buffer. + * Returns an array of { eventType, payload } objects. + */ +function decodeEventStreamFrames(buf: Buffer): Array<{ eventType: string; payload: object }> { + const frames: Array<{ eventType: string; payload: object }> = []; + let offset = 0; + + while (offset < buf.length) { + if (offset + 12 > buf.length) break; + + const totalLength = buf.readUInt32BE(offset); + const headersLength = buf.readUInt32BE(offset + 4); + const preludeCrc = buf.readUInt32BE(offset + 8); + + const computedPreludeCrc = crc32(buf.subarray(offset, offset + 8)); + if (computedPreludeCrc >>> 0 !== preludeCrc) { + throw new Error("Prelude CRC mismatch"); + } + + const headersStart = offset + 12; + const headersEnd = headersStart + headersLength; + const headers: Record = {}; + let hOff = headersStart; + while (hOff < headersEnd) { + const nameLen = buf.readUInt8(hOff); + hOff += 1; + const name = buf.subarray(hOff, hOff + nameLen).toString("utf8"); + hOff += nameLen; + hOff += 1; // skip header type byte (7 = STRING) + const valueLen = buf.readUInt16BE(hOff); + hOff += 2; + const value = buf.subarray(hOff, hOff + valueLen).toString("utf8"); + hOff += valueLen; + headers[name] = value; + } + + const payloadStart = headersEnd; + const payloadEnd = offset + totalLength - 4; // minus message CRC + const payloadBuf = buf.subarray(payloadStart, payloadEnd); + const payload = payloadBuf.length > 0 ? JSON.parse(payloadBuf.toString("utf8")) : {}; + + frames.push({ + eventType: headers[":event-type"] ?? "", + payload, + }); + + offset += totalLength; + } + + return frames; +} + +// Predicates over decoded Converse stream frames +function hasConverseReasoningDelta(frames: Array<{ eventType: string; payload: object }>): boolean { + return frames.some( + (f) => + f.eventType === "contentBlockDelta" && + (f.payload as { delta?: { reasoningContent?: unknown } }).delta?.reasoningContent !== + undefined, + ); +} + +function hasConverseToolUseStart(frames: Array<{ eventType: string; payload: object }>): boolean { + return frames.some( + (f) => + f.eventType === "contentBlockStart" && + (f.payload as { start?: { toolUse?: unknown } }).start?.toolUse !== undefined, + ); +} + +// --------------------------------------------------------------------------- +// Fixtures — tool-call-only path (no `content` string) +// --------------------------------------------------------------------------- + +const toolOnlyReasoningFixture: Fixture = { + match: { userMessage: "reason-then-call" }, + response: { + reasoning: "I should call get_weather to find out.", + toolCalls: [{ name: "get_weather", arguments: '{"city":"SF"}' }], + }, +}; + +const toolOnlyPlainFixture: Fixture = { + match: { userMessage: "just-call" }, + response: { + toolCalls: [{ name: "get_weather", arguments: '{"city":"SF"}' }], + }, +}; + +const allFixtures: Fixture[] = [toolOnlyReasoningFixture, toolOnlyPlainFixture]; + +// Model ids carrying Bedrock provider prefixes. +const REASONING_MODEL = "anthropic.claude-opus-4-20250514-v1:0"; // Claude 4 → reasoning-capable +const NON_REASONING_MODEL = "anthropic.claude-3-5-sonnet-20241022-v2:0"; // 3.5 → not reasoning-capable + +// --------------------------------------------------------------------------- +// Server lifecycle (default: strict OFF, warn-level logging so warns surface) +// --------------------------------------------------------------------------- + +beforeEach(async () => { + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + baseUrl = instance.url; +}); + +afterEach(async () => { + vi.restoreAllMocks(); + if (instance) { + await new Promise((resolve) => { + instance!.server.close(() => resolve()); + }); + instance = null; + } +}); + +// --------------------------------------------------------------------------- +// Bedrock Converse tool-only — non-streaming +// --------------------------------------------------------------------------- + +describe("Bedrock Converse tool-only reasoning capability gating (non-streaming)", () => { + it("emits reasoningContent block for a reasoning-capable model", async () => { + const res = await post(`/model/${REASONING_MODEL}/converse`, { + messages: [{ role: "user", content: [{ text: "reason-then-call" }] }], + }); + + expect(res.status).toBe(200); + const content = JSON.parse(res.body).output.message.content as Array>; + expect(content[0].reasoningContent).toBeDefined(); + expect( + (content[0].reasoningContent as { reasoningText: { text: string } }).reasoningText.text, + ).toBe("I should call get_weather to find out."); + // reasoning block precedes the toolUse block + expect(content.some((b) => "toolUse" in b)).toBe(true); + expect("reasoningContent" in content[0]).toBe(true); + expect("toolUse" in content[content.length - 1]).toBe(true); + }); + + it("non-reasoning model + strict OFF: still emits reasoningContent + warns", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const res = await post(`/model/${NON_REASONING_MODEL}/converse`, { + messages: [{ role: "user", content: [{ text: "reason-then-call" }] }], + }); + + expect(res.status).toBe(200); + const content = JSON.parse(res.body).output.message.content as Array>; + expect(content[0].reasoningContent).toBeDefined(); + expect(warnSpy.mock.calls.flat().join(" ").includes("not reasoning-capable")).toBe(true); + }); + + it("non-reasoning model + strict ON: suppresses reasoningContent", async () => { + const res = await post( + `/model/${NON_REASONING_MODEL}/converse`, + { messages: [{ role: "user", content: [{ text: "reason-then-call" }] }] }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const content = JSON.parse(res.body).output.message.content as Array>; + expect(content.some((b) => "reasoningContent" in b)).toBe(false); + expect(content.some((b) => "toolUse" in b)).toBe(true); + }); + + it("reasoning absent: no reasoningContent, no gating log", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const res = await post(`/model/${NON_REASONING_MODEL}/converse`, { + messages: [{ role: "user", content: [{ text: "just-call" }] }], + }); + + expect(res.status).toBe(200); + const content = JSON.parse(res.body).output.message.content as Array>; + expect(content.some((b) => "reasoningContent" in b)).toBe(false); + expect(content.some((b) => "toolUse" in b)).toBe(true); + expect(warnSpy.mock.calls.flat().join(" ").includes("not reasoning-capable")).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// Bedrock Converse tool-only — streaming +// --------------------------------------------------------------------------- + +describe("Bedrock Converse tool-only reasoning capability gating (streaming)", () => { + it("emits reasoningContent delta for a reasoning-capable model", async () => { + const res = await postRaw(`/model/${REASONING_MODEL}/converse-stream`, { + messages: [{ role: "user", content: [{ text: "reason-then-call" }] }], + }); + + expect(res.status).toBe(200); + const frames = decodeEventStreamFrames(res.body); + expect(hasConverseReasoningDelta(frames)).toBe(true); + expect(hasConverseToolUseStart(frames)).toBe(true); + }); + + it("non-reasoning model + strict OFF: still emits reasoningContent delta + warns", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const res = await postRaw(`/model/${NON_REASONING_MODEL}/converse-stream`, { + messages: [{ role: "user", content: [{ text: "reason-then-call" }] }], + }); + + expect(res.status).toBe(200); + expect(hasConverseReasoningDelta(decodeEventStreamFrames(res.body))).toBe(true); + expect(warnSpy.mock.calls.flat().join(" ").includes("not reasoning-capable")).toBe(true); + }); + + it("non-reasoning model + strict ON: suppresses reasoningContent delta", async () => { + const res = await postRaw( + `/model/${NON_REASONING_MODEL}/converse-stream`, + { messages: [{ role: "user", content: [{ text: "reason-then-call" }] }] }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const frames = decodeEventStreamFrames(res.body); + expect(hasConverseReasoningDelta(frames)).toBe(false); + expect(hasConverseToolUseStart(frames)).toBe(true); + }); + + it("reasoning absent: no reasoningContent delta", async () => { + const res = await postRaw(`/model/${NON_REASONING_MODEL}/converse-stream`, { + messages: [{ role: "user", content: [{ text: "just-call" }] }], + }); + + expect(res.status).toBe(200); + const frames = decodeEventStreamFrames(res.body); + expect(hasConverseReasoningDelta(frames)).toBe(false); + expect(hasConverseToolUseStart(frames)).toBe(true); + }); + + it("contentBlockIndex ordering: reasoning block at index 0, toolUse blocks shifted by +1", async () => { + const res = await postRaw(`/model/${REASONING_MODEL}/converse-stream`, { + messages: [{ role: "user", content: [{ text: "reason-then-call" }] }], + }); + + expect(res.status).toBe(200); + const frames = decodeEventStreamFrames(res.body); + + // Reasoning block occupies contentBlockIndex 0. + const reasoningStart = frames.find( + (f) => + f.eventType === "contentBlockStart" && + (f.payload as { start?: { reasoningContent?: unknown } }).start?.reasoningContent !== + undefined, + ); + expect(reasoningStart).toBeDefined(); + expect((reasoningStart!.payload as { contentBlockIndex: number }).contentBlockIndex).toBe(0); + + // The single toolUse block is shifted to contentBlockIndex 1 (not 0). + const toolUseStart = frames.find( + (f) => + f.eventType === "contentBlockStart" && + (f.payload as { start?: { toolUse?: unknown } }).start?.toolUse !== undefined, + ); + expect(toolUseStart).toBeDefined(); + expect((toolUseStart!.payload as { contentBlockIndex: number }).contentBlockIndex).toBe(1); + + // All reasoning frames carry index 0; the toolUse delta/stop carry index 1. + const reasoningDelta = frames.find( + (f) => + f.eventType === "contentBlockDelta" && + (f.payload as { delta?: { reasoningContent?: unknown } }).delta?.reasoningContent !== + undefined, + ); + expect((reasoningDelta!.payload as { contentBlockIndex: number }).contentBlockIndex).toBe(0); + + const toolUseDelta = frames.find( + (f) => + f.eventType === "contentBlockDelta" && + (f.payload as { delta?: { toolUse?: unknown } }).delta?.toolUse !== undefined, + ); + expect((toolUseDelta!.payload as { contentBlockIndex: number }).contentBlockIndex).toBe(1); + }); +}); diff --git a/src/__tests__/bedrock-toolonly-reasoning.test.ts b/src/__tests__/bedrock-toolonly-reasoning.test.ts new file mode 100644 index 00000000..166ef581 --- /dev/null +++ b/src/__tests__/bedrock-toolonly-reasoning.test.ts @@ -0,0 +1,365 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import * as http from "node:http"; +import { crc32 } from "node:zlib"; +import type { Fixture } from "../types.js"; +import { createServer, type ServerInstance } from "../server.js"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +let instance: ServerInstance | null = null; +let baseUrl: string; + +function post( + path: string, + body: unknown, + headers?: Record, +): Promise<{ status: number; headers: http.IncomingHttpHeaders; body: string }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify(body); + const parsed = new URL(baseUrl); + const req = http.request( + { + hostname: parsed.hostname, + port: parsed.port, + path, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => { + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + body: Buffer.concat(chunks).toString(), + }); + }); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +function postRaw( + path: string, + body: unknown, + headers?: Record, +): Promise<{ status: number; headers: http.IncomingHttpHeaders; body: Buffer }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify(body); + const parsed = new URL(baseUrl); + const req = http.request( + { + hostname: parsed.hostname, + port: parsed.port, + path, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => { + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + body: Buffer.concat(chunks), + }); + }); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +interface StreamFrame { + eventType: string; + payload: { + type?: string; + index?: number; + delta?: { type?: string }; + content_block?: { type?: string }; + }; +} + +/** + * Decode AWS Event Stream binary frames from a Buffer. + * Returns an array of { eventType, payload } objects. + */ +function decodeEventStreamFrames(buf: Buffer): StreamFrame[] { + const frames: StreamFrame[] = []; + let offset = 0; + + while (offset < buf.length) { + if (offset + 12 > buf.length) break; + + const totalLength = buf.readUInt32BE(offset); + const headersLength = buf.readUInt32BE(offset + 4); + const preludeCrc = buf.readUInt32BE(offset + 8); + + const computedPreludeCrc = crc32(buf.subarray(offset, offset + 8)); + if (computedPreludeCrc >>> 0 !== preludeCrc) { + throw new Error("Prelude CRC mismatch"); + } + + const headersStart = offset + 12; + const headersEnd = headersStart + headersLength; + const headers: Record = {}; + let hOff = headersStart; + while (hOff < headersEnd) { + const nameLen = buf.readUInt8(hOff); + hOff += 1; + const name = buf.subarray(hOff, hOff + nameLen).toString("utf8"); + hOff += nameLen; + hOff += 1; // skip header type byte (7 = STRING) + const valueLen = buf.readUInt16BE(hOff); + hOff += 2; + const value = buf.subarray(hOff, hOff + valueLen).toString("utf8"); + hOff += valueLen; + headers[name] = value; + } + + const payloadStart = headersEnd; + const payloadEnd = offset + totalLength - 4; // minus message CRC + const payloadBuf = buf.subarray(payloadStart, payloadEnd); + const payload = payloadBuf.length > 0 ? JSON.parse(payloadBuf.toString("utf8")) : {}; + + frames.push({ + eventType: headers[":event-type"] ?? "", + payload, + }); + + offset += totalLength; + } + + return frames; +} + +function hasInvokeThinkingDelta(frames: StreamFrame[]): boolean { + return frames.some( + (f) => f.payload.type === "content_block_delta" && f.payload.delta?.type === "thinking_delta", + ); +} + +// --------------------------------------------------------------------------- +// Fixtures — tool-only responses (toolCalls present, no `content`) +// --------------------------------------------------------------------------- + +const toolOnlyReasoningFixture: Fixture = { + match: { userMessage: "tool-think" }, + response: { + reasoning: "I should call get_weather to answer this.", + toolCalls: [{ name: "get_weather", arguments: '{"city":"SF"}' }], + }, +}; + +const toolOnlyPlainFixture: Fixture = { + match: { userMessage: "tool-plain" }, + response: { + toolCalls: [{ name: "get_weather", arguments: '{"city":"SF"}' }], + }, +}; + +const allFixtures: Fixture[] = [toolOnlyReasoningFixture, toolOnlyPlainFixture]; + +// Model ids carrying Bedrock provider prefixes. +const REASONING_MODEL = "anthropic.claude-opus-4-20250514-v1:0"; // Claude 4 → reasoning-capable +const NON_REASONING_MODEL = "anthropic.claude-3-5-sonnet-20241022-v2:0"; // 3.5 → not reasoning-capable + +// --------------------------------------------------------------------------- +// Server lifecycle (default: strict OFF, warn-level logging so warns surface) +// --------------------------------------------------------------------------- + +beforeEach(async () => { + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + baseUrl = instance.url; +}); + +afterEach(async () => { + vi.restoreAllMocks(); + if (instance) { + await new Promise((resolve) => { + instance!.server.close(() => resolve()); + }); + instance = null; + } +}); + +// --------------------------------------------------------------------------- +// Bedrock InvokeModel tool-only — non-streaming +// --------------------------------------------------------------------------- + +describe("Bedrock InvokeModel tool-only reasoning capability gating (non-streaming)", () => { + it("emits a thinking block before tool_use for a reasoning-capable model", async () => { + const res = await post(`/model/${REASONING_MODEL}/invoke`, { + messages: [{ role: "user", content: [{ type: "text", text: "tool-think" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + const types = (body.content as Array<{ type: string }>).map((b) => b.type); + expect(types[0]).toBe("thinking"); + expect(body.content[0].thinking).toBe("I should call get_weather to answer this."); + expect(types).toContain("tool_use"); + expect(body.stop_reason).toBe("tool_use"); + }); + + it("non-reasoning model + strict OFF: still emits thinking and logs a warning", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const res = await post(`/model/${NON_REASONING_MODEL}/invoke`, { + messages: [{ role: "user", content: [{ type: "text", text: "tool-think" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.content[0].type).toBe("thinking"); + expect(warnSpy.mock.calls.flat().join(" ").includes("not reasoning-capable")).toBe(true); + }); + + it("non-reasoning model + strict ON: suppresses thinking block", async () => { + const res = await post( + `/model/${NON_REASONING_MODEL}/invoke`, + { + messages: [{ role: "user", content: [{ type: "text", text: "tool-think" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + const types = (body.content as Array<{ type: string }>).map((b) => b.type); + expect(types).not.toContain("thinking"); + expect(types).toContain("tool_use"); + }); + + it("reasoning absent: no thinking block, no gating log", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const res = await post(`/model/${NON_REASONING_MODEL}/invoke`, { + messages: [{ role: "user", content: [{ type: "text", text: "tool-plain" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + const types = (body.content as Array<{ type: string }>).map((b) => b.type); + expect(types).not.toContain("thinking"); + expect(types).toContain("tool_use"); + expect(warnSpy.mock.calls.flat().join(" ").includes("not reasoning-capable")).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// Bedrock InvokeModel tool-only — streaming +// --------------------------------------------------------------------------- + +describe("Bedrock InvokeModel tool-only reasoning capability gating (streaming)", () => { + it("emits thinking_delta for a reasoning-capable model", async () => { + const res = await postRaw(`/model/${REASONING_MODEL}/invoke-with-response-stream`, { + messages: [{ role: "user", content: [{ type: "text", text: "tool-think" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }); + + expect(res.status).toBe(200); + expect(hasInvokeThinkingDelta(decodeEventStreamFrames(res.body))).toBe(true); + }); + + it("non-reasoning model + strict OFF: still emits thinking_delta + warns", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const res = await postRaw(`/model/${NON_REASONING_MODEL}/invoke-with-response-stream`, { + messages: [{ role: "user", content: [{ type: "text", text: "tool-think" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }); + + expect(res.status).toBe(200); + expect(hasInvokeThinkingDelta(decodeEventStreamFrames(res.body))).toBe(true); + expect(warnSpy.mock.calls.flat().join(" ").includes("not reasoning-capable")).toBe(true); + }); + + it("non-reasoning model + strict ON: suppresses thinking_delta", async () => { + const res = await postRaw( + `/model/${NON_REASONING_MODEL}/invoke-with-response-stream`, + { + messages: [{ role: "user", content: [{ type: "text", text: "tool-think" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + expect(hasInvokeThinkingDelta(decodeEventStreamFrames(res.body))).toBe(false); + }); + + it("reasoning absent: no thinking_delta", async () => { + const res = await postRaw(`/model/${NON_REASONING_MODEL}/invoke-with-response-stream`, { + messages: [{ role: "user", content: [{ type: "text", text: "tool-plain" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }); + + expect(res.status).toBe(200); + expect(hasInvokeThinkingDelta(decodeEventStreamFrames(res.body))).toBe(false); + }); + + it("block-index ordering: leading thinking block shifts tool_use to index 1", async () => { + const res = await postRaw(`/model/${REASONING_MODEL}/invoke-with-response-stream`, { + messages: [{ role: "user", content: [{ type: "text", text: "tool-think" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }); + + expect(res.status).toBe(200); + const frames = decodeEventStreamFrames(res.body); + + const starts = frames.filter((f) => f.payload.type === "content_block_start"); + // First content_block_start is the thinking block at index 0. + expect(starts[0].payload.index).toBe(0); + expect(starts[0].payload.content_block?.type).toBe("thinking"); + // The tool_use block must follow at index 1 (shifted by the leading thinking block). + const toolStart = starts.find((f) => f.payload.content_block?.type === "tool_use"); + expect(toolStart).toBeDefined(); + expect(toolStart!.payload.index).toBe(1); + + // Indices must be contiguous and unique across all content_block_start frames. + const indices = starts.map((f) => f.payload.index); + expect(indices).toEqual([...Array(starts.length).keys()]); + }); + + it("block-index ordering: without reasoning, tool_use stays at index 0", async () => { + const res = await postRaw(`/model/${REASONING_MODEL}/invoke-with-response-stream`, { + messages: [{ role: "user", content: [{ type: "text", text: "tool-plain" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }); + + expect(res.status).toBe(200); + const frames = decodeEventStreamFrames(res.body); + const starts = frames.filter((f) => f.payload.type === "content_block_start"); + expect(starts[0].payload.content_block?.type).toBe("tool_use"); + expect(starts[0].payload.index).toBe(0); + }); +}); diff --git a/src/__tests__/cohere-toolonly-reasoning.test.ts b/src/__tests__/cohere-toolonly-reasoning.test.ts new file mode 100644 index 00000000..09e45f97 --- /dev/null +++ b/src/__tests__/cohere-toolonly-reasoning.test.ts @@ -0,0 +1,249 @@ +import { describe, it, expect, afterEach, vi } from "vitest"; +import * as http from "node:http"; +import type { Fixture } from "../types.js"; +import { createServer, type ServerInstance } from "../server.js"; + +// ─── HTTP helper ──────────────────────────────────────────────────────────── + +function post( + url: string, + body: unknown, + extraHeaders: Record = {}, +): Promise<{ status: number; headers: http.IncomingHttpHeaders; body: string }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify(body); + const parsed = new URL(url); + const req = http.request( + { + hostname: parsed.hostname, + port: parsed.port, + path: parsed.pathname, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + ...extraHeaders, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => { + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + body: Buffer.concat(chunks).toString(), + }); + }); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +interface SSEEvent { + event: string; + data: Record; +} + +function parseSSEEvents(body: string): SSEEvent[] { + const events: SSEEvent[] = []; + const blocks = body.split("\n\n").filter((b) => b.trim() !== ""); + for (const block of blocks) { + const lines = block.split("\n"); + let eventType = ""; + let dataStr = ""; + for (const line of lines) { + if (line.startsWith("event: ")) { + eventType = line.slice(7); + } else if (line.startsWith("data: ")) { + dataStr = line.slice(6); + } + } + if (eventType && dataStr) { + events.push({ event: eventType, data: JSON.parse(dataStr) as Record }); + } + } + return events; +} + +interface CohereToolOnlyResponse { + message: { + content: { type: string; text: string }[]; + tool_calls: { function: { name: string } }[]; + }; +} + +/** Extract every text-block string emitted by a non-streaming Cohere response. */ +function nonStreamTexts(rawBody: string): string[] { + const body = JSON.parse(rawBody) as CohereToolOnlyResponse; + return body.message.content.filter((b) => b.type === "text").map((b) => b.text); +} + +/** Tool-call names emitted by a non-streaming Cohere tool-only response. */ +function nonStreamToolNames(rawBody: string): string[] { + const body = JSON.parse(rawBody) as CohereToolOnlyResponse; + return body.message.tool_calls.map((tc) => tc.function.name); +} + +/** Concatenate all content-delta text slices from a streamed Cohere response. */ +function streamDeltaTexts(rawBody: string): string { + const events = parseSSEEvents(rawBody); + return events + .filter((e) => e.event === "content-delta") + .map((e) => { + const delta = e.data.delta as + | { message?: { content?: { type?: string; text?: string } } } + | undefined; + return delta?.message?.content?.text ?? ""; + }) + .join(""); +} + +/** Did the stream emit any tool-call-start event? */ +function streamHasToolCall(rawBody: string): boolean { + return parseSSEEvents(rawBody).some((e) => e.event === "tool-call-start"); +} + +const REASONING = "Let me think step by step about which tool to call."; + +// ─── Fixtures (tool-call-only: toolCalls present, no content) ──────────────── + +const reasoningToolOnlyFixture: Fixture = { + match: { userMessage: "weather-only" }, + response: { + reasoning: REASONING, + toolCalls: [{ name: "get_weather", arguments: '{"city":"SF"}' }], + }, +}; + +const noReasoningToolOnlyFixture: Fixture = { + match: { userMessage: "plain-tool" }, + response: { + toolCalls: [{ name: "get_weather", arguments: '{"city":"SF"}' }], + }, +}; + +const allFixtures: Fixture[] = [reasoningToolOnlyFixture, noReasoningToolOnlyFixture]; + +const COHERE_PATH = "/v2/chat"; + +function chatReq(model: string, userMessage: string, stream = false): unknown { + return { model, stream, messages: [{ role: "user", content: userMessage }] }; +} + +// ─── Tests ────────────────────────────────────────────────────────────────── + +let instance: ServerInstance | null = null; + +afterEach(async () => { + if (instance) { + await new Promise((resolve) => instance!.server.close(() => resolve())); + instance = null; + } + vi.restoreAllMocks(); +}); + +describe("Cohere tool-only reasoning capability gating", () => { + describe("reasoning-capable model", () => { + it("emits reasoning text block alongside the tool call (non-stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await post( + `${instance.url}${COHERE_PATH}`, + chatReq("command-a-reasoning", "weather-only"), + ); + expect(res.status).toBe(200); + expect(nonStreamTexts(res.body)).toContain(REASONING); + expect(nonStreamToolNames(res.body)).toContain("get_weather"); + }); + + it("emits reasoning deltas alongside the tool call (stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await post( + `${instance.url}${COHERE_PATH}`, + chatReq("command-a-reasoning", "weather-only", true), + ); + expect(streamDeltaTexts(res.body)).toContain(REASONING); + expect(streamHasToolCall(res.body)).toBe(true); + }); + }); + + describe("non-reasoning model + reasoning fixture", () => { + it("strict OFF: still emits reasoning and warns (non-stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await post(`${instance.url}${COHERE_PATH}`, chatReq("gpt-4.1", "weather-only")); + expect(res.status).toBe(200); + expect(nonStreamTexts(res.body)).toContain(REASONING); + expect(nonStreamToolNames(res.body)).toContain("get_weather"); + expect(warnSpy).toHaveBeenCalled(); + const warned = warnSpy.mock.calls.flat().join(" "); + expect(warned).toContain("gpt-4.1"); + expect(warned).toContain("not reasoning-capable"); + }); + + it("strict OFF: still emits reasoning deltas + warns (stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await post( + `${instance.url}${COHERE_PATH}`, + chatReq("gpt-4.1", "weather-only", true), + ); + expect(streamDeltaTexts(res.body)).toContain(REASONING); + expect(streamHasToolCall(res.body)).toBe(true); + expect(warnSpy).toHaveBeenCalled(); + }); + + it("strict ON: suppresses reasoning, keeps tool call + logs error (non-stream)", async () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await post(`${instance.url}${COHERE_PATH}`, chatReq("gpt-4.1", "weather-only"), { + "X-AIMock-Strict": "true", + }); + expect(res.status).toBe(200); + expect(nonStreamTexts(res.body)).not.toContain(REASONING); + expect(nonStreamToolNames(res.body)).toContain("get_weather"); + expect(errorSpy).toHaveBeenCalled(); + const errored = errorSpy.mock.calls.flat().join(" "); + expect(errored).toContain("gpt-4.1"); + }); + + it("strict ON: suppresses reasoning deltas, keeps tool call (stream)", async () => { + vi.spyOn(console, "error").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await post( + `${instance.url}${COHERE_PATH}`, + chatReq("gpt-4.1", "weather-only", true), + { "X-AIMock-Strict": "true" }, + ); + expect(streamDeltaTexts(res.body)).not.toContain(REASONING); + expect(streamHasToolCall(res.body)).toBe(true); + }); + }); + + describe("reasoning absent (no-op)", () => { + it("non-reasoning model, no fixture reasoning: tool call only, no warn/error (non-stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await post(`${instance.url}${COHERE_PATH}`, chatReq("gpt-4.1", "plain-tool")); + expect(res.status).toBe(200); + expect(nonStreamTexts(res.body)).toEqual([]); + expect(nonStreamToolNames(res.body)).toContain("get_weather"); + expect(warnSpy).not.toHaveBeenCalled(); + expect(errorSpy).not.toHaveBeenCalled(); + }); + + it("no fixture reasoning: tool call only, no reasoning deltas (stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await post( + `${instance.url}${COHERE_PATH}`, + chatReq("gpt-4.1", "plain-tool", true), + ); + expect(streamDeltaTexts(res.body)).toBe(""); + expect(streamHasToolCall(res.body)).toBe(true); + }); + }); +}); diff --git a/src/__tests__/gemini-audio-reasoning-capability.test.ts b/src/__tests__/gemini-audio-reasoning-capability.test.ts new file mode 100644 index 00000000..610736ff --- /dev/null +++ b/src/__tests__/gemini-audio-reasoning-capability.test.ts @@ -0,0 +1,273 @@ +import { describe, it, expect, afterEach, vi } from "vitest"; +import * as http from "node:http"; +import type { Fixture } from "../types.js"; +import { createServer, type ServerInstance } from "../server.js"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function post( + url: string, + body: unknown, + headers?: Record, +): Promise<{ status: number; headers: http.IncomingHttpHeaders; body: string }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify(body); + const parsed = new URL(url); + const req = http.request( + { + hostname: parsed.hostname, + port: parsed.port, + path: parsed.pathname, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => { + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + body: Buffer.concat(chunks).toString(), + }); + }); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +function parseGeminiSSEChunks(body: string): GeminiChunk[] { + const chunks: GeminiChunk[] = []; + for (const line of body.split("\n")) { + if (line.startsWith("data: ")) { + chunks.push(JSON.parse(line.slice(6)) as GeminiChunk); + } + } + return chunks; +} + +interface GeminiPart { + text?: string; + thought?: boolean; + inlineData?: { mimeType: string; data: string }; + functionCall?: { name: string; args: Record }; +} + +interface GeminiChunk { + candidates: { + content: { role: string; parts: GeminiPart[] }; + finishReason?: string; + index: number; + }[]; +} + +// "thought parts" = parts emitted on the synthesized reasoning channel. +function thoughtParts(parts: GeminiPart[]): GeminiPart[] { + return parts.filter((p) => p.thought === true); +} + +function reasoningInChunks(chunks: GeminiChunk[]): boolean { + return chunks.some((c) => thoughtParts(c.candidates[0].content.parts).length > 0); +} + +function allParts(chunks: GeminiChunk[]): GeminiPart[] { + return chunks.flatMap((c) => c.candidates[0].content.parts); +} + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const REASONING = "User wants audio plus a lookup."; + +// An audio turn carrying a companion reasoning channel, text content, and a +// tool call — the full companion set the recorder can preserve on an audio +// fixture. The capability gate must only suppress the reasoning companion. +const audioReasoningFixture: Fixture = { + match: { userMessage: "audio reasoning" }, + response: { + audio: "SGVsbG8=", + format: "mp3", + content: "Here is the audio you asked for.", + reasoning: REASONING, + toolCalls: [{ id: "call_1", name: "lookup", arguments: '{"query":"weather"}' }], + }, +}; + +// An audio turn with no reasoning channel — gating must be a no-op (no thought +// part, and no warn/error log even under strict). +const audioNoReasoningFixture: Fixture = { + match: { userMessage: "audio plain" }, + response: { audio: "SGVsbG8=", format: "mp3", content: "Just audio." }, +}; + +const allFixtures: Fixture[] = [audioReasoningFixture, audioNoReasoningFixture]; + +const GEN = "generateContent"; +const STREAM_GEN = "streamGenerateContent"; + +function geminiUrl(base: string, model: string, op: string): string { + return `${base}/v1beta/models/${model}:${op}`; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +let instance: ServerInstance | null = null; + +afterEach(async () => { + vi.restoreAllMocks(); + if (instance) { + await new Promise((resolve) => { + instance!.server.close(() => resolve()); + }); + instance = null; + } +}); + +describe("Gemini audio reasoning capability gating (non-streaming)", () => { + it("emits the thought companion for a reasoning-capable model", async () => { + instance = await createServer(allFixtures); + const res = await post(geminiUrl(instance.url, "gemini-2.5-pro", GEN), { + contents: [{ role: "user", parts: [{ text: "audio reasoning" }] }], + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as GeminiChunk; + const parts = body.candidates[0].content.parts; + const thoughts = thoughtParts(parts); + expect(thoughts).toHaveLength(1); + expect(thoughts[0].text).toBe(REASONING); + // Audio companion intact and first. + expect(parts[0].inlineData).toEqual({ mimeType: "audio/mpeg", data: "SGVsbG8=" }); + }); + + it("strict ON: suppresses the thought companion for a non-reasoning model, leaving audio/content/toolCalls intact", async () => { + instance = await createServer(allFixtures); + const res = await post( + geminiUrl(instance.url, "gemini-1.5-flash", GEN), + { contents: [{ role: "user", parts: [{ text: "audio reasoning" }] }] }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as GeminiChunk; + const parts = body.candidates[0].content.parts; + // Reasoning companion suppressed. + expect(thoughtParts(parts)).toHaveLength(0); + // Audio still present and first. + expect(parts[0].inlineData).toEqual({ mimeType: "audio/mpeg", data: "SGVsbG8=" }); + // Content companion intact. + expect(parts.some((p) => p.text === "Here is the audio you asked for." && !p.thought)).toBe( + true, + ); + // Tool call companion intact + FUNCTION_CALL finish reason. + const fc = parts.find((p) => p.functionCall); + expect(fc?.functionCall?.name).toBe("lookup"); + expect(body.candidates[0].finishReason).toBe("FUNCTION_CALL"); + }); + + it("strict OFF: emits the thought companion for a non-reasoning model but warns", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { logLevel: "warn" }); + const res = await post(geminiUrl(instance.url, "gemini-1.5-flash", GEN), { + contents: [{ role: "user", parts: [{ text: "audio reasoning" }] }], + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as GeminiChunk; + expect(thoughtParts(body.candidates[0].content.parts)).toHaveLength(1); + expect(warn).toHaveBeenCalled(); + expect(warn.mock.calls.flat().join(" ")).toContain("gemini-1.5-flash"); + }); + + it("emits the thought companion for an unknown model (defaults to capable)", async () => { + instance = await createServer(allFixtures); + const res = await post(geminiUrl(instance.url, "lyria-3", GEN), { + contents: [{ role: "user", parts: [{ text: "audio reasoning" }] }], + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as GeminiChunk; + expect(thoughtParts(body.candidates[0].content.parts)).toHaveLength(1); + }); + + it("no-op when the audio fixture carries no reasoning (no thought part, no log)", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + instance = await createServer(allFixtures, { logLevel: "warn" }); + const res = await post( + geminiUrl(instance.url, "gemini-1.5-flash", GEN), + { contents: [{ role: "user", parts: [{ text: "audio plain" }] }] }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as GeminiChunk; + const parts = body.candidates[0].content.parts; + expect(thoughtParts(parts)).toHaveLength(0); + expect(parts[0].inlineData).toEqual({ mimeType: "audio/mpeg", data: "SGVsbG8=" }); + expect(warn).not.toHaveBeenCalled(); + expect(error).not.toHaveBeenCalled(); + }); +}); + +describe("Gemini audio reasoning capability gating (streaming)", () => { + it("emits the thought companion chunk for a reasoning-capable model", async () => { + instance = await createServer(allFixtures); + const res = await post(geminiUrl(instance.url, "gemini-2.5-pro", STREAM_GEN), { + contents: [{ role: "user", parts: [{ text: "audio reasoning" }] }], + }); + + expect(res.status).toBe(200); + const chunks = parseGeminiSSEChunks(res.body); + expect(reasoningInChunks(chunks)).toBe(true); + }); + + it("strict ON: suppresses the thought companion chunk for a non-reasoning model, leaving audio/content/toolCalls intact", async () => { + instance = await createServer(allFixtures); + const res = await post( + geminiUrl(instance.url, "gemini-1.5-flash", STREAM_GEN), + { contents: [{ role: "user", parts: [{ text: "audio reasoning" }] }] }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const chunks = parseGeminiSSEChunks(res.body); + const parts = allParts(chunks); + // Reasoning companion suppressed. + expect(reasoningInChunks(chunks)).toBe(false); + // Audio still present. + expect(parts.some((p) => p.inlineData)).toBe(true); + // Content companion intact. + expect(parts.some((p) => p.text === "Here is the audio you asked for." && !p.thought)).toBe( + true, + ); + // Tool call companion intact + FUNCTION_CALL finish reason. + expect(parts.some((p) => p.functionCall?.name === "lookup")).toBe(true); + expect(chunks.some((c) => c.candidates[0].finishReason === "FUNCTION_CALL")).toBe(true); + }); + + it("strict OFF: emits the thought companion chunk for a non-reasoning model and warns", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { logLevel: "warn" }); + const res = await post(geminiUrl(instance.url, "gemini-1.5-flash", STREAM_GEN), { + contents: [{ role: "user", parts: [{ text: "audio reasoning" }] }], + }); + + expect(res.status).toBe(200); + const chunks = parseGeminiSSEChunks(res.body); + expect(reasoningInChunks(chunks)).toBe(true); + expect(warn).toHaveBeenCalled(); + }); +}); diff --git a/src/__tests__/gemini-toolonly-reasoning-capability.test.ts b/src/__tests__/gemini-toolonly-reasoning-capability.test.ts new file mode 100644 index 00000000..08f11ba7 --- /dev/null +++ b/src/__tests__/gemini-toolonly-reasoning-capability.test.ts @@ -0,0 +1,260 @@ +import { describe, it, expect, afterEach, vi } from "vitest"; +import * as http from "node:http"; +import type { Fixture } from "../types.js"; +import { createServer, type ServerInstance } from "../server.js"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function post( + url: string, + body: unknown, + headers?: Record, +): Promise<{ status: number; headers: http.IncomingHttpHeaders; body: string }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify(body); + const parsed = new URL(url); + const req = http.request( + { + hostname: parsed.hostname, + port: parsed.port, + path: parsed.pathname, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => { + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + body: Buffer.concat(chunks).toString(), + }); + }); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +function parseGeminiSSEChunks(body: string): GeminiChunk[] { + const chunks: GeminiChunk[] = []; + for (const line of body.split("\n")) { + if (line.startsWith("data: ")) { + chunks.push(JSON.parse(line.slice(6)) as GeminiChunk); + } + } + return chunks; +} + +interface GeminiPart { + text?: string; + thought?: boolean; + functionCall?: { name: string; args: Record }; +} + +interface GeminiChunk { + candidates: { + content: { role: string; parts: GeminiPart[] }; + finishReason?: string; + index: number; + }[]; +} + +// "thought parts" = parts emitted on the synthesized reasoning channel. +function thoughtParts(parts: GeminiPart[]): GeminiPart[] { + return parts.filter((p) => p.thought === true); +} + +function reasoningInChunks(chunks: GeminiChunk[]): boolean { + return chunks.some((c) => thoughtParts(c.candidates[0].content.parts).length > 0); +} + +function functionCallInChunks(chunks: GeminiChunk[]): boolean { + return chunks.some((c) => c.candidates[0].content.parts.some((p) => p.functionCall)); +} + +// --------------------------------------------------------------------------- +// Fixtures — tool-only (no content) responses +// --------------------------------------------------------------------------- + +const REASONING = "Let me think about which tool to call."; + +const toolOnlyReasoningFixture: Fixture = { + match: { userMessage: "call-with-reasoning" }, + response: { + reasoning: REASONING, + toolCalls: [{ name: "get_weather", arguments: '{"city":"NYC"}' }], + }, +}; + +const toolOnlyNoReasoningFixture: Fixture = { + match: { userMessage: "call-no-reasoning" }, + response: { + toolCalls: [{ name: "get_weather", arguments: '{"city":"NYC"}' }], + }, +}; + +const allFixtures: Fixture[] = [toolOnlyReasoningFixture, toolOnlyNoReasoningFixture]; + +const GEN = "generateContent"; +const STREAM_GEN = "streamGenerateContent"; + +function geminiUrl(base: string, model: string, op: string): string { + return `${base}/v1beta/models/${model}:${op}`; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +let instance: ServerInstance | null = null; + +afterEach(async () => { + vi.restoreAllMocks(); + if (instance) { + await new Promise((resolve) => { + instance!.server.close(() => resolve()); + }); + instance = null; + } +}); + +describe("Gemini tool-only reasoning capability gating (non-streaming)", () => { + it("emits a thought part for a reasoning-capable model, tool call intact", async () => { + instance = await createServer(allFixtures); + const res = await post(geminiUrl(instance.url, "gemini-2.5-pro", GEN), { + contents: [{ role: "user", parts: [{ text: "call-with-reasoning" }] }], + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as GeminiChunk; + const parts = body.candidates[0].content.parts; + const thoughts = thoughtParts(parts); + expect(thoughts).toHaveLength(1); + expect(thoughts[0].text).toBe(REASONING); + // Tool call still present. + expect(parts.some((p) => p.functionCall?.name === "get_weather")).toBe(true); + }); + + it("strict ON: suppresses thought for a non-reasoning model, tool call intact", async () => { + instance = await createServer(allFixtures); + const res = await post( + geminiUrl(instance.url, "gemini-1.5-flash", GEN), + { contents: [{ role: "user", parts: [{ text: "call-with-reasoning" }] }] }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as GeminiChunk; + const parts = body.candidates[0].content.parts; + expect(thoughtParts(parts)).toHaveLength(0); + expect(parts.some((p) => p.functionCall?.name === "get_weather")).toBe(true); + }); + + it("strict OFF: emits thought for a non-reasoning model but warns", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { logLevel: "warn" }); + const res = await post(geminiUrl(instance.url, "gemini-1.5-flash", GEN), { + contents: [{ role: "user", parts: [{ text: "call-with-reasoning" }] }], + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as GeminiChunk; + expect(thoughtParts(body.candidates[0].content.parts)).toHaveLength(1); + expect(warn).toHaveBeenCalled(); + expect(warn.mock.calls.flat().join(" ")).toContain("gemini-1.5-flash"); + }); + + it("emits a thought part for an unknown model (defaults to capable)", async () => { + instance = await createServer(allFixtures); + const res = await post(geminiUrl(instance.url, "some-future-model", GEN), { + contents: [{ role: "user", parts: [{ text: "call-with-reasoning" }] }], + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as GeminiChunk; + expect(thoughtParts(body.candidates[0].content.parts)).toHaveLength(1); + }); + + it("no-op when fixture carries no reasoning (no thought parts, no log)", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + instance = await createServer(allFixtures, { logLevel: "warn" }); + const res = await post( + geminiUrl(instance.url, "gemini-1.5-flash", GEN), + { contents: [{ role: "user", parts: [{ text: "call-no-reasoning" }] }] }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as GeminiChunk; + const parts = body.candidates[0].content.parts; + expect(thoughtParts(parts)).toHaveLength(0); + expect(parts.some((p) => p.functionCall?.name === "get_weather")).toBe(true); + expect(warn).not.toHaveBeenCalled(); + expect(error).not.toHaveBeenCalled(); + }); +}); + +describe("Gemini tool-only reasoning capability gating (streaming)", () => { + it("emits thought chunks for a reasoning-capable model, function_call intact", async () => { + instance = await createServer(allFixtures); + const res = await post(geminiUrl(instance.url, "gemini-2.5-pro", STREAM_GEN), { + contents: [{ role: "user", parts: [{ text: "call-with-reasoning" }] }], + }); + + expect(res.status).toBe(200); + const chunks = parseGeminiSSEChunks(res.body); + expect(reasoningInChunks(chunks)).toBe(true); + expect(functionCallInChunks(chunks)).toBe(true); + }); + + it("strict ON: suppresses thought chunks for a non-reasoning model, function_call intact", async () => { + instance = await createServer(allFixtures); + const res = await post( + geminiUrl(instance.url, "gemini-1.5-flash", STREAM_GEN), + { contents: [{ role: "user", parts: [{ text: "call-with-reasoning" }] }] }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const chunks = parseGeminiSSEChunks(res.body); + expect(reasoningInChunks(chunks)).toBe(false); + expect(functionCallInChunks(chunks)).toBe(true); + }); + + it("strict OFF: emits thought chunks for a non-reasoning model and warns", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { logLevel: "warn" }); + const res = await post(geminiUrl(instance.url, "gemini-1.5-flash", STREAM_GEN), { + contents: [{ role: "user", parts: [{ text: "call-with-reasoning" }] }], + }); + + expect(res.status).toBe(200); + const chunks = parseGeminiSSEChunks(res.body); + expect(reasoningInChunks(chunks)).toBe(true); + expect(functionCallInChunks(chunks)).toBe(true); + expect(warn).toHaveBeenCalled(); + }); + + it("no-op when fixture carries no reasoning (no thought chunks)", async () => { + instance = await createServer(allFixtures); + const res = await post(geminiUrl(instance.url, "gemini-2.5-pro", STREAM_GEN), { + contents: [{ role: "user", parts: [{ text: "call-no-reasoning" }] }], + }); + + expect(res.status).toBe(200); + const chunks = parseGeminiSSEChunks(res.body); + expect(reasoningInChunks(chunks)).toBe(false); + expect(functionCallInChunks(chunks)).toBe(true); + }); +}); diff --git a/src/__tests__/openai-chat-toolonly-reasoning-capability.test.ts b/src/__tests__/openai-chat-toolonly-reasoning-capability.test.ts new file mode 100644 index 00000000..8fcbebe8 --- /dev/null +++ b/src/__tests__/openai-chat-toolonly-reasoning-capability.test.ts @@ -0,0 +1,250 @@ +import { describe, it, expect, afterEach, vi } from "vitest"; +import * as http from "node:http"; +import type { Fixture } from "../types.js"; +import { createServer, type ServerInstance } from "../server.js"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +async function httpPost( + url: string, + body: unknown, + headers?: Record, +): Promise<{ status: number; body: string }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify(body); + const req = http.request( + url, + { + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => + resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString() }), + ); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +interface ChatSSEEvent { + choices?: { + delta?: { + content?: string; + reasoning_content?: string; + role?: string; + tool_calls?: { + index: number; + id?: string; + type?: string; + function?: { name?: string; arguments?: string }; + }[]; + }; + finish_reason?: string | null; + }[]; + [key: string]: unknown; +} + +function parseSSEEvents(body: string): ChatSSEEvent[] { + const events: ChatSSEEvent[] = []; + for (const line of body.split("\n")) { + if (line.startsWith("data: ") && line.slice(6).trim() !== "[DONE]") { + events.push(JSON.parse(line.slice(6)) as ChatSSEEvent); + } + } + return events; +} + +/** Non-stream: does the assistant message carry a reasoning_content field? */ +function nonStreamReasoning(body: string): string | undefined { + const parsed = JSON.parse(body) as { + choices?: { message?: { reasoning_content?: string } }[]; + }; + return parsed.choices?.[0]?.message?.reasoning_content; +} + +/** Non-stream: the name of the first tool call in the assistant message. */ +function nonStreamToolCallName(body: string): string | undefined { + const parsed = JSON.parse(body) as { + choices?: { message?: { tool_calls?: { function?: { name?: string } }[] } }[]; + }; + return parsed.choices?.[0]?.message?.tool_calls?.[0]?.function?.name; +} + +/** Stream: are there any reasoning_content deltas? */ +function hasReasoningStreamDeltas(body: string): boolean { + return parseSSEEvents(body).some((e) => e.choices?.[0]?.delta?.reasoning_content !== undefined); +} + +/** Stream: the name of the first tool call seen across deltas. */ +function streamToolCallName(body: string): string | undefined { + for (const e of parseSSEEvents(body)) { + const name = e.choices?.[0]?.delta?.tool_calls?.[0]?.function?.name; + if (name) return name; + } + return undefined; +} + +// --------------------------------------------------------------------------- +// Fixtures — tool-only responses (no content) +// --------------------------------------------------------------------------- + +const toolOnlyReasoningFixture: Fixture = { + match: { userMessage: "think-tool" }, + response: { + toolCalls: [{ name: "get_weather", arguments: '{"city":"SF"}' }], + reasoning: "Let me reason step by step.", + }, +}; + +const toolOnlyNoReasoningFixture: Fixture = { + match: { userMessage: "plain-tool" }, + response: { + toolCalls: [{ name: "get_weather", arguments: '{"city":"SF"}' }], + }, +}; + +const allFixtures: Fixture[] = [toolOnlyReasoningFixture, toolOnlyNoReasoningFixture]; + +// --------------------------------------------------------------------------- +// Server lifecycle +// --------------------------------------------------------------------------- + +let instance: ServerInstance | null = null; + +afterEach(async () => { + vi.restoreAllMocks(); + if (instance) { + await new Promise((resolve) => instance!.server.close(() => resolve())); + instance = null; + } +}); + +function chatBody(userMessage: string, model: string, stream: boolean) { + return { + model, + messages: [{ role: "user", content: userMessage }], + stream, + }; +} + +// --------------------------------------------------------------------------- +// Tests — tool-only response path reasoning capability gating +// --------------------------------------------------------------------------- + +describe("/v1/chat/completions tool-only reasoning capability gating", () => { + it("reasoning-capable model o3-mini: emits reasoning, tool_call intact, no warn (non-stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/chat/completions`, + chatBody("think-tool", "o3-mini", false), + ); + expect(res.status).toBe(200); + expect(nonStreamReasoning(res.body)).toBe("Let me reason step by step."); + expect(nonStreamToolCallName(res.body)).toBe("get_weather"); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it("reasoning-capable model o3-mini: emits reasoning, tool_call intact, no warn (stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/chat/completions`, + chatBody("think-tool", "o3-mini", true), + ); + expect(res.status).toBe(200); + expect(hasReasoningStreamDeltas(res.body)).toBe(true); + expect(streamToolCallName(res.body)).toBe("get_weather"); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it("non-reasoning model gpt-4.1, strict OFF: still emits reasoning but warns (non-stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/chat/completions`, + chatBody("think-tool", "gpt-4.1", false), + ); + expect(res.status).toBe(200); + expect(nonStreamReasoning(res.body)).toBe("Let me reason step by step."); + expect(nonStreamToolCallName(res.body)).toBe("get_weather"); + expect(warnSpy).toHaveBeenCalled(); + expect(warnSpy.mock.calls.flat().join(" ")).toContain("gpt-4.1"); + }); + + it("non-reasoning model gpt-4.1, strict OFF: still emits reasoning but warns (stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/chat/completions`, + chatBody("think-tool", "gpt-4.1", true), + ); + expect(res.status).toBe(200); + expect(hasReasoningStreamDeltas(res.body)).toBe(true); + expect(streamToolCallName(res.body)).toBe("get_weather"); + expect(warnSpy).toHaveBeenCalled(); + expect(warnSpy.mock.calls.flat().join(" ")).toContain("gpt-4.1"); + }); + + it("non-reasoning model gpt-4.1, strict ON (header): suppresses reasoning, tool_call intact (non-stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await httpPost( + `${instance.url}/v1/chat/completions`, + chatBody("think-tool", "gpt-4.1", false), + { "X-AIMock-Strict": "true" }, + ); + expect(res.status).toBe(200); + expect(nonStreamReasoning(res.body)).toBeUndefined(); + expect(nonStreamToolCallName(res.body)).toBe("get_weather"); + }); + + it("non-reasoning model gpt-4.1, strict ON (header): suppresses reasoning, tool_call intact (stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await httpPost( + `${instance.url}/v1/chat/completions`, + chatBody("think-tool", "gpt-4.1", true), + { "X-AIMock-Strict": "true" }, + ); + expect(res.status).toBe(200); + expect(hasReasoningStreamDeltas(res.body)).toBe(false); + expect(streamToolCallName(res.body)).toBe("get_weather"); + }); + + it("fixture with no reasoning, gpt-4.1: no-op, no warn, tool_call intact (non-stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/chat/completions`, + chatBody("plain-tool", "gpt-4.1", false), + ); + expect(res.status).toBe(200); + expect(nonStreamReasoning(res.body)).toBeUndefined(); + expect(nonStreamToolCallName(res.body)).toBe("get_weather"); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it("fixture with no reasoning, gpt-4.1: no-op, no warn, tool_call intact (stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/chat/completions`, + chatBody("plain-tool", "gpt-4.1", true), + ); + expect(res.status).toBe(200); + expect(hasReasoningStreamDeltas(res.body)).toBe(false); + expect(streamToolCallName(res.body)).toBe("get_weather"); + expect(warnSpy).not.toHaveBeenCalled(); + }); +}); diff --git a/src/__tests__/reasoning-capability-ollama-tools.test.ts b/src/__tests__/reasoning-capability-ollama-tools.test.ts new file mode 100644 index 00000000..0fcbd28f --- /dev/null +++ b/src/__tests__/reasoning-capability-ollama-tools.test.ts @@ -0,0 +1,386 @@ +import { describe, it, expect, afterEach, vi } from "vitest"; +import http from "node:http"; +import { createServer, type ServerInstance } from "../server.js"; +import type { Fixture } from "../types.js"; + +// --------------------------------------------------------------------------- +// Reasoning capability gating on the Ollama /api/chat tool-bearing paths: +// - content + toolCalls (combined response) +// - tool-only (toolCalls without content) +// +// Mirrors reasoning-capability-ollama.test.ts (TEXT path) but exercises the +// tool builders, which previously dropped reasoning entirely. Non-reasoning +// ids are forced via AIMOCK_NONREASONING_MODELS (same mechanism the TEXT +// suite uses); unknown local ids fail open to reasoning-capable. +// --------------------------------------------------------------------------- + +async function httpPost( + url: string, + body: object, + headers?: Record, +): Promise<{ status: number; body: string }> { + return new Promise((resolve, reject) => { + const req = http.request( + url, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => + resolve({ + status: res.statusCode!, + body: Buffer.concat(chunks).toString(), + }), + ); + }, + ); + req.on("error", reject); + req.write(JSON.stringify(body)); + req.end(); + }); +} + +function parseNDJSON(body: string): Array> { + return body + .split("\n") + .filter((line) => line.trim() !== "") + .map((line) => JSON.parse(line) as Record); +} + +const REASONING = "Let me reason about which tool to call."; + +// Match on userMessage only so any requested model id routes to the same +// reasoning-bearing fixture; the model id drives the capability gate. +const fixtures: Fixture[] = [ + { + // content + tool calls, with reasoning + match: { userMessage: "combo" }, + response: { + content: "Calling the tool now.", + reasoning: REASONING, + toolCalls: [{ name: "get_weather", arguments: '{"city":"SF"}' }], + }, + }, + { + // content + tool calls, no reasoning (no-op). Disjoint match string (not a + // substring of "combo") so the router does not fall through to the + // reasoning-bearing fixture above. + match: { userMessage: "alphacontent" }, + response: { + content: "Calling the tool now.", + toolCalls: [{ name: "get_weather", arguments: '{"city":"SF"}' }], + }, + }, + { + // tool-only, with reasoning + match: { userMessage: "toolonly" }, + response: { + reasoning: REASONING, + toolCalls: [{ name: "get_weather", arguments: '{"city":"SF"}' }], + }, + }, + { + // tool-only, no reasoning (no-op). Disjoint match string (not a substring + // of "toolonly"). + match: { userMessage: "betatool" }, + response: { + toolCalls: [{ name: "get_weather", arguments: '{"city":"SF"}' }], + }, + }, +]; + +let instance: ServerInstance | null = null; + +afterEach(async () => { + vi.unstubAllEnvs(); + if (instance) { + await new Promise((resolve) => { + instance!.server.close(() => resolve()); + }); + instance = null; + } +}); + +// --------------------------------------------------------------------------- +// content + toolCalls — reasoning-capable model emits reasoning + tool_calls +// --------------------------------------------------------------------------- + +describe("Ollama /api/chat content+tool reasoning gating — capable model", () => { + it("non-streaming: emits reasoning_content alongside content + tool_calls", async () => { + instance = await createServer(fixtures); + const res = await httpPost(`${instance.url}/api/chat`, { + model: "deepseek-r1", + messages: [{ role: "user", content: "combo" }], + stream: false, + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.message.content).toBe("Calling the tool now."); + expect(body.message.reasoning_content).toBe(REASONING); + expect(body.message.tool_calls).toHaveLength(1); + expect(body.message.tool_calls[0].function.name).toBe("get_weather"); + }); + + it("streaming: emits reasoning_content chunks alongside content + tool_calls", async () => { + instance = await createServer(fixtures); + const res = await httpPost(`${instance.url}/api/chat`, { + model: "deepseek-r1", + messages: [{ role: "user", content: "combo" }], + stream: true, + }); + + expect(res.status).toBe(200); + const chunks = parseNDJSON(res.body) as Array<{ + message: { + content?: string; + reasoning_content?: string; + tool_calls?: Array<{ function: { name: string } }>; + }; + done: boolean; + }>; + const reasoningChunks = chunks.filter( + (c) => !c.done && c.message?.reasoning_content !== undefined, + ); + expect(reasoningChunks.length).toBeGreaterThan(0); + expect(reasoningChunks.map((c) => c.message.reasoning_content).join("")).toBe(REASONING); + + const toolChunk = chunks.find((c) => c.message?.tool_calls !== undefined); + expect(toolChunk).toBeDefined(); + expect(toolChunk!.message.tool_calls![0].function.name).toBe("get_weather"); + }); +}); + +describe("Ollama /api/chat content+tool reasoning gating — non-reasoning model", () => { + it("strict ON: suppresses reasoning_content but keeps content + tool_calls (non-streaming)", async () => { + vi.stubEnv("AIMOCK_NONREASONING_MODELS", "no-think-model"); + instance = await createServer(fixtures); + const res = await httpPost( + `${instance.url}/api/chat`, + { + model: "no-think-model", + messages: [{ role: "user", content: "combo" }], + stream: false, + }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.message.content).toBe("Calling the tool now."); + expect(body.message.reasoning_content).toBeUndefined(); + expect(body.message.tool_calls).toHaveLength(1); + }); + + it("strict ON: suppresses reasoning_content chunks but keeps tool_calls (streaming)", async () => { + vi.stubEnv("AIMOCK_NONREASONING_MODELS", "no-think-model"); + instance = await createServer(fixtures); + const res = await httpPost( + `${instance.url}/api/chat`, + { + model: "no-think-model", + messages: [{ role: "user", content: "combo" }], + stream: true, + }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const chunks = parseNDJSON(res.body) as Array<{ + message: { reasoning_content?: string; tool_calls?: unknown[] }; + done: boolean; + }>; + const reasoningChunks = chunks.filter( + (c) => !c.done && c.message?.reasoning_content !== undefined, + ); + expect(reasoningChunks.length).toBe(0); + const toolChunk = chunks.find((c) => c.message?.tool_calls !== undefined); + expect(toolChunk).toBeDefined(); + }); + + it("strict OFF: still emits reasoning_content (warn-by-default)", async () => { + vi.stubEnv("AIMOCK_NONREASONING_MODELS", "no-think-model"); + instance = await createServer(fixtures); + const res = await httpPost(`${instance.url}/api/chat`, { + model: "no-think-model", + messages: [{ role: "user", content: "combo" }], + stream: false, + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.message.reasoning_content).toBe(REASONING); + }); +}); + +describe("Ollama /api/chat content+tool reasoning gating — no fixture reasoning", () => { + it("no reasoning_content when fixture has no reasoning (non-streaming)", async () => { + instance = await createServer(fixtures); + const res = await httpPost(`${instance.url}/api/chat`, { + model: "deepseek-r1", + messages: [{ role: "user", content: "alphacontent" }], + stream: false, + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.message.content).toBe("Calling the tool now."); + expect(body.message.reasoning_content).toBeUndefined(); + expect(body.message.tool_calls).toHaveLength(1); + }); + + it("no reasoning_content when fixture has no reasoning (streaming)", async () => { + instance = await createServer(fixtures); + const res = await httpPost(`${instance.url}/api/chat`, { + model: "deepseek-r1", + messages: [{ role: "user", content: "alphacontent" }], + stream: true, + }); + + expect(res.status).toBe(200); + const chunks = parseNDJSON(res.body) as Array<{ + message: { reasoning_content?: string }; + done: boolean; + }>; + const reasoningChunks = chunks.filter( + (c) => !c.done && c.message?.reasoning_content !== undefined, + ); + expect(reasoningChunks.length).toBe(0); + }); +}); + +// --------------------------------------------------------------------------- +// tool-only — reasoning gating on the tool-call-only path +// --------------------------------------------------------------------------- + +describe("Ollama /api/chat tool-only reasoning gating — capable model", () => { + it("non-streaming: emits reasoning_content alongside tool_calls", async () => { + instance = await createServer(fixtures); + const res = await httpPost(`${instance.url}/api/chat`, { + model: "deepseek-r1", + messages: [{ role: "user", content: "toolonly" }], + stream: false, + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.message.reasoning_content).toBe(REASONING); + expect(body.message.tool_calls).toHaveLength(1); + expect(body.message.tool_calls[0].function.name).toBe("get_weather"); + }); + + it("streaming: emits reasoning_content chunks alongside tool_calls", async () => { + instance = await createServer(fixtures); + const res = await httpPost(`${instance.url}/api/chat`, { + model: "deepseek-r1", + messages: [{ role: "user", content: "toolonly" }], + stream: true, + }); + + expect(res.status).toBe(200); + const chunks = parseNDJSON(res.body) as Array<{ + message: { + reasoning_content?: string; + tool_calls?: Array<{ function: { name: string } }>; + }; + done: boolean; + }>; + const reasoningChunks = chunks.filter( + (c) => !c.done && c.message?.reasoning_content !== undefined, + ); + expect(reasoningChunks.length).toBeGreaterThan(0); + expect(reasoningChunks.map((c) => c.message.reasoning_content).join("")).toBe(REASONING); + + const toolChunk = chunks.find((c) => c.message?.tool_calls !== undefined); + expect(toolChunk).toBeDefined(); + expect(toolChunk!.message.tool_calls![0].function.name).toBe("get_weather"); + }); +}); + +describe("Ollama /api/chat tool-only reasoning gating — non-reasoning model", () => { + it("strict ON: suppresses reasoning_content but keeps tool_calls (non-streaming)", async () => { + vi.stubEnv("AIMOCK_NONREASONING_MODELS", "no-think-model"); + instance = await createServer(fixtures); + const res = await httpPost( + `${instance.url}/api/chat`, + { + model: "no-think-model", + messages: [{ role: "user", content: "toolonly" }], + stream: false, + }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.message.reasoning_content).toBeUndefined(); + expect(body.message.tool_calls).toHaveLength(1); + }); + + it("strict ON: suppresses reasoning_content chunks but keeps tool_calls (streaming)", async () => { + vi.stubEnv("AIMOCK_NONREASONING_MODELS", "no-think-model"); + instance = await createServer(fixtures); + const res = await httpPost( + `${instance.url}/api/chat`, + { + model: "no-think-model", + messages: [{ role: "user", content: "toolonly" }], + stream: true, + }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const chunks = parseNDJSON(res.body) as Array<{ + message: { reasoning_content?: string; tool_calls?: unknown[] }; + done: boolean; + }>; + const reasoningChunks = chunks.filter( + (c) => !c.done && c.message?.reasoning_content !== undefined, + ); + expect(reasoningChunks.length).toBe(0); + const toolChunk = chunks.find((c) => c.message?.tool_calls !== undefined); + expect(toolChunk).toBeDefined(); + }); +}); + +describe("Ollama /api/chat tool-only reasoning gating — no fixture reasoning", () => { + it("no reasoning_content when fixture has no reasoning (non-streaming)", async () => { + instance = await createServer(fixtures); + const res = await httpPost(`${instance.url}/api/chat`, { + model: "deepseek-r1", + messages: [{ role: "user", content: "betatool" }], + stream: false, + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.message.reasoning_content).toBeUndefined(); + expect(body.message.tool_calls).toHaveLength(1); + }); + + it("no reasoning_content when fixture has no reasoning (streaming)", async () => { + instance = await createServer(fixtures); + const res = await httpPost(`${instance.url}/api/chat`, { + model: "deepseek-r1", + messages: [{ role: "user", content: "betatool" }], + stream: true, + }); + + expect(res.status).toBe(200); + const chunks = parseNDJSON(res.body) as Array<{ + message: { reasoning_content?: string }; + done: boolean; + }>; + const reasoningChunks = chunks.filter( + (c) => !c.done && c.message?.reasoning_content !== undefined, + ); + expect(reasoningChunks.length).toBe(0); + }); +}); diff --git a/src/__tests__/responses-toolonly-reasoning-capability.test.ts b/src/__tests__/responses-toolonly-reasoning-capability.test.ts new file mode 100644 index 00000000..cdac2333 --- /dev/null +++ b/src/__tests__/responses-toolonly-reasoning-capability.test.ts @@ -0,0 +1,282 @@ +import { describe, it, expect, afterEach, vi } from "vitest"; +import * as http from "node:http"; +import type { Fixture } from "../types.js"; +import { createServer, type ServerInstance } from "../server.js"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +async function httpPost( + url: string, + body: unknown, + headers?: Record, +): Promise<{ status: number; body: string }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify(body); + const req = http.request( + url, + { + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => + resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString() }), + ); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +interface ResponsesSSEEvent { + type?: string; + output_index?: number; + item?: { type?: string }; + [key: string]: unknown; +} + +function parseSSEEvents(body: string): ResponsesSSEEvent[] { + const events: ResponsesSSEEvent[] = []; + for (const line of body.split("\n")) { + if (line.startsWith("data: ") && line.slice(6).trim() !== "[DONE]") { + events.push(JSON.parse(line.slice(6)) as ResponsesSSEEvent); + } + } + return events; +} + +/** Non-stream: does the response output[] contain a reasoning item? */ +function hasReasoningOutputItem(body: string): boolean { + const parsed = JSON.parse(body) as { output?: Array<{ type?: string }> }; + return (parsed.output ?? []).some((item) => item.type === "reasoning"); +} + +/** Non-stream: does the response output[] contain a function_call item? */ +function hasFunctionCallOutputItem(body: string): boolean { + const parsed = JSON.parse(body) as { output?: Array<{ type?: string }> }; + return (parsed.output ?? []).some((item) => item.type === "function_call"); +} + +/** Stream: are there any reasoning summary events? */ +function hasReasoningStreamEvents(body: string): boolean { + return parseSSEEvents(body).some( + (e) => typeof e.type === "string" && e.type.startsWith("response.reasoning_summary_text"), + ); +} + +/** Stream: are there any function_call argument events? */ +function hasFunctionCallStreamEvents(body: string): boolean { + return parseSSEEvents(body).some( + (e) => typeof e.type === "string" && e.type.startsWith("response.function_call_arguments"), + ); +} + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const toolOnlyReasoningFixture: Fixture = { + match: { userMessage: "tool-only-think" }, + response: { + reasoning: "Deciding which tool to call.", + toolCalls: [{ name: "lookup", arguments: '{"q":"x"}' }], + }, +}; + +const toolOnlyNoReasoningFixture: Fixture = { + match: { userMessage: "tool-only-plain" }, + response: { + toolCalls: [{ name: "lookup", arguments: '{"q":"x"}' }], + }, +}; + +const allFixtures: Fixture[] = [toolOnlyReasoningFixture, toolOnlyNoReasoningFixture]; + +// --------------------------------------------------------------------------- +// Server lifecycle +// --------------------------------------------------------------------------- + +let instance: ServerInstance | null = null; + +afterEach(async () => { + vi.restoreAllMocks(); + if (instance) { + await new Promise((resolve) => instance!.server.close(() => resolve())); + instance = null; + } +}); + +function responsesBody(userMessage: string, model: string, stream: boolean) { + return { + model, + input: [{ role: "user", content: userMessage }], + stream, + }; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe("Responses API reasoning capability gating — tool-call-only response", () => { + it("reasoning-capable model emits reasoning (non-stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("tool-only-think", "o3", false), + ); + expect(res.status).toBe(200); + expect(hasReasoningOutputItem(res.body)).toBe(true); + expect(hasFunctionCallOutputItem(res.body)).toBe(true); + }); + + it("reasoning-capable model emits reasoning (stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("tool-only-think", "o3", true), + ); + expect(res.status).toBe(200); + expect(hasReasoningStreamEvents(res.body)).toBe(true); + expect(hasFunctionCallStreamEvents(res.body)).toBe(true); + }); + + it("non-reasoning model, strict OFF: still emits reasoning but warns (non-stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("tool-only-think", "gpt-4.1", false), + ); + expect(res.status).toBe(200); + expect(hasReasoningOutputItem(res.body)).toBe(true); + expect(warnSpy).toHaveBeenCalled(); + expect(warnSpy.mock.calls.flat().join(" ")).toContain("gpt-4.1"); + }); + + it("non-reasoning model, strict OFF: still emits reasoning but warns (stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("tool-only-think", "gpt-4.1", true), + ); + expect(res.status).toBe(200); + expect(hasReasoningStreamEvents(res.body)).toBe(true); + expect(warnSpy).toHaveBeenCalled(); + }); + + it("non-reasoning model, strict ON (header): suppresses reasoning (non-stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("tool-only-think", "gpt-4.1", false), + { "X-AIMock-Strict": "true" }, + ); + expect(res.status).toBe(200); + expect(hasReasoningOutputItem(res.body)).toBe(false); + // Tool calls must still be present when reasoning is suppressed. + expect(hasFunctionCallOutputItem(res.body)).toBe(true); + }); + + it("non-reasoning model, strict ON (header): suppresses reasoning (stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("tool-only-think", "gpt-4.1", true), + { "X-AIMock-Strict": "true" }, + ); + expect(res.status).toBe(200); + expect(hasReasoningStreamEvents(res.body)).toBe(false); + expect(hasFunctionCallStreamEvents(res.body)).toBe(true); + }); + + it("non-reasoning model, server --strict: suppresses reasoning (non-stream)", async () => { + instance = await createServer(allFixtures, { port: 0, strict: true }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("tool-only-think", "gpt-4.1", false), + ); + expect(res.status).toBe(200); + expect(hasReasoningOutputItem(res.body)).toBe(false); + expect(hasFunctionCallOutputItem(res.body)).toBe(true); + }); + + it("no fixture reasoning: no-op, no warn even for non-reasoning model (non-stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("tool-only-plain", "gpt-4.1", false), + ); + expect(res.status).toBe(200); + expect(hasReasoningOutputItem(res.body)).toBe(false); + expect(hasFunctionCallOutputItem(res.body)).toBe(true); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + // ─────────────────────────────────────────────────────────────────────── + // output_index ordering: a leading reasoning item occupies output_index 0, + // which must shift the function_call output items to start at index 1. This + // guards against the index-arithmetic regression called out in the spec. + // ─────────────────────────────────────────────────────────────────────── + it("leading reasoning item shifts function_call output_index to 1 (stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("tool-only-think", "o3", true), + ); + expect(res.status).toBe(200); + const events = parseSSEEvents(res.body); + + // The reasoning output item must be added at output_index 0. + const reasoningAdded = events.find( + (e) => e.type === "response.output_item.added" && e.item?.type === "reasoning", + ); + expect(reasoningAdded).toBeDefined(); + expect(reasoningAdded?.output_index).toBe(0); + + // The function_call output item must be added at output_index 1 (shifted by + // the leading reasoning item). + const fcAdded = events.find( + (e) => e.type === "response.output_item.added" && e.item?.type === "function_call", + ); + expect(fcAdded).toBeDefined(); + expect(fcAdded?.output_index).toBe(1); + + // All function_call argument events carry the shifted output_index too. + const fcArgEvents = events.filter( + (e) => typeof e.type === "string" && e.type.startsWith("response.function_call_arguments"), + ); + expect(fcArgEvents.length).toBeGreaterThan(0); + for (const e of fcArgEvents) { + expect(e.output_index).toBe(1); + } + }); + + it("without reasoning the function_call output_index starts at 0 (stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("tool-only-plain", "o3", true), + ); + expect(res.status).toBe(200); + const events = parseSSEEvents(res.body); + + const fcAdded = events.find( + (e) => e.type === "response.output_item.added" && e.item?.type === "function_call", + ); + expect(fcAdded).toBeDefined(); + expect(fcAdded?.output_index).toBe(0); + }); +}); diff --git a/src/__tests__/token-estimation.test.ts b/src/__tests__/token-estimation.test.ts index bd9b41b8..166ce17b 100644 --- a/src/__tests__/token-estimation.test.ts +++ b/src/__tests__/token-estimation.test.ts @@ -129,6 +129,7 @@ describe("buildToolCallCompletion token estimation", () => { [{ name: "get_weather", arguments: '{"city":"New York"}' }], "gpt-4", undefined, + undefined, [{ role: "user", content: "What is the weather in NYC?" }], ); expect(result.usage.prompt_tokens).toBeGreaterThan(0); @@ -142,6 +143,7 @@ describe("buildToolCallCompletion token estimation", () => { const result = buildToolCallCompletion( [{ name: "get_weather", arguments: '{"city":"NYC"}' }], "gpt-4", + undefined, { usage: { prompt_tokens: 20, completion_tokens: 10, total_tokens: 30 } }, ); expect(result.usage).toEqual({ diff --git a/src/__tests__/ws-responses.test.ts b/src/__tests__/ws-responses.test.ts index 6da5e675..2e7ca5c1 100644 --- a/src/__tests__/ws-responses.test.ts +++ b/src/__tests__/ws-responses.test.ts @@ -38,12 +38,24 @@ const capabilityReasoningFixture: Fixture = { response: { content: "The answer.", reasoning: "Let me reason about this." }, }; +// Tool-only fixture that also carries reasoning. Distinct match key so it can +// be exercised against reasoning-capable / non-reasoning models without +// colliding with the plain `weather` tool fixture above. +const toolReasoningFixture: Fixture = { + match: { userMessage: "tool-reason" }, + response: { + toolCalls: [{ name: "get_weather", arguments: '{"city":"NYC"}' }], + reasoning: "Let me reason about the tool call.", + }, +}; + const allFixtures: Fixture[] = [ textFixture, toolFixture, errorFixture, reasoningFixture, capabilityReasoningFixture, + toolReasoningFixture, ]; // --- tests --- @@ -586,4 +598,70 @@ describe("WebSocket /v1/responses reasoning capability gating", () => { ws.close(); }); + + // ── Tool-only path: reasoning must gate the same as text / content+tool, so + // emission is transport-independent (HTTP tool-only already gates+emits). ── + + it("emits reasoning on a tool-only response for a reasoning-capable model", async () => { + instance = await createServer(allFixtures); + const ws = await connectWebSocket(instance.url, "/v1/responses"); + + ws.send(responseCreateMsg("tool-reason", "o3-mini")); + + const raw = await ws.waitForMessages(12); + const events = parseEvents(raw); + const types = events.map((e) => e.type); + + // Reasoning channel is emitted, and the function call is intact. + expect(types).toContain("response.reasoning_summary_text.delta"); + expect(types).toContain("response.function_call_arguments.delta"); + expect(types).toContain("response.function_call_arguments.done"); + + const argDeltas = events.filter((e) => e.type === "response.function_call_arguments.delta"); + expect(argDeltas.map((d) => d.delta).join("")).toBe('{"city":"NYC"}'); + + ws.close(); + }); + + it("suppresses tool-only reasoning for a non-reasoning model under strict (function call intact)", async () => { + instance = await createServer(allFixtures); + const ws = await connectWebSocket(instance.url, "/v1/responses", { + "X-AIMock-Strict": "true", + }); + + ws.send(responseCreateMsg("tool-reason", "gpt-4.1")); + + const raw = await ws.waitForMessages(7); + const events = parseEvents(raw); + const types = events.map((e) => e.type); + + // Reasoning suppressed, but the function call still streams. + expect(types).not.toContain("response.reasoning_summary_text.delta"); + expect(types).not.toContain("response.reasoning_summary_text.done"); + expect(types).toContain("response.function_call_arguments.delta"); + expect(types).toContain("response.function_call_arguments.done"); + expect(types[types.length - 1]).toBe("response.completed"); + + const argDeltas = events.filter((e) => e.type === "response.function_call_arguments.delta"); + expect(argDeltas.map((d) => d.delta).join("")).toBe('{"city":"NYC"}'); + + ws.close(); + }); + + it("emits tool-only reasoning for a non-reasoning model when strict is off", async () => { + instance = await createServer(allFixtures); + const ws = await connectWebSocket(instance.url, "/v1/responses"); + + ws.send(responseCreateMsg("tool-reason", "gpt-4.1")); + + const raw = await ws.waitForMessages(12); + const events = parseEvents(raw); + const types = events.map((e) => e.type); + + expect(types).toContain("response.reasoning_summary_text.delta"); + expect(types).toContain("response.function_call_arguments.delta"); + expect(types).toContain("response.function_call_arguments.done"); + + ws.close(); + }); }); From 984c990dc671fc036077cfcc91954cc26b220f59 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Tue, 9 Jun 2026 10:22:40 -0700 Subject: [PATCH 3/3] docs: changelog entry for tool-only reasoning emission --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index f723d1f7..7bce1a87 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ ### Changed - **Reasoning emission** — replaying a reasoning channel is now gated on the requested model's capability. aimock no longer synthesizes a reasoning channel (chat `reasoning_content` / Responses `reasoning_summary_text` / Anthropic thinking / etc.) for models that would not emit reasoning against the real provider. A new `isReasoningModel` classifier and `resolveReasoningForModel` gate are applied across OpenAI chat + Responses, Anthropic, Ollama, Gemini, Cohere, Bedrock (invoke + Converse), and WebSocket Responses: a non-reasoning model paired with a reasoning fixture has its reasoning suppressed under strict mode, or warns-and-emits otherwise. The `AIMOCK_REASONING_MODELS` and `AIMOCK_NONREASONING_MODELS` env vars override the classifier. +- **Tool-only reasoning emission** — the capability gate now also covers the tool-call-only response path of every provider (OpenAI chat + Responses, Cohere, Bedrock invoke + Converse, Ollama, WebSocket Responses, and the Gemini chat tool-only path). Previously a tool-only fixture carrying reasoning dropped the reasoning channel entirely; it now emits the provider-native reasoning channel for reasoning-capable models and is suppressed under strict mode (or warns-and-emits otherwise), matching the text and content+tool paths, with leading reasoning blocks shifting streaming tool/output indices by one where applicable. The Gemini audio companion's reasoning is now capability-gated as well. ## [1.29.0] - 2026-06-04