From 7e70b608c038568f6ba1e79f64831ccbec4c1bf3 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Mon, 8 Jun 2026 15:13:37 -0700 Subject: [PATCH 1/4] feat: add model reasoning-capability helpers Add isReasoningModel (model-utils.ts) to classify whether a model id would emit a reasoning channel against the real provider, and resolveReasoningForModel (helpers.ts) to gate a fixture's reasoning value on that classification: suppress under strict mode, warn-and-emit otherwise. Conservative non-reasoning denylist with fail-open default for unknown ids; AIMOCK_REASONING_MODELS / AIMOCK_NONREASONING_MODELS env overrides, normalized identically to incoming model ids (date-suffix + provider-prefix stripped). --- src/helpers.ts | 40 ++++++++++++++ src/model-utils.ts | 128 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 168 insertions(+) diff --git a/src/helpers.ts b/src/helpers.ts index f1316601..7e08e2a7 100644 --- a/src/helpers.ts +++ b/src/helpers.ts @@ -2,6 +2,8 @@ import { createHash, randomBytes } from "node:crypto"; import type * as http from "node:http"; import type { IncomingHttpHeaders } from "node:http"; import { DEFAULT_TEST_ID } from "./constants.js"; +import type { Logger } from "./logger.js"; +import { isReasoningModel } from "./model-utils.js"; import type { ChatCompletionRequest, Fixture, @@ -64,6 +66,44 @@ export function strictOverrideField( return {}; } +/** + * Resolve the reasoning string to actually emit for a given model. + * + * aimock synthesizes a reasoning channel whenever a fixture carries a + * `reasoning` string, regardless of the requested model. But a non-reasoning + * model (e.g. `gpt-4.1`) would emit no reasoning against the real provider, so + * replaying it is a false green (see aimock#254). This gates the emission on + * the requested model's capability: + * + * - no fixture reasoning → undefined (no-op, short-circuit) + * - reasoning-capable model → emit unchanged, no log + * - non-reasoning model, strict OFF → `logger.warn`, still emit (preserves + * current behavior) + * - non-reasoning model, strict ON → `logger.error`, suppress (return undefined) + * + * Capability is decided from the REQUESTED model id (what the backend was wired + * to), not any `overrides.model` echoed in the payload. + */ +export function resolveReasoningForModel( + reasoning: string | undefined, + model: string | undefined, + strict: boolean, + logger: Logger, +): string | undefined { + if (!reasoning) return undefined; + if (isReasoningModel(model)) return reasoning; + if (strict) { + logger.error( + `Strict mode: fixture has a reasoning channel but model "${model}" is not reasoning-capable — suppressing reasoning emission`, + ); + return undefined; + } + logger.warn( + `Fixture has a reasoning channel but model "${model}" is not reasoning-capable — the real provider would emit no reasoning. Emitting anyway (set X-AIMock-Strict: true to suppress).`, + ); + return reasoning; +} + export function flattenHeaders(headers: http.IncomingHttpHeaders): Record { const flat: Record = {}; for (const [key, value] of Object.entries(headers)) { diff --git a/src/model-utils.ts b/src/model-utils.ts index 25459591..a2956f3b 100644 --- a/src/model-utils.ts +++ b/src/model-utils.ts @@ -7,3 +7,131 @@ export function normalizeModelName( if (!model || skipNormalization) return model; return model.replace(DATE_SUFFIX_RE, ""); } + +// ─── Reasoning-capability map ─────────────────────────────────────────────── +// +// `isReasoningModel` answers: "would the REAL provider for this model id emit a +// reasoning/thinking channel?" aimock uses it to gate synthesized reasoning so a +// fixture's `reasoning` field is not replayed for a model that cannot reason +// (the gpt-4.1 false-green described in aimock#254). +// +// Policy (see the #254 spec for the full rationale): +// - The KNOWN-NON-REASONING list (NONREASONING_FAMILIES) is the load-bearing +// guard. A match there → false. +// - The reasoning list (REASONING_FAMILIES) is mostly documentation; because +// unknown ids default to `true` it is not load-bearing for correctness. +// - UNKNOWN id → true (assume reasoning-capable). A false negative silently +// breaks legitimate replays and is hard to notice; a false positive merely +// fails to catch one mis-wiring class. Failing open keeps the blast radius +// small and bounds maintenance to the explicit non-reasoning list. +// +// Env overrides let users correct drift without a release: +// AIMOCK_REASONING_MODELS — comma-separated ids/prefixes forced capable. +// AIMOCK_NONREASONING_MODELS — comma-separated ids/prefixes forced incapable. +// Env entries are normalized IDENTICALLY to the incoming model id (date suffix +// + provider/region prefix stripped, lowercased) so a prefixed entry such as +// `anthropic.claude-opus-4` matches the already-stripped id. +// +// Precedence is BETWEEN lists, enforced by the sequential `if` checks below: +// env-nonreasoning > env-reasoning > built-in-nonreasoning > built-in-reasoning > unknown(true) +// Within any single list the result is order-INDEPENDENT — `matchesAny` uses +// `Array.some(startsWith)`, so any prefix match yields that list's verdict. + +// Known non-reasoning families — the load-bearing denylist. Entries are matched +// as prefixes against the normalized, lowercased, provider-stripped model id. +// Order within this list does not matter (see `matchesAny`): any prefix that +// matches the id produces `false`. +const NONREASONING_FAMILIES = [ + "gpt-4.1", // the exact model from aimock#254 (covers -mini / -nano) + "gpt-4o", // covers gpt-4o-mini + "gpt-4-turbo", + "gpt-4", // plain gpt-4; order-independent, and prefix-safe since longer ids (gpt-4.1/gpt-4o/gpt-4-turbo) are also denylisted + "gpt-3.5", + "claude-3-5-", // Sonnet/Haiku 3.5 — no extended thinking + "claude-3-haiku", + "claude-3-opus", + "claude-3-sonnet", + "gemini-1.5-", +]; + +// Reasoning-capable families (informational; unknown already defaults to true). +const REASONING_FAMILIES = [ + "o1", + "o3", + "o4", + "gpt-5", + "deepseek-r1", + "deepseek-reasoner", + "claude-3-7-sonnet", + "claude-opus-4", + "claude-sonnet-4", + "claude-haiku-4", + "gpt-oss", + "gemini-2.5-pro", + "gemini-2.5-flash", + "gemini-2.0-flash-thinking", + "qwq", + "qwen3", +]; + +/** + * Strip a leading provider segment that Bedrock (and similar) prepend before + * the actual family token — e.g. `anthropic.`, `us.anthropic.`, `eu.`. Keeps + * everything from the recognised family token onward so prefix matching works. + */ +function stripProviderPrefix(id: string): string { + // Remove leading region/provider segments (dot-separated) until we hit the + // model token. Provider/region segments never contain a hyphen, whereas + // model families do (e.g. `claude-3-5-sonnet`, `gpt-4.1`). So drop any + // leading dot-separated segment that has no hyphen. + let rest = id; + while (true) { + const dot = rest.indexOf("."); + if (dot === -1) break; + const head = rest.slice(0, dot); + if (head.length === 0 || head.includes("-")) break; + rest = rest.slice(dot + 1); + } + return rest; +} + +// Normalize each comma-separated env entry IDENTICALLY to the incoming model id +// (date-suffix + provider/region prefix stripped, lowercased) so a prefixed +// entry such as `anthropic.claude-opus-4` matches the already-stripped id. +function parseEnvList(raw: string | undefined): string[] { + if (!raw) return []; + return raw + .split(",") + .map((s) => stripProviderPrefix(normalizeModelName(s.trim())?.toLowerCase() ?? "")) + .filter((s) => s.length > 0); +} + +function matchesAny(id: string, families: readonly string[]): boolean { + return families.some((fam) => id.startsWith(fam)); +} + +/** + * Returns whether the real provider for `model` would emit a reasoning/thinking + * channel. Absence of a model id, or an unrecognised id, defaults to `true` + * (assume capable) — see the policy note above. + */ +export function isReasoningModel(model: string | undefined): boolean { + if (!model) return true; + + const normalized = normalizeModelName(model)?.toLowerCase() ?? ""; + const id = stripProviderPrefix(normalized); + + // Env overrides win, parsed fresh so they remain test-stubbable. + const envNon = parseEnvList(process.env.AIMOCK_NONREASONING_MODELS); + if (matchesAny(id, envNon)) return false; + const envReasoning = parseEnvList(process.env.AIMOCK_REASONING_MODELS); + if (matchesAny(id, envReasoning)) return true; + + // Built-in denylist is the active guard. + if (matchesAny(id, NONREASONING_FAMILIES)) return false; + // Built-in reasoning list short-circuits to capable. + if (matchesAny(id, REASONING_FAMILIES)) return true; + + // Unknown → assume reasoning-capable (fail open). + return true; +} From 8b950c7a745f69498cfa3f71a59fc08338e23627 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Mon, 8 Jun 2026 15:13:37 -0700 Subject: [PATCH 2/4] feat: gate reasoning emission on model capability across providers Route synthesized reasoning through resolveReasoningForModel at every provider dispatch point (OpenAI chat + Responses, Anthropic Messages, Ollama, Gemini, Cohere, Bedrock invoke + Converse, WebSocket Responses), computing effectiveStrict fresh at each site. The WS path resolves strict from upgradeHeaders. Stops aimock replaying a reasoning channel for models the real provider would not emit it for. --- src/bedrock-converse.ts | 33 +++++++++++++++++++++++++++++---- src/bedrock.ts | 33 +++++++++++++++++++++++++++++---- src/cohere.ts | 23 +++++++++++++++++++---- src/gemini.ts | 23 +++++++++++++++++++---- src/messages.ts | 23 +++++++++++++++++++---- src/ollama.ts | 29 +++++++++++++++++++++-------- src/responses.ts | 25 +++++++++++++++++++++---- src/server.ts | 23 +++++++++++++++++++---- src/ws-responses.ts | 20 ++++++++++++++++++-- 9 files changed, 194 insertions(+), 38 deletions(-) diff --git a/src/bedrock-converse.ts b/src/bedrock-converse.ts index e9d51315..35da99c7 100644 --- a/src/bedrock-converse.ts +++ b/src/bedrock-converse.ts @@ -28,6 +28,7 @@ import { getContext, getTestId, resolveResponse, + resolveReasoningForModel, resolveStrictMode, strictOverrideField, } from "./helpers.js"; @@ -710,6 +711,12 @@ export async function handleConverse( ); } const overrides = extractOverrides(response); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + resolveStrictMode(defaults.strict, req.headers), + logger, + ); journal.add({ method: req.method ?? "POST", path: urlPath, @@ -721,7 +728,7 @@ export async function handleConverse( response.content, response.toolCalls, logger, - response.reasoning, + effReasoning, overrides, ); res.writeHead(200, { "Content-Type": "application/json" }); @@ -737,6 +744,12 @@ export async function handleConverse( ); } const overrides = extractOverrides(response); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + resolveStrictMode(defaults.strict, req.headers), + logger, + ); journal.add({ method: req.method ?? "POST", path: urlPath, @@ -744,7 +757,7 @@ export async function handleConverse( body: completionReq, response: { status: 200, fixture }, }); - const body = buildConverseTextResponse(response.content, response.reasoning, overrides); + const body = buildConverseTextResponse(response.content, effReasoning, overrides); res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); return; @@ -1006,6 +1019,12 @@ export async function handleConverseStream( ); } const overrides = extractOverrides(response); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + resolveStrictMode(defaults.strict, req.headers), + logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path: urlPath, @@ -1018,7 +1037,7 @@ export async function handleConverseStream( response.toolCalls, chunkSize, logger, - response.reasoning, + effReasoning, overrides, ); const interruption = createInterruptionSignal(fixture); @@ -1047,6 +1066,12 @@ export async function handleConverseStream( ); } const overrides = extractOverrides(response); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + resolveStrictMode(defaults.strict, req.headers), + logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path: urlPath, @@ -1057,7 +1082,7 @@ export async function handleConverseStream( const events = buildBedrockStreamTextEvents( response.content, chunkSize, - response.reasoning, + effReasoning, overrides, ); const interruption = createInterruptionSignal(fixture); diff --git a/src/bedrock.ts b/src/bedrock.ts index 33ae5e56..282b524d 100644 --- a/src/bedrock.ts +++ b/src/bedrock.ts @@ -39,6 +39,7 @@ import { getContext, getTestId, resolveResponse, + resolveReasoningForModel, resolveStrictMode, strictOverrideField, } from "./helpers.js"; @@ -538,6 +539,12 @@ export async function handleBedrock( logger.warn("webSearches in fixture response are not supported for Bedrock API — ignoring"); } const overrides = extractOverrides(response); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + resolveStrictMode(defaults.strict, req.headers), + logger, + ); journal.add({ method: req.method ?? "POST", path: urlPath, @@ -548,7 +555,7 @@ export async function handleBedrock( const textBody = buildBedrockTextResponse( response.content, completionReq.model, - response.reasoning, + effReasoning, overrides, ); const toolBody = buildBedrockToolCallResponse( @@ -577,6 +584,12 @@ export async function handleBedrock( logger.warn("webSearches in fixture response are not supported for Bedrock API — ignoring"); } const overrides = extractOverrides(response); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + resolveStrictMode(defaults.strict, req.headers), + logger, + ); journal.add({ method: req.method ?? "POST", path: urlPath, @@ -587,7 +600,7 @@ export async function handleBedrock( const body = buildBedrockTextResponse( response.content, completionReq.model, - response.reasoning, + effReasoning, overrides, ); res.writeHead(200, { "Content-Type": "application/json" }); @@ -1184,6 +1197,12 @@ export async function handleBedrockStream( logger.warn("webSearches in fixture response are not supported for Bedrock API — ignoring"); } const overrides = extractOverrides(response); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + resolveStrictMode(defaults.strict, req.headers), + logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path: urlPath, @@ -1197,7 +1216,7 @@ export async function handleBedrockStream( completionReq.model, chunkSize, logger, - response.reasoning, + effReasoning, overrides, ); const interruption = createInterruptionSignal(fixture); @@ -1224,6 +1243,12 @@ export async function handleBedrockStream( logger.warn("webSearches in fixture response are not supported for Bedrock API — ignoring"); } const overrides = extractOverrides(response); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + resolveStrictMode(defaults.strict, req.headers), + logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path: urlPath, @@ -1235,7 +1260,7 @@ export async function handleBedrockStream( response.content, completionReq.model, chunkSize, - response.reasoning, + effReasoning, overrides, ); const interruption = createInterruptionSignal(fixture); diff --git a/src/cohere.ts b/src/cohere.ts index 6caaaf6d..946234d6 100644 --- a/src/cohere.ts +++ b/src/cohere.ts @@ -36,6 +36,7 @@ import { getTestId, resolveResponse, resolveStrictMode, + resolveReasoningForModel, strictOverrideField, getContext, } from "./helpers.js"; @@ -977,6 +978,13 @@ export async function handleCohere( ); } const overrides = extractOverrides(response); + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + cohereReq.model, + effectiveStrict, + logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path: req.url ?? "/v2/chat", @@ -989,7 +997,7 @@ export async function handleCohere( response.content, response.toolCalls, logger, - response.reasoning, + effReasoning, overrides, ); res.writeHead(200, { "Content-Type": "application/json" }); @@ -1000,7 +1008,7 @@ export async function handleCohere( response.toolCalls, chunkSize, logger, - response.reasoning, + effReasoning, overrides, ); const interruption = createInterruptionSignal(fixture); @@ -1030,6 +1038,13 @@ export async function handleCohere( ); } const overrides = extractOverrides(response); + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + cohereReq.model, + effectiveStrict, + logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path: req.url ?? "/v2/chat", @@ -1038,14 +1053,14 @@ export async function handleCohere( response: { status: 200, fixture }, }); if (cohereReq.stream !== true) { - const body = buildCohereTextResponse(response.content, response.reasoning, overrides); + const body = buildCohereTextResponse(response.content, effReasoning, overrides); res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); } else { const events = buildCohereTextStreamEvents( response.content, chunkSize, - response.reasoning, + effReasoning, overrides, ); const interruption = createInterruptionSignal(fixture); diff --git a/src/gemini.ts b/src/gemini.ts index f240f168..9fd5d780 100644 --- a/src/gemini.ts +++ b/src/gemini.ts @@ -33,6 +33,7 @@ import { getTestId, resolveResponse, resolveStrictMode, + resolveReasoningForModel, strictOverrideField, } from "./helpers.js"; import { matchFixture } from "./router.js"; @@ -841,6 +842,13 @@ export async function handleGemini( logger.warn("webSearches in fixture response are not supported for Gemini API — ignoring"); } const overrides = extractOverrides(response); + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + model, + effectiveStrict, + defaults.logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path, @@ -853,7 +861,7 @@ export async function handleGemini( response.content, response.toolCalls, logger, - response.reasoning, + effReasoning, overrides, ); res.writeHead(200, { "Content-Type": "application/json" }); @@ -864,7 +872,7 @@ export async function handleGemini( response.toolCalls, chunkSize, logger, - response.reasoning, + effReasoning, overrides, ); const interruption = createInterruptionSignal(fixture); @@ -892,6 +900,13 @@ export async function handleGemini( logger.warn("webSearches in fixture response are not supported for Gemini API — ignoring"); } const overrides = extractOverrides(response); + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + model, + effectiveStrict, + defaults.logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path, @@ -900,14 +915,14 @@ export async function handleGemini( response: { status: 200, fixture }, }); if (!streaming) { - const body = buildGeminiTextResponse(response.content, response.reasoning, overrides); + const body = buildGeminiTextResponse(response.content, effReasoning, overrides); res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); } else { const chunks = buildGeminiTextStreamChunks( response.content, chunkSize, - response.reasoning, + effReasoning, overrides, ); const interruption = createInterruptionSignal(fixture); diff --git a/src/messages.ts b/src/messages.ts index 6e0a9ce0..f47b4bfd 100644 --- a/src/messages.ts +++ b/src/messages.ts @@ -30,6 +30,7 @@ import { getTestId, resolveResponse, resolveStrictMode, + resolveReasoningForModel, strictOverrideField, getContext, } from "./helpers.js"; @@ -904,6 +905,13 @@ export async function handleMessages( ); } const overrides = extractOverrides(response); + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + effectiveStrict, + defaults.logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path: req.url ?? "/v1/messages", @@ -917,7 +925,7 @@ export async function handleMessages( response.toolCalls, completionReq.model, logger, - response.reasoning, + effReasoning, overrides, ); res.writeHead(200, { "Content-Type": "application/json" }); @@ -929,7 +937,7 @@ export async function handleMessages( completionReq.model, chunkSize, logger, - response.reasoning, + effReasoning, overrides, ); const interruption = createInterruptionSignal(fixture); @@ -959,6 +967,13 @@ export async function handleMessages( ); } const overrides = extractOverrides(response); + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + effectiveStrict, + defaults.logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path: req.url ?? "/v1/messages", @@ -970,7 +985,7 @@ export async function handleMessages( const body = buildClaudeTextResponse( response.content, completionReq.model, - response.reasoning, + effReasoning, overrides, ); res.writeHead(200, { "Content-Type": "application/json" }); @@ -980,7 +995,7 @@ export async function handleMessages( response.content, completionReq.model, chunkSize, - response.reasoning, + effReasoning, overrides, ); const interruption = createInterruptionSignal(fixture); diff --git a/src/ollama.ts b/src/ollama.ts index 6ee5c7b3..c73f969c 100644 --- a/src/ollama.ts +++ b/src/ollama.ts @@ -34,6 +34,7 @@ import { getTestId, resolveResponse, resolveStrictMode, + resolveReasoningForModel, strictOverrideField, getContext, } from "./helpers.js"; @@ -736,12 +737,16 @@ export async function handleOllama( body: completionReq, response: { status: 200, fixture }, }); + // Gate reasoning emission on the requested model's capability (aimock#254). + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + effectiveStrict, + logger, + ); if (!streaming) { - const body = buildOllamaChatTextResponse( - response.content, - completionReq.model, - response.reasoning, - ); + const body = buildOllamaChatTextResponse(response.content, completionReq.model, effReasoning); res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); } else { @@ -749,7 +754,7 @@ export async function handleOllama( response.content, completionReq.model, chunkSize, - response.reasoning, + effReasoning, ); const interruption = createInterruptionSignal(fixture); const completed = await writeNDJSONStream(res, chunks, { @@ -1036,11 +1041,19 @@ export async function handleOllamaGenerate( body: completionReq, response: { status: 200, fixture }, }); + // Gate reasoning emission on the requested model's capability (aimock#254). + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + effectiveStrict, + defaults.logger, + ); if (!streaming) { const body = buildOllamaGenerateTextResponse( response.content, completionReq.model, - response.reasoning, + effReasoning, ); res.writeHead(200, { "Content-Type": "application/json" }); res.end(JSON.stringify(body)); @@ -1049,7 +1062,7 @@ export async function handleOllamaGenerate( response.content, completionReq.model, chunkSize, - response.reasoning, + effReasoning, ); const interruption = createInterruptionSignal(fixture); const completed = await writeNDJSONStream(res, chunks, { diff --git a/src/responses.ts b/src/responses.ts index e5b475f5..3febd700 100644 --- a/src/responses.ts +++ b/src/responses.ts @@ -30,6 +30,7 @@ import { getTestId, resolveResponse, resolveStrictMode, + resolveReasoningForModel, strictOverrideField, getContext, } from "./helpers.js"; @@ -1093,6 +1094,14 @@ export async function handleResponses( // Combined content + tool calls response if (isContentWithToolCallsResponse(response)) { const overrides = extractOverrides(response); + // Gate reasoning emission on the requested model's capability (aimock#254). + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + effectiveStrict, + defaults.logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path: req.url ?? "/v1/responses", @@ -1105,7 +1114,7 @@ export async function handleResponses( response.content, response.toolCalls, completionReq.model, - response.reasoning, + effReasoning, response.webSearches, overrides, ); @@ -1117,7 +1126,7 @@ export async function handleResponses( response.toolCalls, completionReq.model, chunkSize, - response.reasoning, + effReasoning, response.webSearches, overrides, ); @@ -1143,6 +1152,14 @@ export async function handleResponses( // Text response if (isTextResponse(response)) { const overrides = extractOverrides(response); + // Gate reasoning emission on the requested model's capability (aimock#254). + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + effectiveStrict, + defaults.logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path: req.url ?? "/v1/responses", @@ -1154,7 +1171,7 @@ export async function handleResponses( const body = buildTextResponse( response.content, completionReq.model, - response.reasoning, + effReasoning, response.webSearches, overrides, ); @@ -1165,7 +1182,7 @@ export async function handleResponses( response.content, completionReq.model, chunkSize, - response.reasoning, + effReasoning, response.webSearches, overrides, ); diff --git a/src/server.ts b/src/server.ts index 29c3479e..5f67a436 100644 --- a/src/server.ts +++ b/src/server.ts @@ -35,6 +35,7 @@ import { readBody, resolveResponse, resolveStrictMode, + resolveReasoningForModel, strictOverrideField, getContext, } from "./helpers.js"; @@ -766,6 +767,13 @@ async function handleCompletions( ); } const overrides = extractOverrides(response); + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + body.model, + effectiveStrict, + defaults.logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path: req.url ?? COMPLETIONS_PATH, @@ -778,7 +786,7 @@ async function handleCompletions( response.content, response.toolCalls, body.model, - response.reasoning, + effReasoning, overrides, body.messages, ); @@ -790,7 +798,7 @@ async function handleCompletions( response.toolCalls, body.model, chunkSize, - response.reasoning, + effReasoning, overrides, ); // Build usage chunk for stream_options.include_usage @@ -846,6 +854,13 @@ async function handleCompletions( ); } const overrides = extractOverrides(response); + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + body.model, + effectiveStrict, + defaults.logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path: req.url ?? COMPLETIONS_PATH, @@ -857,7 +872,7 @@ async function handleCompletions( const completion = buildTextCompletion( response.content, body.model, - response.reasoning, + effReasoning, overrides, body.messages, ); @@ -868,7 +883,7 @@ async function handleCompletions( response.content, body.model, chunkSize, - response.reasoning, + effReasoning, overrides, ); const usageChunk = includeUsage diff --git a/src/ws-responses.ts b/src/ws-responses.ts index a5f2ea85..56820516 100644 --- a/src/ws-responses.ts +++ b/src/ws-responses.ts @@ -23,6 +23,7 @@ import { extractOverrides, resolveResponse, resolveStrictMode, + resolveReasoningForModel, strictOverrideField, flattenHeaders, } from "./helpers.js"; @@ -232,6 +233,11 @@ async function processMessage( const latency = fixture.latency ?? defaults.latency; const chunkSize = Math.max(1, fixture.chunkSize ?? defaults.chunkSize); + // The WS path has no per-request `req.headers`; strict is resolved from the + // connection's upgrade headers (see the `!fixture` branch above). Used below + // to gate the synthesized reasoning channel on the requested model's capability. + const effectiveStrict = resolveStrictMode(defaults.strict, defaults.upgradeHeaders); + // Error response if (isErrorResponse(response)) { const status = response.status ?? 500; @@ -265,7 +271,12 @@ async function processMessage( response.toolCalls, completionReq.model, chunkSize, - response.reasoning, + resolveReasoningForModel( + response.reasoning, + completionReq.model, + effectiveStrict, + defaults.logger, + ), response.webSearches, extractOverrides(response), ); @@ -303,7 +314,12 @@ async function processMessage( response.content, completionReq.model, chunkSize, - response.reasoning, + resolveReasoningForModel( + response.reasoning, + completionReq.model, + effectiveStrict, + defaults.logger, + ), response.webSearches, extractOverrides(response), ); From 174c24325d4b4f4580d011830ced4071a5314faf Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Mon, 8 Jun 2026 15:13:37 -0700 Subject: [PATCH 3/4] test: cover reasoning-capability gating across providers Unit tests for isReasoningModel (incl. provider-prefixed env overrides) and resolveReasoningForModel, plus per-provider integration tests asserting suppress-under-strict / warn-and-emit / capable-emit / no-op. Align existing cross-provider reasoning tests to reasoning-capable model ids. --- .../bedrock-reasoning-capability.test.ts | 517 ++++++++++++++++++ .../cohere-reasoning-capability.test.ts | 274 ++++++++++ .../gemini-reasoning-capability.test.ts | 289 ++++++++++ src/__tests__/model-utils.test.ts | 93 +++- .../openai-chat-reasoning-capability.test.ts | 258 +++++++++ src/__tests__/reasoning-all-providers.test.ts | 4 +- .../reasoning-capability-anthropic.test.ts | 495 +++++++++++++++++ .../reasoning-capability-ollama.test.ts | 337 ++++++++++++ src/__tests__/reasoning-web-search.test.ts | 24 +- .../resolve-reasoning-for-model.test.ts | 61 +++ .../responses-reasoning-capability.test.ts | 264 +++++++++ src/__tests__/ws-responses.test.ts | 118 +++- 12 files changed, 2717 insertions(+), 17 deletions(-) create mode 100644 src/__tests__/bedrock-reasoning-capability.test.ts create mode 100644 src/__tests__/cohere-reasoning-capability.test.ts create mode 100644 src/__tests__/gemini-reasoning-capability.test.ts create mode 100644 src/__tests__/openai-chat-reasoning-capability.test.ts create mode 100644 src/__tests__/reasoning-capability-anthropic.test.ts create mode 100644 src/__tests__/reasoning-capability-ollama.test.ts create mode 100644 src/__tests__/resolve-reasoning-for-model.test.ts create mode 100644 src/__tests__/responses-reasoning-capability.test.ts diff --git a/src/__tests__/bedrock-reasoning-capability.test.ts b/src/__tests__/bedrock-reasoning-capability.test.ts new file mode 100644 index 00000000..2d3c1f83 --- /dev/null +++ b/src/__tests__/bedrock-reasoning-capability.test.ts @@ -0,0 +1,517 @@ +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import * as http from "node:http"; +import { crc32 } from "node:zlib"; +import type { Fixture } from "../types.js"; +import { createServer, type ServerInstance } from "../server.js"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +let instance: ServerInstance | null = null; +let baseUrl: string; + +function post( + path: string, + body: unknown, + headers?: Record, +): Promise<{ status: number; headers: http.IncomingHttpHeaders; body: string }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify(body); + const parsed = new URL(baseUrl); + const req = http.request( + { + hostname: parsed.hostname, + port: parsed.port, + path, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => { + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + body: Buffer.concat(chunks).toString(), + }); + }); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +function postRaw( + path: string, + body: unknown, + headers?: Record, +): Promise<{ status: number; headers: http.IncomingHttpHeaders; body: Buffer }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify(body); + const parsed = new URL(baseUrl); + const req = http.request( + { + hostname: parsed.hostname, + port: parsed.port, + path, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => { + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + body: Buffer.concat(chunks), + }); + }); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +/** + * Decode AWS Event Stream binary frames from a Buffer. + * Returns an array of { eventType, payload } objects. + */ +function decodeEventStreamFrames(buf: Buffer): Array<{ eventType: string; payload: object }> { + const frames: Array<{ eventType: string; payload: object }> = []; + let offset = 0; + + while (offset < buf.length) { + if (offset + 12 > buf.length) break; + + const totalLength = buf.readUInt32BE(offset); + const headersLength = buf.readUInt32BE(offset + 4); + const preludeCrc = buf.readUInt32BE(offset + 8); + + const computedPreludeCrc = crc32(buf.subarray(offset, offset + 8)); + if (computedPreludeCrc >>> 0 !== preludeCrc) { + throw new Error("Prelude CRC mismatch"); + } + + const headersStart = offset + 12; + const headersEnd = headersStart + headersLength; + const headers: Record = {}; + let hOff = headersStart; + while (hOff < headersEnd) { + const nameLen = buf.readUInt8(hOff); + hOff += 1; + const name = buf.subarray(hOff, hOff + nameLen).toString("utf8"); + hOff += nameLen; + hOff += 1; // skip header type byte (7 = STRING) + const valueLen = buf.readUInt16BE(hOff); + hOff += 2; + const value = buf.subarray(hOff, hOff + valueLen).toString("utf8"); + hOff += valueLen; + headers[name] = value; + } + + const payloadStart = headersEnd; + const payloadEnd = offset + totalLength - 4; // minus message CRC + const payloadBuf = buf.subarray(payloadStart, payloadEnd); + const payload = payloadBuf.length > 0 ? JSON.parse(payloadBuf.toString("utf8")) : {}; + + frames.push({ + eventType: headers[":event-type"] ?? "", + payload, + }); + + offset += totalLength; + } + + return frames; +} + +// Predicates over decoded InvokeModel stream frames +function hasInvokeThinkingDelta(frames: Array<{ eventType: string; payload: object }>): boolean { + return frames.some( + (f) => + (f.payload as { type?: string }).type === "content_block_delta" && + (f.payload as { delta?: { type?: string } }).delta?.type === "thinking_delta", + ); +} + +// Predicates over decoded Converse stream frames +function hasConverseReasoningDelta(frames: Array<{ eventType: string; payload: object }>): boolean { + return frames.some( + (f) => + f.eventType === "contentBlockDelta" && + (f.payload as { delta?: { reasoningContent?: unknown } }).delta?.reasoningContent !== + undefined, + ); +} + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const reasoningFixture: Fixture = { + match: { userMessage: "think" }, + response: { + content: "The answer is 42.", + reasoning: "Let me think step by step about this problem.", + }, +}; + +const reasoningWithToolsFixture: Fixture = { + match: { userMessage: "reason-and-call" }, + response: { + content: "Calling a tool.", + reasoning: "I should call get_weather.", + toolCalls: [{ name: "get_weather", arguments: '{"city":"SF"}' }], + }, +}; + +const plainFixture: Fixture = { + match: { userMessage: "plain" }, + response: { content: "Just plain text." }, +}; + +const allFixtures: Fixture[] = [reasoningFixture, reasoningWithToolsFixture, plainFixture]; + +// Model ids carrying Bedrock provider prefixes. +const REASONING_MODEL = "anthropic.claude-opus-4-20250514-v1:0"; // Claude 4 → reasoning-capable +const NON_REASONING_MODEL = "anthropic.claude-3-5-sonnet-20241022-v2:0"; // 3.5 → not reasoning-capable +const UNKNOWN_MODEL = "anthropic.future-model-x"; // unknown → fail open (capable) + +// --------------------------------------------------------------------------- +// Server lifecycle (default: strict OFF, warn-level logging so warns surface) +// --------------------------------------------------------------------------- + +beforeEach(async () => { + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + baseUrl = instance.url; +}); + +afterEach(async () => { + vi.restoreAllMocks(); + if (instance) { + await new Promise((resolve) => { + instance!.server.close(() => resolve()); + }); + instance = null; + } +}); + +// --------------------------------------------------------------------------- +// Bedrock InvokeModel — non-streaming +// --------------------------------------------------------------------------- + +describe("Bedrock InvokeModel reasoning capability gating (non-streaming)", () => { + it("emits thinking block for a reasoning-capable bedrock-prefixed model", async () => { + const res = await post(`/model/${REASONING_MODEL}/invoke`, { + messages: [{ role: "user", content: [{ type: "text", text: "think" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.content[0].type).toBe("thinking"); + expect(body.content[0].thinking).toBe("Let me think step by step about this problem."); + }); + + it("non-reasoning model + strict OFF: still emits thinking and logs a warning", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const res = await post(`/model/${NON_REASONING_MODEL}/invoke`, { + messages: [{ role: "user", content: [{ type: "text", text: "think" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.content[0].type).toBe("thinking"); + expect(warnSpy.mock.calls.flat().join(" ").includes("not reasoning-capable")).toBe(true); + }); + + it("non-reasoning model + strict ON: suppresses thinking block", async () => { + const res = await post( + `/model/${NON_REASONING_MODEL}/invoke`, + { + messages: [{ role: "user", content: [{ type: "text", text: "think" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.content).toHaveLength(1); + expect(body.content[0].type).toBe("text"); + }); + + it("unknown model: emits thinking (fail open)", async () => { + const res = await post(`/model/${UNKNOWN_MODEL}/invoke`, { + messages: [{ role: "user", content: [{ type: "text", text: "think" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.content[0].type).toBe("thinking"); + }); + + it("reasoning absent: no thinking block, no gating log", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const res = await post(`/model/${NON_REASONING_MODEL}/invoke`, { + messages: [{ role: "user", content: [{ type: "text", text: "plain" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.content).toHaveLength(1); + expect(body.content[0].type).toBe("text"); + expect(warnSpy.mock.calls.flat().join(" ").includes("not reasoning-capable")).toBe(false); + }); + + it("content+tool calls: non-reasoning model + strict ON suppresses thinking", async () => { + const res = await post( + `/model/${NON_REASONING_MODEL}/invoke`, + { + messages: [{ role: "user", content: [{ type: "text", text: "reason-and-call" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + const types = (body.content as Array<{ type: string }>).map((b) => b.type); + expect(types).not.toContain("thinking"); + expect(types).toContain("text"); + expect(types).toContain("tool_use"); + }); +}); + +// --------------------------------------------------------------------------- +// Bedrock InvokeModel — streaming +// --------------------------------------------------------------------------- + +describe("Bedrock InvokeModel reasoning capability gating (streaming)", () => { + it("emits thinking_delta for a reasoning-capable model", async () => { + const res = await postRaw(`/model/${REASONING_MODEL}/invoke-with-response-stream`, { + messages: [{ role: "user", content: [{ type: "text", text: "think" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }); + + expect(res.status).toBe(200); + expect(hasInvokeThinkingDelta(decodeEventStreamFrames(res.body))).toBe(true); + }); + + it("non-reasoning model + strict OFF: still emits thinking_delta + warns", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const res = await postRaw(`/model/${NON_REASONING_MODEL}/invoke-with-response-stream`, { + messages: [{ role: "user", content: [{ type: "text", text: "think" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }); + + expect(res.status).toBe(200); + expect(hasInvokeThinkingDelta(decodeEventStreamFrames(res.body))).toBe(true); + expect(warnSpy.mock.calls.flat().join(" ").includes("not reasoning-capable")).toBe(true); + }); + + it("non-reasoning model + strict ON: suppresses thinking_delta", async () => { + const res = await postRaw( + `/model/${NON_REASONING_MODEL}/invoke-with-response-stream`, + { + messages: [{ role: "user", content: [{ type: "text", text: "think" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + expect(hasInvokeThinkingDelta(decodeEventStreamFrames(res.body))).toBe(false); + }); + + it("content+tool calls: non-reasoning model + strict ON suppresses thinking_delta", async () => { + const res = await postRaw( + `/model/${NON_REASONING_MODEL}/invoke-with-response-stream`, + { + messages: [{ role: "user", content: [{ type: "text", text: "reason-and-call" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + expect(hasInvokeThinkingDelta(decodeEventStreamFrames(res.body))).toBe(false); + }); + + it("unknown model: emits thinking_delta (fail open)", async () => { + const res = await postRaw(`/model/${UNKNOWN_MODEL}/invoke-with-response-stream`, { + messages: [{ role: "user", content: [{ type: "text", text: "think" }] }], + max_tokens: 1024, + anthropic_version: "bedrock-2023-05-31", + }); + + expect(res.status).toBe(200); + expect(hasInvokeThinkingDelta(decodeEventStreamFrames(res.body))).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// Bedrock Converse — non-streaming +// --------------------------------------------------------------------------- + +describe("Bedrock Converse reasoning capability gating (non-streaming)", () => { + it("emits reasoningContent block for a reasoning-capable model", async () => { + const res = await post(`/model/${REASONING_MODEL}/converse`, { + messages: [{ role: "user", content: [{ text: "think" }] }], + }); + + expect(res.status).toBe(200); + const content = JSON.parse(res.body).output.message.content; + expect(content[0].reasoningContent).toBeDefined(); + expect(content[0].reasoningContent.reasoningText.text).toBe( + "Let me think step by step about this problem.", + ); + }); + + it("non-reasoning model + strict OFF: still emits reasoningContent + warns", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const res = await post(`/model/${NON_REASONING_MODEL}/converse`, { + messages: [{ role: "user", content: [{ text: "think" }] }], + }); + + expect(res.status).toBe(200); + const content = JSON.parse(res.body).output.message.content; + expect(content[0].reasoningContent).toBeDefined(); + expect(warnSpy.mock.calls.flat().join(" ").includes("not reasoning-capable")).toBe(true); + }); + + it("non-reasoning model + strict ON: suppresses reasoningContent", async () => { + const res = await post( + `/model/${NON_REASONING_MODEL}/converse`, + { messages: [{ role: "user", content: [{ text: "think" }] }] }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const content = JSON.parse(res.body).output.message.content; + expect(content).toHaveLength(1); + expect(content[0].reasoningContent).toBeUndefined(); + expect(content[0].text).toBe("The answer is 42."); + }); + + it("unknown model: emits reasoningContent (fail open)", async () => { + const res = await post(`/model/${UNKNOWN_MODEL}/converse`, { + messages: [{ role: "user", content: [{ text: "think" }] }], + }); + + expect(res.status).toBe(200); + const content = JSON.parse(res.body).output.message.content; + expect(content[0].reasoningContent).toBeDefined(); + }); + + it("content+tool calls: non-reasoning model + strict ON suppresses reasoningContent", async () => { + const res = await post( + `/model/${NON_REASONING_MODEL}/converse`, + { messages: [{ role: "user", content: [{ text: "reason-and-call" }] }] }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const content = JSON.parse(res.body).output.message.content as Array>; + expect(content.some((b) => "reasoningContent" in b)).toBe(false); + expect(content.some((b) => "toolUse" in b)).toBe(true); + }); + + it("reasoning absent: no reasoningContent, no gating log", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const res = await post(`/model/${NON_REASONING_MODEL}/converse`, { + messages: [{ role: "user", content: [{ text: "plain" }] }], + }); + + expect(res.status).toBe(200); + const content = JSON.parse(res.body).output.message.content; + expect(content).toHaveLength(1); + expect(content[0].reasoningContent).toBeUndefined(); + expect(warnSpy.mock.calls.flat().join(" ").includes("not reasoning-capable")).toBe(false); + }); +}); + +// --------------------------------------------------------------------------- +// Bedrock Converse — streaming +// --------------------------------------------------------------------------- + +describe("Bedrock Converse reasoning capability gating (streaming)", () => { + it("emits reasoningContent delta for a reasoning-capable model", async () => { + const res = await postRaw(`/model/${REASONING_MODEL}/converse-stream`, { + messages: [{ role: "user", content: [{ text: "think" }] }], + }); + + expect(res.status).toBe(200); + expect(hasConverseReasoningDelta(decodeEventStreamFrames(res.body))).toBe(true); + }); + + it("non-reasoning model + strict OFF: still emits reasoningContent delta + warns", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const res = await postRaw(`/model/${NON_REASONING_MODEL}/converse-stream`, { + messages: [{ role: "user", content: [{ text: "think" }] }], + }); + + expect(res.status).toBe(200); + expect(hasConverseReasoningDelta(decodeEventStreamFrames(res.body))).toBe(true); + expect(warnSpy.mock.calls.flat().join(" ").includes("not reasoning-capable")).toBe(true); + }); + + it("non-reasoning model + strict ON: suppresses reasoningContent delta", async () => { + const res = await postRaw( + `/model/${NON_REASONING_MODEL}/converse-stream`, + { messages: [{ role: "user", content: [{ text: "think" }] }] }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + expect(hasConverseReasoningDelta(decodeEventStreamFrames(res.body))).toBe(false); + }); + + it("content+tool calls: non-reasoning model + strict ON suppresses reasoningContent delta", async () => { + const res = await postRaw( + `/model/${NON_REASONING_MODEL}/converse-stream`, + { messages: [{ role: "user", content: [{ text: "reason-and-call" }] }] }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + expect(hasConverseReasoningDelta(decodeEventStreamFrames(res.body))).toBe(false); + }); + + it("unknown model: emits reasoningContent delta (fail open)", async () => { + const res = await postRaw(`/model/${UNKNOWN_MODEL}/converse-stream`, { + messages: [{ role: "user", content: [{ text: "think" }] }], + }); + + expect(res.status).toBe(200); + expect(hasConverseReasoningDelta(decodeEventStreamFrames(res.body))).toBe(true); + }); +}); diff --git a/src/__tests__/cohere-reasoning-capability.test.ts b/src/__tests__/cohere-reasoning-capability.test.ts new file mode 100644 index 00000000..5ed3e845 --- /dev/null +++ b/src/__tests__/cohere-reasoning-capability.test.ts @@ -0,0 +1,274 @@ +import { describe, it, expect, afterEach, vi } from "vitest"; +import * as http from "node:http"; +import type { Fixture } from "../types.js"; +import { createServer, type ServerInstance } from "../server.js"; + +// ─── HTTP helper ──────────────────────────────────────────────────────────── + +function post( + url: string, + body: unknown, + extraHeaders: Record = {}, +): Promise<{ status: number; headers: http.IncomingHttpHeaders; body: string }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify(body); + const parsed = new URL(url); + const req = http.request( + { + hostname: parsed.hostname, + port: parsed.port, + path: parsed.pathname, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + ...extraHeaders, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => { + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + body: Buffer.concat(chunks).toString(), + }); + }); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +interface SSEEvent { + event: string; + data: Record; +} + +function parseSSEEvents(body: string): SSEEvent[] { + const events: SSEEvent[] = []; + const blocks = body.split("\n\n").filter((b) => b.trim() !== ""); + for (const block of blocks) { + const lines = block.split("\n"); + let eventType = ""; + let dataStr = ""; + for (const line of lines) { + if (line.startsWith("event: ")) { + eventType = line.slice(7); + } else if (line.startsWith("data: ")) { + dataStr = line.slice(6); + } + } + if (eventType && dataStr) { + events.push({ event: eventType, data: JSON.parse(dataStr) as Record }); + } + } + return events; +} + +interface CohereTextResponse { + message: { content: { type: string; text: string }[] }; +} + +/** Extract every text-block string emitted by a non-streaming Cohere response. */ +function nonStreamTexts(rawBody: string): string[] { + const body = JSON.parse(rawBody) as CohereTextResponse; + return body.message.content.filter((b) => b.type === "text").map((b) => b.text); +} + +/** Concatenate all content-delta text slices from a streamed Cohere response. */ +function streamDeltaTexts(rawBody: string): string { + const events = parseSSEEvents(rawBody); + return events + .filter((e) => e.event === "content-delta") + .map((e) => { + const delta = e.data.delta as + | { message?: { content?: { type?: string; text?: string } } } + | undefined; + return delta?.message?.content?.text ?? ""; + }) + .join(""); +} + +const REASONING = "Let me think step by step about this problem."; +const CONTENT = "The capital of France is Paris."; + +// ─── Fixtures ─────────────────────────────────────────────────────────────── + +const reasoningFixture: Fixture = { + match: { userMessage: "hello" }, + response: { content: CONTENT, reasoning: REASONING }, +}; + +const noReasoningFixture: Fixture = { + match: { userMessage: "plain" }, + response: { content: CONTENT }, +}; + +const reasoningWithToolFixture: Fixture = { + match: { userMessage: "weather" }, + response: { + content: CONTENT, + reasoning: REASONING, + toolCalls: [{ name: "get_weather", arguments: '{"city":"SF"}' }], + }, +}; + +const allFixtures: Fixture[] = [reasoningFixture, noReasoningFixture, reasoningWithToolFixture]; + +const COHERE_PATH = "/v2/chat"; + +function chatReq(model: string, userMessage: string, stream = false): unknown { + return { model, stream, messages: [{ role: "user", content: userMessage }] }; +} + +// ─── Tests ────────────────────────────────────────────────────────────────── + +let instance: ServerInstance | null = null; + +afterEach(async () => { + if (instance) { + await new Promise((resolve) => instance!.server.close(() => resolve())); + instance = null; + } + vi.restoreAllMocks(); +}); + +describe("Cohere reasoning capability gating", () => { + describe("reasoning-capable model", () => { + it("emits the reasoning text block (non-stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await post( + `${instance.url}${COHERE_PATH}`, + chatReq("command-a-reasoning", "hello"), + ); + expect(res.status).toBe(200); + const texts = nonStreamTexts(res.body); + // capable model leaves reasoning untouched: reasoning block precedes content + expect(texts).toContain(REASONING); + expect(texts).toContain(CONTENT); + }); + + it("emits reasoning deltas (stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await post( + `${instance.url}${COHERE_PATH}`, + chatReq("command-a-reasoning", "hello", true), + ); + expect(streamDeltaTexts(res.body)).toContain(REASONING); + }); + }); + + describe("non-reasoning model + reasoning fixture", () => { + it("strict OFF: still emits reasoning and logs a warn (non-stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await post(`${instance.url}${COHERE_PATH}`, chatReq("gpt-4.1", "hello")); + expect(res.status).toBe(200); + expect(nonStreamTexts(res.body)).toContain(REASONING); + expect(warnSpy).toHaveBeenCalled(); + const warned = warnSpy.mock.calls.flat().join(" "); + expect(warned).toContain("gpt-4.1"); + expect(warned).toContain("not reasoning-capable"); + }); + + it("strict OFF: still emits reasoning deltas + warns (stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await post(`${instance.url}${COHERE_PATH}`, chatReq("gpt-4.1", "hello", true)); + expect(streamDeltaTexts(res.body)).toContain(REASONING); + expect(warnSpy).toHaveBeenCalled(); + }); + + it("strict ON: suppresses reasoning and logs an error (non-stream)", async () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await post(`${instance.url}${COHERE_PATH}`, chatReq("gpt-4.1", "hello"), { + "X-AIMock-Strict": "true", + }); + expect(res.status).toBe(200); + const texts = nonStreamTexts(res.body); + expect(texts).not.toContain(REASONING); + expect(texts).toContain(CONTENT); + expect(errorSpy).toHaveBeenCalled(); + const errored = errorSpy.mock.calls.flat().join(" "); + expect(errored).toContain("gpt-4.1"); + }); + + it("strict ON: suppresses reasoning deltas (stream)", async () => { + vi.spyOn(console, "error").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await post(`${instance.url}${COHERE_PATH}`, chatReq("gpt-4.1", "hello", true), { + "X-AIMock-Strict": "true", + }); + const streamed = streamDeltaTexts(res.body); + expect(streamed).not.toContain(REASONING); + expect(streamed).toContain(CONTENT); + }); + }); + + describe("content + tool calls branch", () => { + it("strict ON: suppresses reasoning, keeps content + tool call (non-stream)", async () => { + vi.spyOn(console, "error").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await post(`${instance.url}${COHERE_PATH}`, chatReq("gpt-4.1", "weather"), { + "X-AIMock-Strict": "true", + }); + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as { + message: { content: { type: string; text: string }[]; tool_calls: unknown[] }; + }; + const texts = body.message.content.filter((b) => b.type === "text").map((b) => b.text); + expect(texts).not.toContain(REASONING); + expect(texts).toContain(CONTENT); + expect(body.message.tool_calls.length).toBeGreaterThan(0); + }); + + it("capable model: emits reasoning in content+tool stream", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await post( + `${instance.url}${COHERE_PATH}`, + chatReq("command-a-reasoning", "weather", true), + ); + expect(streamDeltaTexts(res.body)).toContain(REASONING); + }); + }); + + describe("reasoning absent", () => { + it("non-reasoning model, no fixture reasoning: no warn, no error (non-stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await post(`${instance.url}${COHERE_PATH}`, chatReq("gpt-4.1", "plain")); + expect(res.status).toBe(200); + expect(nonStreamTexts(res.body)).toEqual([CONTENT]); + expect(warnSpy).not.toHaveBeenCalled(); + expect(errorSpy).not.toHaveBeenCalled(); + }); + + it("strict ON + no fixture reasoning: short-circuits, no error (non-stream)", async () => { + const errorSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await post(`${instance.url}${COHERE_PATH}`, chatReq("gpt-4.1", "plain"), { + "X-AIMock-Strict": "true", + }); + expect(res.status).toBe(200); + expect(nonStreamTexts(res.body)).toEqual([CONTENT]); + expect(errorSpy).not.toHaveBeenCalled(); + }); + }); + + describe("unknown model", () => { + it("emits reasoning with no warn (fail-open default)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await post(`${instance.url}${COHERE_PATH}`, chatReq("command-future-x", "hello")); + expect(res.status).toBe(200); + expect(nonStreamTexts(res.body)).toContain(REASONING); + expect(warnSpy).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/src/__tests__/gemini-reasoning-capability.test.ts b/src/__tests__/gemini-reasoning-capability.test.ts new file mode 100644 index 00000000..f657c32e --- /dev/null +++ b/src/__tests__/gemini-reasoning-capability.test.ts @@ -0,0 +1,289 @@ +import { describe, it, expect, afterEach, vi } from "vitest"; +import * as http from "node:http"; +import type { Fixture } from "../types.js"; +import { createServer, type ServerInstance } from "../server.js"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function post( + url: string, + body: unknown, + headers?: Record, +): Promise<{ status: number; headers: http.IncomingHttpHeaders; body: string }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify(body); + const parsed = new URL(url); + const req = http.request( + { + hostname: parsed.hostname, + port: parsed.port, + path: parsed.pathname, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => { + resolve({ + status: res.statusCode ?? 0, + headers: res.headers, + body: Buffer.concat(chunks).toString(), + }); + }); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +function parseGeminiSSEChunks(body: string): GeminiChunk[] { + const chunks: GeminiChunk[] = []; + for (const line of body.split("\n")) { + if (line.startsWith("data: ")) { + chunks.push(JSON.parse(line.slice(6)) as GeminiChunk); + } + } + return chunks; +} + +interface GeminiPart { + text?: string; + thought?: boolean; + functionCall?: { name: string; args: Record }; +} + +interface GeminiChunk { + candidates: { + content: { role: string; parts: GeminiPart[] }; + finishReason?: string; + index: number; + }[]; +} + +// "thought parts" = parts emitted on the synthesized reasoning channel. +function thoughtParts(parts: GeminiPart[]): GeminiPart[] { + return parts.filter((p) => p.thought === true); +} + +function reasoningInChunks(chunks: GeminiChunk[]): boolean { + return chunks.some((c) => thoughtParts(c.candidates[0].content.parts).length > 0); +} + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const REASONING = "Let me think step by step about this."; + +const textReasoningFixture: Fixture = { + match: { userMessage: "explain" }, + response: { content: "The answer is 42.", reasoning: REASONING }, +}; + +const textNoReasoningFixture: Fixture = { + match: { userMessage: "plain" }, + response: { content: "No reasoning here." }, +}; + +const contentWithToolsReasoningFixture: Fixture = { + match: { userMessage: "tooled" }, + response: { + content: "I will call a tool.", + reasoning: REASONING, + toolCalls: [{ name: "get_weather", arguments: '{"city":"NYC"}' }], + }, +}; + +const allFixtures: Fixture[] = [ + textReasoningFixture, + textNoReasoningFixture, + contentWithToolsReasoningFixture, +]; + +const GEN = "generateContent"; +const STREAM_GEN = "streamGenerateContent"; + +function geminiUrl(base: string, model: string, op: string): string { + return `${base}/v1beta/models/${model}:${op}`; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +let instance: ServerInstance | null = null; + +afterEach(async () => { + vi.restoreAllMocks(); + if (instance) { + await new Promise((resolve) => { + instance!.server.close(() => resolve()); + }); + instance = null; + } +}); + +describe("Gemini reasoning capability gating (non-streaming)", () => { + it("emits thought parts for a reasoning-capable model", async () => { + instance = await createServer(allFixtures); + const res = await post(geminiUrl(instance.url, "gemini-2.5-pro", GEN), { + contents: [{ role: "user", parts: [{ text: "explain" }] }], + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as GeminiChunk; + const thoughts = thoughtParts(body.candidates[0].content.parts); + expect(thoughts).toHaveLength(1); + expect(thoughts[0].text).toBe(REASONING); + }); + + it("strict OFF: emits thought parts for a non-reasoning model but warns", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { logLevel: "warn" }); + const res = await post(geminiUrl(instance.url, "gemini-1.5-pro", GEN), { + contents: [{ role: "user", parts: [{ text: "explain" }] }], + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as GeminiChunk; + expect(thoughtParts(body.candidates[0].content.parts)).toHaveLength(1); + expect(warn).toHaveBeenCalled(); + expect(warn.mock.calls.flat().join(" ")).toContain("gemini-1.5-pro"); + }); + + it("strict ON: suppresses thought parts for a non-reasoning model", async () => { + instance = await createServer(allFixtures); + const res = await post( + geminiUrl(instance.url, "gemini-1.5-pro", GEN), + { contents: [{ role: "user", parts: [{ text: "explain" }] }] }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as GeminiChunk; + expect(thoughtParts(body.candidates[0].content.parts)).toHaveLength(0); + // Content is still present. + expect(body.candidates[0].content.parts.some((p) => p.text === "The answer is 42.")).toBe(true); + }); + + it("emits thought parts for an unknown model (defaults to capable)", async () => { + instance = await createServer(allFixtures); + const res = await post(geminiUrl(instance.url, "some-future-model", GEN), { + contents: [{ role: "user", parts: [{ text: "explain" }] }], + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as GeminiChunk; + expect(thoughtParts(body.candidates[0].content.parts)).toHaveLength(1); + }); + + it("no-op when fixture carries no reasoning (no thought parts, no log)", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + const error = vi.spyOn(console, "error").mockImplementation(() => {}); + instance = await createServer(allFixtures, { logLevel: "warn" }); + const res = await post( + geminiUrl(instance.url, "gemini-1.5-pro", GEN), + { contents: [{ role: "user", parts: [{ text: "plain" }] }] }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as GeminiChunk; + expect(thoughtParts(body.candidates[0].content.parts)).toHaveLength(0); + expect(warn).not.toHaveBeenCalled(); + expect(error).not.toHaveBeenCalled(); + }); + + it("strict ON: suppresses reasoning on the content+toolCalls path", async () => { + instance = await createServer(allFixtures); + const res = await post( + geminiUrl(instance.url, "gemini-1.5-pro", GEN), + { contents: [{ role: "user", parts: [{ text: "tooled" }] }] }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as GeminiChunk; + expect(thoughtParts(body.candidates[0].content.parts)).toHaveLength(0); + // Tool call still present. + expect(body.candidates[0].content.parts.some((p) => p.functionCall)).toBe(true); + }); + + it("content+toolCalls path emits reasoning for a capable model", async () => { + instance = await createServer(allFixtures); + const res = await post(geminiUrl(instance.url, "gemini-2.5-pro", GEN), { + contents: [{ role: "user", parts: [{ text: "tooled" }] }], + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as GeminiChunk; + expect(thoughtParts(body.candidates[0].content.parts)).toHaveLength(1); + }); +}); + +describe("Gemini reasoning capability gating (streaming)", () => { + it("emits thought chunks for a reasoning-capable model", async () => { + instance = await createServer(allFixtures); + const res = await post(geminiUrl(instance.url, "gemini-2.5-pro", STREAM_GEN), { + contents: [{ role: "user", parts: [{ text: "explain" }] }], + }); + + expect(res.status).toBe(200); + const chunks = parseGeminiSSEChunks(res.body); + expect(reasoningInChunks(chunks)).toBe(true); + }); + + it("strict ON: suppresses thought chunks for a non-reasoning model", async () => { + instance = await createServer(allFixtures); + const res = await post( + geminiUrl(instance.url, "gemini-1.5-pro", STREAM_GEN), + { contents: [{ role: "user", parts: [{ text: "explain" }] }] }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const chunks = parseGeminiSSEChunks(res.body); + expect(reasoningInChunks(chunks)).toBe(false); + // Content chunks still present. + const fullText = chunks.map((c) => c.candidates[0].content.parts[0]?.text ?? "").join(""); + expect(fullText).toContain("The answer is 42."); + }); + + it("strict OFF: emits thought chunks for a non-reasoning model and warns", async () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { logLevel: "warn" }); + const res = await post(geminiUrl(instance.url, "gemini-1.5-pro", STREAM_GEN), { + contents: [{ role: "user", parts: [{ text: "explain" }] }], + }); + + expect(res.status).toBe(200); + const chunks = parseGeminiSSEChunks(res.body); + expect(reasoningInChunks(chunks)).toBe(true); + expect(warn).toHaveBeenCalled(); + }); + + it("strict ON: suppresses reasoning on the streaming content+toolCalls path", async () => { + instance = await createServer(allFixtures); + const res = await post( + geminiUrl(instance.url, "gemini-1.5-pro", STREAM_GEN), + { contents: [{ role: "user", parts: [{ text: "tooled" }] }] }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const chunks = parseGeminiSSEChunks(res.body); + expect(reasoningInChunks(chunks)).toBe(false); + expect(chunks.some((c) => c.candidates[0].content.parts.some((p) => p.functionCall))).toBe( + true, + ); + }); +}); diff --git a/src/__tests__/model-utils.test.ts b/src/__tests__/model-utils.test.ts index bf998556..bd85edab 100644 --- a/src/__tests__/model-utils.test.ts +++ b/src/__tests__/model-utils.test.ts @@ -1,5 +1,5 @@ -import { describe, it, expect } from "vitest"; -import { normalizeModelName } from "../model-utils.js"; +import { describe, it, expect, afterEach, vi } from "vitest"; +import { normalizeModelName, isReasoningModel } from "../model-utils.js"; describe("normalizeModelName", () => { it("strips 8-digit date suffix", () => { @@ -36,3 +36,92 @@ describe("normalizeModelName", () => { expect(normalizeModelName("claude-opus-4-20250514", true)).toBe("claude-opus-4-20250514"); }); }); + +describe("isReasoningModel", () => { + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("classifies reasoning-capable families as true", () => { + expect(isReasoningModel("o1")).toBe(true); + expect(isReasoningModel("o1-mini")).toBe(true); + expect(isReasoningModel("o3-mini")).toBe(true); + expect(isReasoningModel("o4-mini")).toBe(true); + expect(isReasoningModel("gpt-5")).toBe(true); + expect(isReasoningModel("gpt-5-mini")).toBe(true); + expect(isReasoningModel("deepseek-r1")).toBe(true); + expect(isReasoningModel("deepseek-reasoner")).toBe(true); + expect(isReasoningModel("claude-3-7-sonnet")).toBe(true); + expect(isReasoningModel("claude-opus-4-20250514")).toBe(true); // date-suffixed + expect(isReasoningModel("claude-sonnet-4")).toBe(true); + expect(isReasoningModel("gemini-2.5-pro")).toBe(true); + expect(isReasoningModel("gemini-2.0-flash-thinking-exp")).toBe(true); + expect(isReasoningModel("qwq-32b")).toBe(true); + }); + + it("classifies known non-reasoning families as false", () => { + expect(isReasoningModel("gpt-4.1")).toBe(false); + expect(isReasoningModel("gpt-4.1-mini")).toBe(false); + expect(isReasoningModel("gpt-4.1-nano")).toBe(false); + expect(isReasoningModel("gpt-4o")).toBe(false); + expect(isReasoningModel("gpt-4o-mini")).toBe(false); + expect(isReasoningModel("gpt-4o-2024-11-20")).toBe(false); // date-suffixed + expect(isReasoningModel("gpt-4")).toBe(false); + expect(isReasoningModel("gpt-4-turbo")).toBe(false); + expect(isReasoningModel("gpt-3.5-turbo")).toBe(false); + expect(isReasoningModel("claude-3-5-sonnet-20241022")).toBe(false); + expect(isReasoningModel("claude-3-haiku")).toBe(false); + expect(isReasoningModel("claude-3-opus")).toBe(false); + expect(isReasoningModel("gemini-1.5-pro")).toBe(false); + expect(isReasoningModel("gemini-1.5-flash")).toBe(false); + }); + + it("strips a Bedrock provider prefix before matching", () => { + expect(isReasoningModel("anthropic.claude-3-5-sonnet-20241022-v2:0")).toBe(false); + expect(isReasoningModel("us.anthropic.claude-3-5-sonnet-20241022-v2:0")).toBe(false); + expect(isReasoningModel("anthropic.claude-opus-4-20250514-v1:0")).toBe(true); + }); + + it("defaults unknown models to true (reasoning-capable)", () => { + expect(isReasoningModel("some-future-model")).toBe(true); + expect(isReasoningModel("llama3.1")).toBe(true); + expect(isReasoningModel("mistral")).toBe(true); + }); + + it("defaults undefined/empty to true (no model id is not evidence of incapability)", () => { + expect(isReasoningModel(undefined)).toBe(true); + expect(isReasoningModel("")).toBe(true); + }); + + it("env override AIMOCK_NONREASONING_MODELS forces a model to false", () => { + vi.stubEnv("AIMOCK_NONREASONING_MODELS", "my-model,another-one"); + expect(isReasoningModel("my-model")).toBe(false); + expect(isReasoningModel("another-one")).toBe(false); + // unrelated model unaffected + expect(isReasoningModel("o3-mini")).toBe(true); + }); + + it("env override AIMOCK_REASONING_MODELS forces a built-in non-reasoning model to true", () => { + vi.stubEnv("AIMOCK_REASONING_MODELS", "gpt-4.1"); + expect(isReasoningModel("gpt-4.1")).toBe(true); + }); + + it("env-nonreasoning wins over env-reasoning (documented precedence)", () => { + vi.stubEnv("AIMOCK_REASONING_MODELS", "shared-model"); + vi.stubEnv("AIMOCK_NONREASONING_MODELS", "shared-model"); + expect(isReasoningModel("shared-model")).toBe(false); + }); + + it("env-nonreasoning entry with a provider prefix matches the stripped id", () => { + vi.stubEnv("AIMOCK_NONREASONING_MODELS", "anthropic.claude-opus-4"); + expect(isReasoningModel("anthropic.claude-opus-4-20250514-v1:0")).toBe(false); + }); + + it("env-reasoning entry with a provider prefix fires for the stripped id", () => { + // Without the override the stripped id `gpt-4.1` hits the built-in + // non-reasoning denylist (→ false); the prefixed env-reasoning entry must + // match the stripped id and force it back to true. + vi.stubEnv("AIMOCK_REASONING_MODELS", "anthropic.gpt-4.1"); + expect(isReasoningModel("anthropic.gpt-4.1-mini")).toBe(true); + }); +}); diff --git a/src/__tests__/openai-chat-reasoning-capability.test.ts b/src/__tests__/openai-chat-reasoning-capability.test.ts new file mode 100644 index 00000000..c3ea15fc --- /dev/null +++ b/src/__tests__/openai-chat-reasoning-capability.test.ts @@ -0,0 +1,258 @@ +import { describe, it, expect, afterEach, vi } from "vitest"; +import * as http from "node:http"; +import type { Fixture } from "../types.js"; +import { createServer, type ServerInstance } from "../server.js"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +async function httpPost( + url: string, + body: unknown, + headers?: Record, +): Promise<{ status: number; body: string }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify(body); + const req = http.request( + url, + { + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => + resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString() }), + ); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +interface ChatSSEEvent { + choices?: { + delta?: { content?: string; reasoning_content?: string; role?: string }; + finish_reason?: string | null; + }[]; + [key: string]: unknown; +} + +function parseSSEEvents(body: string): ChatSSEEvent[] { + const events: ChatSSEEvent[] = []; + for (const line of body.split("\n")) { + if (line.startsWith("data: ") && line.slice(6).trim() !== "[DONE]") { + events.push(JSON.parse(line.slice(6)) as ChatSSEEvent); + } + } + return events; +} + +/** Non-stream: does the assistant message carry a reasoning_content field? */ +function nonStreamReasoning(body: string): string | undefined { + const parsed = JSON.parse(body) as { + choices?: { message?: { content?: string; reasoning_content?: string } }[]; + }; + return parsed.choices?.[0]?.message?.reasoning_content; +} + +/** Non-stream: the normal assistant content. */ +function nonStreamContent(body: string): string | undefined { + const parsed = JSON.parse(body) as { choices?: { message?: { content?: string } }[] }; + return parsed.choices?.[0]?.message?.content; +} + +/** Stream: are there any reasoning_content deltas? */ +function hasReasoningStreamDeltas(body: string): boolean { + return parseSSEEvents(body).some((e) => e.choices?.[0]?.delta?.reasoning_content !== undefined); +} + +/** Stream: concatenated normal content deltas. */ +function streamContent(body: string): string { + return parseSSEEvents(body) + .filter((e) => { + const c = e.choices?.[0]?.delta?.content; + return c !== undefined && c !== ""; + }) + .map((e) => e.choices![0].delta!.content) + .join(""); +} + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const reasoningFixture: Fixture = { + match: { userMessage: "think" }, + response: { + content: "The answer is 42.", + reasoning: "Let me reason step by step.", + }, +}; + +const noReasoningFixture: Fixture = { + match: { userMessage: "plain" }, + response: { content: "Just plain text." }, +}; + +const allFixtures: Fixture[] = [reasoningFixture, noReasoningFixture]; + +// --------------------------------------------------------------------------- +// Server lifecycle +// --------------------------------------------------------------------------- + +let instance: ServerInstance | null = null; + +afterEach(async () => { + vi.restoreAllMocks(); + if (instance) { + await new Promise((resolve) => instance!.server.close(() => resolve())); + instance = null; + } +}); + +function chatBody(userMessage: string, model: string, stream: boolean) { + return { + model, + messages: [{ role: "user", content: userMessage }], + stream, + }; +} + +// --------------------------------------------------------------------------- +// Tests — canonical aimock#254 repro path: /v1/chat/completions + gpt-4.1 +// --------------------------------------------------------------------------- + +describe("/v1/chat/completions reasoning capability gating (aimock#254 repro)", () => { + it("non-reasoning model gpt-4.1, strict OFF: still emits reasoning but warns (non-stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/chat/completions`, + chatBody("think", "gpt-4.1", false), + ); + expect(res.status).toBe(200); + expect(nonStreamReasoning(res.body)).toBe("Let me reason step by step."); + expect(nonStreamContent(res.body)).toBe("The answer is 42."); + expect(warnSpy).toHaveBeenCalled(); + expect(warnSpy.mock.calls.flat().join(" ")).toContain("gpt-4.1"); + }); + + it("non-reasoning model gpt-4.1, strict OFF: still emits reasoning but warns (stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/chat/completions`, + chatBody("think", "gpt-4.1", true), + ); + expect(res.status).toBe(200); + expect(hasReasoningStreamDeltas(res.body)).toBe(true); + expect(streamContent(res.body)).toBe("The answer is 42."); + expect(warnSpy).toHaveBeenCalled(); + expect(warnSpy.mock.calls.flat().join(" ")).toContain("gpt-4.1"); + }); + + it("non-reasoning model gpt-4.1, strict ON (header): suppresses reasoning, content intact (non-stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await httpPost( + `${instance.url}/v1/chat/completions`, + chatBody("think", "gpt-4.1", false), + { "X-AIMock-Strict": "true" }, + ); + expect(res.status).toBe(200); + expect(nonStreamReasoning(res.body)).toBeUndefined(); + expect(nonStreamContent(res.body)).toBe("The answer is 42."); + }); + + it("non-reasoning model gpt-4.1, strict ON (header): suppresses reasoning, content intact (stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await httpPost( + `${instance.url}/v1/chat/completions`, + chatBody("think", "gpt-4.1", true), + { "X-AIMock-Strict": "true" }, + ); + expect(res.status).toBe(200); + expect(hasReasoningStreamDeltas(res.body)).toBe(false); + expect(streamContent(res.body)).toBe("The answer is 42."); + }); + + it("non-reasoning model gpt-4.1, server --strict: suppresses reasoning (non-stream)", async () => { + instance = await createServer(allFixtures, { port: 0, strict: true }); + const res = await httpPost( + `${instance.url}/v1/chat/completions`, + chatBody("think", "gpt-4.1", false), + ); + expect(res.status).toBe(200); + expect(nonStreamReasoning(res.body)).toBeUndefined(); + expect(nonStreamContent(res.body)).toBe("The answer is 42."); + }); + + it("non-reasoning model gpt-4.1, server --strict: suppresses reasoning (stream)", async () => { + instance = await createServer(allFixtures, { port: 0, strict: true }); + const res = await httpPost( + `${instance.url}/v1/chat/completions`, + chatBody("think", "gpt-4.1", true), + ); + expect(res.status).toBe(200); + expect(hasReasoningStreamDeltas(res.body)).toBe(false); + expect(streamContent(res.body)).toBe("The answer is 42."); + }); + + it("reasoning-capable model o3-mini: emits reasoning, no warn (non-stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/chat/completions`, + chatBody("think", "o3-mini", false), + ); + expect(res.status).toBe(200); + expect(nonStreamReasoning(res.body)).toBe("Let me reason step by step."); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it("reasoning-capable model o3-mini: emits reasoning, no warn (stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/chat/completions`, + chatBody("think", "o3-mini", true), + ); + expect(res.status).toBe(200); + expect(hasReasoningStreamDeltas(res.body)).toBe(true); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it("fixture with no reasoning, gpt-4.1: no-op, no warn (non-stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/chat/completions`, + chatBody("plain", "gpt-4.1", false), + ); + expect(res.status).toBe(200); + expect(nonStreamReasoning(res.body)).toBeUndefined(); + expect(nonStreamContent(res.body)).toBe("Just plain text."); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it("fixture with no reasoning, gpt-4.1: no-op, no warn (stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/chat/completions`, + chatBody("plain", "gpt-4.1", true), + ); + expect(res.status).toBe(200); + expect(hasReasoningStreamDeltas(res.body)).toBe(false); + expect(streamContent(res.body)).toBe("Just plain text."); + expect(warnSpy).not.toHaveBeenCalled(); + }); +}); diff --git a/src/__tests__/reasoning-all-providers.test.ts b/src/__tests__/reasoning-all-providers.test.ts index ae95182f..b3c29c4d 100644 --- a/src/__tests__/reasoning-all-providers.test.ts +++ b/src/__tests__/reasoning-all-providers.test.ts @@ -195,7 +195,7 @@ afterEach(async () => { describe("POST /v1/chat/completions (reasoning non-streaming)", () => { it("includes reasoning_content field on assistant message", async () => { const res = await post(`/v1/chat/completions`, { - model: "gpt-4", + model: "o3-mini", messages: [{ role: "user", content: "think" }], stream: false, }); @@ -224,7 +224,7 @@ describe("POST /v1/chat/completions (reasoning non-streaming)", () => { describe("POST /v1/chat/completions (reasoning streaming)", () => { it("emits reasoning_content deltas before content deltas", async () => { const res = await post(`/v1/chat/completions`, { - model: "gpt-4", + model: "o3-mini", messages: [{ role: "user", content: "think" }], stream: true, }); diff --git a/src/__tests__/reasoning-capability-anthropic.test.ts b/src/__tests__/reasoning-capability-anthropic.test.ts new file mode 100644 index 00000000..b0c75a81 --- /dev/null +++ b/src/__tests__/reasoning-capability-anthropic.test.ts @@ -0,0 +1,495 @@ +/** + * Anthropic Messages API — model-capability-aware reasoning gating (aimock#254). + * + * aimock synthesizes a `thinking` channel whenever a fixture carries a + * `reasoning` string. These tests assert the emission is now gated on the + * REQUESTED model's reasoning capability: + * - reasoning-capable Claude model + reasoning fixture → emit thinking block + * - non-reasoning model + reasoning fixture, strict OFF → emit + logger.warn + * - non-reasoning model + reasoning fixture, strict ON → suppress + logger.error + * - fixture without reasoning → no-op (no thinking, no log) + * + * Driven through `handleMessages` directly so the test owns the Logger (to spy + * on warn/error) and a body-capturing mock response (to assert on the wire + * payload for both streaming and non-streaming paths). + */ +import { describe, it, expect, vi } from "vitest"; +import * as http from "node:http"; +import { PassThrough } from "node:stream"; +import type { Fixture, HandlerDefaults } from "../types.js"; +import { handleMessages } from "../messages.js"; +import { Journal } from "../journal.js"; +import { Logger } from "../logger.js"; + +// --- helpers --- + +const REASONING_TEXT = "Let me think step by step about this problem."; + +const reasoningFixture: Fixture = { + match: { userMessage: "think" }, + response: { content: "The answer is 42.", reasoning: REASONING_TEXT }, +}; + +const plainFixture: Fixture = { + match: { userMessage: "plain" }, + response: { content: "Just plain text." }, +}; + +const CONTENT_TEXT = "I will call a tool."; + +const contentWithToolsReasoningFixture: Fixture = { + match: { userMessage: "tooled" }, + response: { + content: CONTENT_TEXT, + reasoning: REASONING_TEXT, + toolCalls: [{ name: "get_weather", arguments: '{"city":"NYC"}' }], + }, +}; + +const allFixtures: Fixture[] = [reasoningFixture, plainFixture, contentWithToolsReasoningFixture]; + +/** Mock ServerResponse that captures everything written to the body. */ +function createCapturingRes(): { res: http.ServerResponse; getBody: () => string } { + const res = new PassThrough() as unknown as http.ServerResponse; + let ended = false; + let body = ""; + const headers: Record = {}; + res.setHeader = (name: string, value: string | number | readonly string[]) => { + headers[name.toLowerCase()] = String(value); + return res; + }; + res.writeHead = ((statusCode: number, hdrs?: Record) => { + (res as { statusCode: number }).statusCode = statusCode; + if (hdrs) { + for (const [k, v] of Object.entries(hdrs)) headers[k.toLowerCase()] = v; + } + return res; + }) as typeof res.writeHead; + res.write = ((chunk: string | Buffer) => { + body += typeof chunk === "string" ? chunk : chunk.toString(); + return true; + }) as typeof res.write; + res.end = ((...args: unknown[]) => { + const chunk = args[0]; + if (typeof chunk === "string") body += chunk; + else if (Buffer.isBuffer(chunk)) body += chunk.toString(); + ended = true; + return res; + }) as typeof res.end; + Object.defineProperty(res, "writableEnded", { get: () => ended }); + res.destroy = () => { + ended = true; + return res; + }; + return { res, getBody: () => body }; +} + +function makeReq(headers: http.IncomingHttpHeaders = {}): http.IncomingMessage { + return { + method: "POST", + url: "/v1/messages", + headers, + } as unknown as http.IncomingMessage; +} + +function makeDefaults(logger: Logger, strict?: boolean): HandlerDefaults { + return { latency: 0, chunkSize: 10, replaySpeed: 0, logger, strict }; +} + +interface ClaudeContentBlock { + type: string; + thinking?: string; + text?: string; + name?: string; + input?: Record; +} + +/** Parse Claude SSE frames into their JSON `data:` payloads. */ +function parseClaudeSSE(body: string): Array> { + const events: Array> = []; + for (const line of body.split("\n")) { + if (line.startsWith("data: ")) { + events.push(JSON.parse(line.slice(6)) as Record); + } + } + return events; +} + +async function run( + bodyObj: object, + defaults: HandlerDefaults, + reqHeaders: http.IncomingHttpHeaders = {}, +): Promise { + const { res, getBody } = createCapturingRes(); + await handleMessages( + makeReq(reqHeaders), + res, + JSON.stringify(bodyObj), + allFixtures, + new Journal(), + defaults, + () => {}, + ); + return getBody(); +} + +function thinkingBlocks(jsonBody: string): ClaudeContentBlock[] { + const parsed = JSON.parse(jsonBody) as { content: ClaudeContentBlock[] }; + return parsed.content.filter((b) => b.type === "thinking"); +} + +function streamThinkingDeltas(sseBody: string): string[] { + return parseClaudeSSE(sseBody) + .filter( + (e) => + e.type === "content_block_delta" && + (e.delta as { type?: string } | undefined)?.type === "thinking_delta", + ) + .map((e) => (e.delta as { thinking: string }).thinking); +} + +/** All top-level content blocks (in order) from a non-streaming response. */ +function contentBlocks(jsonBody: string): ClaudeContentBlock[] { + return (JSON.parse(jsonBody) as { content: ClaudeContentBlock[] }).content; +} + +/** `content_block_start` blocks from an SSE stream, ordered by their index. */ +function streamStartedBlocks(sseBody: string): ClaudeContentBlock[] { + return parseClaudeSSE(sseBody) + .filter((e) => e.type === "content_block_start") + .sort((a, b) => (a.index as number) - (b.index as number)) + .map((e) => e.content_block as ClaudeContentBlock); +} + +/** Reassembled text from `text_delta` SSE events. */ +function streamTextDeltas(sseBody: string): string { + return parseClaudeSSE(sseBody) + .filter( + (e) => + e.type === "content_block_delta" && + (e.delta as { type?: string } | undefined)?.type === "text_delta", + ) + .map((e) => (e.delta as { text: string }).text) + .join(""); +} + +// ─── reasoning-capable Claude model → emit thinking ───────────────────────── + +describe("Anthropic /v1/messages reasoning gating — capable model", () => { + it("non-streaming: emits a thinking block, no warn/error", async () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + const body = await run( + { model: "claude-opus-4", max_tokens: 1024, messages: [{ role: "user", content: "think" }] }, + makeDefaults(logger), + ); + + const blocks = thinkingBlocks(body); + expect(blocks).toHaveLength(1); + expect(blocks[0].thinking).toBe(REASONING_TEXT); + expect(warn).not.toHaveBeenCalled(); + expect(error).not.toHaveBeenCalled(); + }); + + it("streaming: emits thinking_delta events, no warn/error", async () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + const body = await run( + { + model: "claude-opus-4", + max_tokens: 1024, + stream: true, + messages: [{ role: "user", content: "think" }], + }, + makeDefaults(logger), + ); + + expect(streamThinkingDeltas(body).join("")).toBe(REASONING_TEXT); + expect(warn).not.toHaveBeenCalled(); + expect(error).not.toHaveBeenCalled(); + }); +}); + +// ─── non-reasoning model, strict OFF → emit + warn ────────────────────────── + +describe("Anthropic /v1/messages reasoning gating — non-reasoning model, strict OFF", () => { + it("non-streaming: still emits thinking but logs a warn naming the model", async () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + const body = await run( + { + model: "claude-3-5-sonnet-20241022", + max_tokens: 1024, + messages: [{ role: "user", content: "think" }], + }, + makeDefaults(logger, false), + ); + + expect(thinkingBlocks(body)).toHaveLength(1); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0]?.join(" ")).toContain("claude-3-5-sonnet-20241022"); + expect(error).not.toHaveBeenCalled(); + }); + + it("streaming: still emits thinking deltas + a single warn", async () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + const body = await run( + { + model: "claude-3-5-sonnet-20241022", + max_tokens: 1024, + stream: true, + messages: [{ role: "user", content: "think" }], + }, + makeDefaults(logger, false), + ); + + expect(streamThinkingDeltas(body).join("")).toBe(REASONING_TEXT); + expect(warn).toHaveBeenCalledTimes(1); + expect(error).not.toHaveBeenCalled(); + }); +}); + +// ─── non-reasoning model, strict ON → suppress + error ────────────────────── + +describe("Anthropic /v1/messages reasoning gating — non-reasoning model, strict ON", () => { + it("non-streaming: suppresses the thinking block and logs an error", async () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + const body = await run( + { + model: "claude-3-5-sonnet-20241022", + max_tokens: 1024, + messages: [{ role: "user", content: "think" }], + }, + makeDefaults(logger, true), + ); + + expect(thinkingBlocks(body)).toHaveLength(0); + expect(error).toHaveBeenCalledTimes(1); + expect(error.mock.calls[0]?.join(" ")).toContain("claude-3-5-sonnet-20241022"); + expect(warn).not.toHaveBeenCalled(); + }); + + it("streaming: emits no thinking deltas and logs an error", async () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + const body = await run( + { + model: "claude-3-5-sonnet-20241022", + max_tokens: 1024, + stream: true, + messages: [{ role: "user", content: "think" }], + }, + makeDefaults(logger, true), + ); + + expect(streamThinkingDeltas(body)).toHaveLength(0); + expect(error).toHaveBeenCalledTimes(1); + expect(warn).not.toHaveBeenCalled(); + }); + + it("per-request X-AIMock-Strict header overrides a non-strict server (suppresses)", async () => { + const logger = new Logger("warn"); + const error = vi.spyOn(logger, "error"); + + const body = await run( + { + model: "claude-3-5-sonnet-20241022", + max_tokens: 1024, + messages: [{ role: "user", content: "think" }], + }, + makeDefaults(logger, false), + { "x-aimock-strict": "true" }, + ); + + expect(thinkingBlocks(body)).toHaveLength(0); + expect(error).toHaveBeenCalledTimes(1); + }); +}); + +// ─── fixture without reasoning → no-op ────────────────────────────────────── + +describe("Anthropic /v1/messages reasoning gating — no fixture reasoning", () => { + it("non-reasoning model + no reasoning: no thinking block, no log (even strict)", async () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + const body = await run( + { + model: "claude-3-5-sonnet-20241022", + max_tokens: 1024, + messages: [{ role: "user", content: "plain" }], + }, + makeDefaults(logger, true), + ); + + expect(thinkingBlocks(body)).toHaveLength(0); + expect(warn).not.toHaveBeenCalled(); + expect(error).not.toHaveBeenCalled(); + }); +}); + +// ─── unknown model → fail open (emit) ─────────────────────────────────────── + +describe("Anthropic /v1/messages reasoning gating — unknown model", () => { + it("unknown model + reasoning fixture: emits thinking, no warn (even strict)", async () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + const body = await run( + { + model: "some-future-claude", + max_tokens: 1024, + messages: [{ role: "user", content: "think" }], + }, + makeDefaults(logger, true), + ); + + expect(thinkingBlocks(body)).toHaveLength(1); + expect(warn).not.toHaveBeenCalled(); + expect(error).not.toHaveBeenCalled(); + }); +}); + +// ─── content + toolCalls branch → gating on the combined-output path ───────── +// +// The content-with-tool-calls dispatch synthesizes thinking from the same +// `reasoning` field, but through a separate code path. These cases assert the +// capability gate is wired there too: the thinking block is gated while the +// text content and tool_use blocks always survive, in order. + +describe("Anthropic /v1/messages reasoning gating — content+toolCalls, strict ON suppresses", () => { + it("non-streaming: suppresses thinking, keeps text + tool_use in order", async () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + const body = await run( + { + model: "claude-3-5-sonnet-20241022", + max_tokens: 1024, + messages: [{ role: "user", content: "tooled" }], + }, + makeDefaults(logger, true), + ); + + const blocks = contentBlocks(body); + expect(blocks.filter((b) => b.type === "thinking")).toHaveLength(0); + + // text content + tool_use both intact, and correctly ordered (text first). + const text = blocks.find((b) => b.type === "text"); + const toolUse = blocks.find((b) => b.type === "tool_use"); + expect(text?.text).toBe(CONTENT_TEXT); + expect(toolUse?.name).toBe("get_weather"); + expect(toolUse?.input).toEqual({ city: "NYC" }); + expect(blocks.findIndex((b) => b.type === "text")).toBeLessThan( + blocks.findIndex((b) => b.type === "tool_use"), + ); + + expect(error).toHaveBeenCalledTimes(1); + expect(error.mock.calls[0]?.join(" ")).toContain("claude-3-5-sonnet-20241022"); + expect(warn).not.toHaveBeenCalled(); + }); + + it("streaming: emits no thinking deltas, keeps text + tool_use blocks in order", async () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + const body = await run( + { + model: "claude-3-5-sonnet-20241022", + max_tokens: 1024, + stream: true, + messages: [{ role: "user", content: "tooled" }], + }, + makeDefaults(logger, true), + ); + + expect(streamThinkingDeltas(body)).toHaveLength(0); + + const started = streamStartedBlocks(body); + expect(started.some((b) => b.type === "thinking")).toBe(false); + expect(started.map((b) => b.type)).toEqual(["text", "tool_use"]); + expect(streamTextDeltas(body)).toBe(CONTENT_TEXT); + expect(started.find((b) => b.type === "tool_use")?.name).toBe("get_weather"); + + expect(error).toHaveBeenCalledTimes(1); + expect(warn).not.toHaveBeenCalled(); + }); +}); + +describe("Anthropic /v1/messages reasoning gating — content+toolCalls, capable model emits", () => { + it("non-streaming: emits thinking, keeps text + tool_use in order", async () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + const body = await run( + { + model: "claude-opus-4", + max_tokens: 1024, + messages: [{ role: "user", content: "tooled" }], + }, + makeDefaults(logger), + ); + + const blocks = contentBlocks(body); + const thinking = blocks.filter((b) => b.type === "thinking"); + expect(thinking).toHaveLength(1); + expect(thinking[0].thinking).toBe(REASONING_TEXT); + + const text = blocks.find((b) => b.type === "text"); + const toolUse = blocks.find((b) => b.type === "tool_use"); + expect(text?.text).toBe(CONTENT_TEXT); + expect(toolUse?.name).toBe("get_weather"); + expect(toolUse?.input).toEqual({ city: "NYC" }); + + // Order: thinking → text → tool_use. + expect(blocks.map((b) => b.type)).toEqual(["thinking", "text", "tool_use"]); + + expect(warn).not.toHaveBeenCalled(); + expect(error).not.toHaveBeenCalled(); + }); + + it("streaming: emits thinking deltas, keeps text + tool_use blocks in order", async () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + const body = await run( + { + model: "claude-opus-4", + max_tokens: 1024, + stream: true, + messages: [{ role: "user", content: "tooled" }], + }, + makeDefaults(logger), + ); + + expect(streamThinkingDeltas(body).join("")).toBe(REASONING_TEXT); + + const started = streamStartedBlocks(body); + expect(started.map((b) => b.type)).toEqual(["thinking", "text", "tool_use"]); + expect(streamTextDeltas(body)).toBe(CONTENT_TEXT); + expect(started.find((b) => b.type === "tool_use")?.name).toBe("get_weather"); + + expect(warn).not.toHaveBeenCalled(); + expect(error).not.toHaveBeenCalled(); + }); +}); diff --git a/src/__tests__/reasoning-capability-ollama.test.ts b/src/__tests__/reasoning-capability-ollama.test.ts new file mode 100644 index 00000000..50b23206 --- /dev/null +++ b/src/__tests__/reasoning-capability-ollama.test.ts @@ -0,0 +1,337 @@ +import { describe, it, expect, afterEach, vi } from "vitest"; +import http from "node:http"; +import { createServer, type ServerInstance } from "../server.js"; +import type { Fixture } from "../types.js"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +async function httpPost( + url: string, + body: object, + headers?: Record, +): Promise<{ status: number; body: string }> { + return new Promise((resolve, reject) => { + const req = http.request( + url, + { + method: "POST", + headers: { + "Content-Type": "application/json", + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => + resolve({ + status: res.statusCode!, + body: Buffer.concat(chunks).toString(), + }), + ); + }, + ); + req.on("error", reject); + req.write(JSON.stringify(body)); + req.end(); + }); +} + +function parseNDJSON(body: string): Array> { + return body + .split("\n") + .filter((line) => line.trim() !== "") + .map((line) => JSON.parse(line) as Record); +} + +const REASONING = "Let me think step by step about this problem."; + +// Match on userMessage only so any requested model id routes to the same +// reasoning-bearing fixture; the model id drives the capability gate. +const fixtures: Fixture[] = [ + { + match: { userMessage: "think" }, + response: { content: "The answer is 42.", reasoning: REASONING }, + }, + { + match: { userMessage: "plain" }, + response: { content: "Just plain text." }, + }, +]; + +let instance: ServerInstance | null = null; + +afterEach(async () => { + vi.unstubAllEnvs(); + if (instance) { + await new Promise((resolve) => { + instance!.server.close(() => resolve()); + }); + instance = null; + } +}); + +// --------------------------------------------------------------------------- +// /api/chat — reasoning-capable model emits reasoning +// --------------------------------------------------------------------------- + +describe("Ollama /api/chat reasoning gating — reasoning-capable model", () => { + it("non-streaming: reasoning-capable id emits reasoning_content", async () => { + instance = await createServer(fixtures); + const res = await httpPost(`${instance.url}/api/chat`, { + model: "deepseek-r1", + messages: [{ role: "user", content: "think" }], + stream: false, + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.message.content).toBe("The answer is 42."); + expect(body.message.reasoning_content).toBe(REASONING); + }); + + it("streaming: reasoning-capable id emits reasoning_content chunks", async () => { + instance = await createServer(fixtures); + const res = await httpPost(`${instance.url}/api/chat`, { + model: "deepseek-r1", + messages: [{ role: "user", content: "think" }], + stream: true, + }); + + expect(res.status).toBe(200); + const chunks = parseNDJSON(res.body) as Array<{ + message: { reasoning_content?: string }; + done: boolean; + }>; + const reasoningChunks = chunks.filter( + (c) => !c.done && c.message?.reasoning_content !== undefined, + ); + expect(reasoningChunks.length).toBeGreaterThan(0); + expect(reasoningChunks.map((c) => c.message.reasoning_content).join("")).toBe(REASONING); + }); +}); + +// --------------------------------------------------------------------------- +// /api/chat — non-reasoning model (forced via env override): warn vs suppress +// --------------------------------------------------------------------------- + +describe("Ollama /api/chat reasoning gating — non-reasoning model", () => { + it("strict OFF: still emits reasoning_content (warn-by-default)", async () => { + vi.stubEnv("AIMOCK_NONREASONING_MODELS", "no-think-model"); + instance = await createServer(fixtures); + const res = await httpPost(`${instance.url}/api/chat`, { + model: "no-think-model", + messages: [{ role: "user", content: "think" }], + stream: false, + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.message.content).toBe("The answer is 42."); + expect(body.message.reasoning_content).toBe(REASONING); + }); + + it("strict ON: suppresses reasoning_content (non-streaming)", async () => { + vi.stubEnv("AIMOCK_NONREASONING_MODELS", "no-think-model"); + instance = await createServer(fixtures); + const res = await httpPost( + `${instance.url}/api/chat`, + { + model: "no-think-model", + messages: [{ role: "user", content: "think" }], + stream: false, + }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.message.content).toBe("The answer is 42."); + expect(body.message.reasoning_content).toBeUndefined(); + }); + + it("strict ON: suppresses reasoning_content chunks (streaming)", async () => { + vi.stubEnv("AIMOCK_NONREASONING_MODELS", "no-think-model"); + instance = await createServer(fixtures); + const res = await httpPost( + `${instance.url}/api/chat`, + { + model: "no-think-model", + messages: [{ role: "user", content: "think" }], + stream: true, + }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const chunks = parseNDJSON(res.body) as Array<{ + message: { reasoning_content?: string }; + done: boolean; + }>; + const reasoningChunks = chunks.filter( + (c) => !c.done && c.message?.reasoning_content !== undefined, + ); + expect(reasoningChunks.length).toBe(0); + }); +}); + +// --------------------------------------------------------------------------- +// /api/chat — unknown local model id defaults to reasoning-capable (emit) +// --------------------------------------------------------------------------- + +describe("Ollama /api/chat reasoning gating — unknown local model", () => { + it("unknown local id emits reasoning (fail-open default)", async () => { + instance = await createServer(fixtures); + const res = await httpPost(`${instance.url}/api/chat`, { + model: "llama3.1", + messages: [{ role: "user", content: "think" }], + stream: false, + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.message.reasoning_content).toBe(REASONING); + }); + + it("unknown local id, strict ON, still emits (unknown defaults capable)", async () => { + instance = await createServer(fixtures); + const res = await httpPost( + `${instance.url}/api/chat`, + { + model: "mistral", + messages: [{ role: "user", content: "think" }], + stream: false, + }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.message.reasoning_content).toBe(REASONING); + }); +}); + +// --------------------------------------------------------------------------- +// /api/chat — no fixture reasoning is a no-op +// --------------------------------------------------------------------------- + +describe("Ollama /api/chat reasoning gating — no fixture reasoning", () => { + it("no reasoning_content when fixture has no reasoning (even non-reasoning model)", async () => { + vi.stubEnv("AIMOCK_NONREASONING_MODELS", "no-think-model"); + instance = await createServer(fixtures); + const res = await httpPost(`${instance.url}/api/chat`, { + model: "no-think-model", + messages: [{ role: "user", content: "plain" }], + stream: false, + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.message.content).toBe("Just plain text."); + expect(body.message.reasoning_content).toBeUndefined(); + }); +}); + +// --------------------------------------------------------------------------- +// /api/generate — same gating on the generate path +// --------------------------------------------------------------------------- + +describe("Ollama /api/generate reasoning gating", () => { + it("reasoning-capable id emits reasoning_content (non-streaming)", async () => { + instance = await createServer(fixtures); + const res = await httpPost(`${instance.url}/api/generate`, { + model: "deepseek-r1", + prompt: "think", + stream: false, + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.response).toBe("The answer is 42."); + expect(body.reasoning_content).toBe(REASONING); + }); + + it("non-reasoning id, strict ON, suppresses reasoning_content (non-streaming)", async () => { + vi.stubEnv("AIMOCK_NONREASONING_MODELS", "no-think-model"); + instance = await createServer(fixtures); + const res = await httpPost( + `${instance.url}/api/generate`, + { + model: "no-think-model", + prompt: "think", + stream: false, + }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.response).toBe("The answer is 42."); + expect(body.reasoning_content).toBeUndefined(); + }); + + it("non-reasoning id, strict OFF, still emits reasoning_content (warn-by-default)", async () => { + vi.stubEnv("AIMOCK_NONREASONING_MODELS", "no-think-model"); + instance = await createServer(fixtures); + const res = await httpPost(`${instance.url}/api/generate`, { + model: "no-think-model", + prompt: "think", + stream: false, + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.reasoning_content).toBe(REASONING); + }); + + it("non-reasoning id, strict ON, suppresses reasoning_content chunks (streaming)", async () => { + vi.stubEnv("AIMOCK_NONREASONING_MODELS", "no-think-model"); + instance = await createServer(fixtures); + const res = await httpPost( + `${instance.url}/api/generate`, + { + model: "no-think-model", + prompt: "think", + stream: true, + }, + { "X-AIMock-Strict": "true" }, + ); + + expect(res.status).toBe(200); + const chunks = parseNDJSON(res.body) as Array<{ + reasoning_content?: string; + done: boolean; + }>; + const reasoningChunks = chunks.filter((c) => !c.done && c.reasoning_content !== undefined); + expect(reasoningChunks.length).toBe(0); + }); + + it("unknown local id emits reasoning (fail-open default)", async () => { + instance = await createServer(fixtures); + const res = await httpPost(`${instance.url}/api/generate`, { + model: "llama3.1", + prompt: "think", + stream: false, + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.reasoning_content).toBe(REASONING); + }); + + it("no reasoning_content when fixture has no reasoning", async () => { + instance = await createServer(fixtures); + const res = await httpPost(`${instance.url}/api/generate`, { + model: "deepseek-r1", + prompt: "plain", + stream: false, + }); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body.response).toBe("Just plain text."); + expect(body.reasoning_content).toBeUndefined(); + }); +}); diff --git a/src/__tests__/reasoning-web-search.test.ts b/src/__tests__/reasoning-web-search.test.ts index 435bb4f8..0f40eff6 100644 --- a/src/__tests__/reasoning-web-search.test.ts +++ b/src/__tests__/reasoning-web-search.test.ts @@ -112,7 +112,7 @@ describe("POST /v1/responses (reasoning streaming)", () => { it("emits reasoning events before text events", async () => { instance = await createServer(allFixtures); const res = await post(`${instance.url}/v1/responses`, { - model: "gpt-4", + model: "o3-mini", input: [{ role: "user", content: "think" }], stream: true, }); @@ -135,7 +135,7 @@ describe("POST /v1/responses (reasoning streaming)", () => { it("reasoning deltas reconstruct full reasoning text", async () => { instance = await createServer(allFixtures); const res = await post(`${instance.url}/v1/responses`, { - model: "gpt-4", + model: "o3-mini", input: [{ role: "user", content: "think" }], stream: true, }); @@ -151,7 +151,7 @@ describe("POST /v1/responses (reasoning streaming)", () => { it("text deltas still reconstruct full content", async () => { instance = await createServer(allFixtures); const res = await post(`${instance.url}/v1/responses`, { - model: "gpt-4", + model: "o3-mini", input: [{ role: "user", content: "think" }], stream: true, }); @@ -165,7 +165,7 @@ describe("POST /v1/responses (reasoning streaming)", () => { it("response.completed includes reasoning output item", async () => { instance = await createServer(allFixtures); const res = await post(`${instance.url}/v1/responses`, { - model: "gpt-4", + model: "o3-mini", input: [{ role: "user", content: "think" }], stream: true, }); @@ -268,7 +268,7 @@ describe("POST /v1/responses (combined reasoning + web search)", () => { it("emits reasoning, then web search, then text events in order", async () => { instance = await createServer(allFixtures); const res = await post(`${instance.url}/v1/responses`, { - model: "gpt-4", + model: "o3-mini", input: [{ role: "user", content: "combined" }], stream: true, }); @@ -303,7 +303,7 @@ describe("POST /v1/responses (combined reasoning + web search)", () => { it("response.completed output includes all item types in order", async () => { instance = await createServer(allFixtures); const res = await post(`${instance.url}/v1/responses`, { - model: "gpt-4", + model: "o3-mini", input: [{ role: "user", content: "combined" }], stream: true, }); @@ -325,7 +325,7 @@ describe("POST /v1/responses (non-streaming with reasoning)", () => { it("includes reasoning output item in non-streaming response", async () => { instance = await createServer(allFixtures); const res = await post(`${instance.url}/v1/responses`, { - model: "gpt-4", + model: "o3-mini", input: [{ role: "user", content: "think" }], stream: false, }); @@ -362,7 +362,7 @@ describe("POST /v1/responses (non-streaming with reasoning)", () => { it("combined non-streaming response has correct output order", async () => { instance = await createServer(allFixtures); const res = await post(`${instance.url}/v1/responses`, { - model: "gpt-4", + model: "o3-mini", input: [{ role: "user", content: "combined" }], stream: false, }); @@ -578,7 +578,7 @@ describe("POST /v1/chat/completions (reasoning_content streaming)", () => { it("emits reasoning_content deltas before content deltas", async () => { instance = await createServer(allFixtures); const res = await post(`${instance.url}/v1/chat/completions`, { - model: "gpt-4", + model: "o3-mini", messages: [{ role: "user", content: "think" }], stream: true, }); @@ -603,7 +603,7 @@ describe("POST /v1/chat/completions (reasoning_content streaming)", () => { it("reasoning_content deltas reconstruct full reasoning text", async () => { instance = await createServer(allFixtures); const res = await post(`${instance.url}/v1/chat/completions`, { - model: "gpt-4", + model: "o3-mini", messages: [{ role: "user", content: "think" }], stream: true, }); @@ -616,7 +616,7 @@ describe("POST /v1/chat/completions (reasoning_content streaming)", () => { it("content deltas still reconstruct full text", async () => { instance = await createServer(allFixtures); const res = await post(`${instance.url}/v1/chat/completions`, { - model: "gpt-4", + model: "o3-mini", messages: [{ role: "user", content: "think" }], stream: true, }); @@ -644,7 +644,7 @@ describe("POST /v1/chat/completions (reasoning_content non-streaming)", () => { it("includes reasoning_content in non-streaming response", async () => { instance = await createServer(allFixtures); const res = await post(`${instance.url}/v1/chat/completions`, { - model: "gpt-4", + model: "o3-mini", messages: [{ role: "user", content: "think" }], stream: false, }); diff --git a/src/__tests__/resolve-reasoning-for-model.test.ts b/src/__tests__/resolve-reasoning-for-model.test.ts new file mode 100644 index 00000000..868a6bfa --- /dev/null +++ b/src/__tests__/resolve-reasoning-for-model.test.ts @@ -0,0 +1,61 @@ +import { describe, it, expect, vi } from "vitest"; +import { resolveReasoningForModel } from "../helpers.js"; +import { Logger } from "../logger.js"; + +describe("resolveReasoningForModel", () => { + it("returns undefined when there is no reasoning to emit (no-op)", () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + expect(resolveReasoningForModel(undefined, "gpt-4.1", false, logger)).toBeUndefined(); + expect(resolveReasoningForModel("", "gpt-4.1", true, logger)).toBeUndefined(); + + expect(warn).not.toHaveBeenCalled(); + expect(error).not.toHaveBeenCalled(); + }); + + it("emits unchanged for a reasoning-capable model, no log", () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + expect(resolveReasoningForModel("thinking…", "o3-mini", false, logger)).toBe("thinking…"); + expect(resolveReasoningForModel("thinking…", "o3-mini", true, logger)).toBe("thinking…"); + + expect(warn).not.toHaveBeenCalled(); + expect(error).not.toHaveBeenCalled(); + }); + + it("warns but preserves emission for a non-reasoning model when strict is OFF", () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + const result = resolveReasoningForModel("thinking…", "gpt-4.1", false, logger); + + expect(result).toBe("thinking…"); + expect(warn).toHaveBeenCalledTimes(1); + expect(warn.mock.calls[0]?.join(" ")).toContain("gpt-4.1"); + expect(error).not.toHaveBeenCalled(); + }); + + it("suppresses emission and logs error for a non-reasoning model when strict is ON", () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + const result = resolveReasoningForModel("thinking…", "gpt-4.1", true, logger); + + expect(result).toBeUndefined(); + expect(error).toHaveBeenCalledTimes(1); + expect(error.mock.calls[0]?.join(" ")).toContain("gpt-4.1"); + expect(warn).not.toHaveBeenCalled(); + }); + + it("uses the requested model id when deciding capability", () => { + const logger = new Logger("warn"); + // unknown model defaults to capable → emits unchanged, no log + expect(resolveReasoningForModel("r", "some-future-model", true, logger)).toBe("r"); + }); +}); diff --git a/src/__tests__/responses-reasoning-capability.test.ts b/src/__tests__/responses-reasoning-capability.test.ts new file mode 100644 index 00000000..ea8681b7 --- /dev/null +++ b/src/__tests__/responses-reasoning-capability.test.ts @@ -0,0 +1,264 @@ +import { describe, it, expect, afterEach, vi } from "vitest"; +import * as http from "node:http"; +import type { Fixture } from "../types.js"; +import { createServer, type ServerInstance } from "../server.js"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +async function httpPost( + url: string, + body: unknown, + headers?: Record, +): Promise<{ status: number; body: string }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify(body); + const req = http.request( + url, + { + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => + resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString() }), + ); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +interface ResponsesSSEEvent { + type?: string; + [key: string]: unknown; +} + +function parseSSEEvents(body: string): ResponsesSSEEvent[] { + const events: ResponsesSSEEvent[] = []; + for (const line of body.split("\n")) { + if (line.startsWith("data: ") && line.slice(6).trim() !== "[DONE]") { + events.push(JSON.parse(line.slice(6)) as ResponsesSSEEvent); + } + } + return events; +} + +/** Non-stream: does the response output[] contain a reasoning item? */ +function hasReasoningOutputItem(body: string): boolean { + const parsed = JSON.parse(body) as { output?: Array<{ type?: string }> }; + return (parsed.output ?? []).some((item) => item.type === "reasoning"); +} + +/** Stream: are there any reasoning summary events? */ +function hasReasoningStreamEvents(body: string): boolean { + return parseSSEEvents(body).some( + (e) => typeof e.type === "string" && e.type.startsWith("response.reasoning_summary_text"), + ); +} + +// --------------------------------------------------------------------------- +// Fixtures +// --------------------------------------------------------------------------- + +const reasoningFixture: Fixture = { + match: { userMessage: "think" }, + response: { + content: "The answer is 42.", + reasoning: "Let me reason step by step.", + }, +}; + +const reasoningWithToolFixture: Fixture = { + match: { userMessage: "tool-think" }, + response: { + content: "Calling a tool.", + reasoning: "Deliberating about the tool call.", + toolCalls: [{ name: "lookup", arguments: '{"q":"x"}' }], + }, +}; + +const noReasoningFixture: Fixture = { + match: { userMessage: "plain" }, + response: { content: "Just plain text." }, +}; + +const allFixtures: Fixture[] = [reasoningFixture, reasoningWithToolFixture, noReasoningFixture]; + +// --------------------------------------------------------------------------- +// Server lifecycle +// --------------------------------------------------------------------------- + +let instance: ServerInstance | null = null; + +afterEach(async () => { + vi.restoreAllMocks(); + if (instance) { + await new Promise((resolve) => instance!.server.close(() => resolve())); + instance = null; + } +}); + +function responsesBody(userMessage: string, model: string, stream: boolean) { + return { + model, + input: [{ role: "user", content: userMessage }], + stream, + }; +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +describe("Responses API reasoning capability gating — text response", () => { + it("reasoning-capable model emits reasoning (non-stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await httpPost(`${instance.url}/v1/responses`, responsesBody("think", "o3", false)); + expect(res.status).toBe(200); + expect(hasReasoningOutputItem(res.body)).toBe(true); + }); + + it("reasoning-capable model emits reasoning (stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await httpPost(`${instance.url}/v1/responses`, responsesBody("think", "o3", true)); + expect(res.status).toBe(200); + expect(hasReasoningStreamEvents(res.body)).toBe(true); + }); + + it("non-reasoning model, strict OFF: still emits reasoning but warns (non-stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("think", "gpt-4.1", false), + ); + expect(res.status).toBe(200); + expect(hasReasoningOutputItem(res.body)).toBe(true); + expect(warnSpy).toHaveBeenCalled(); + expect(warnSpy.mock.calls.flat().join(" ")).toContain("gpt-4.1"); + }); + + it("non-reasoning model, strict OFF: still emits reasoning but warns (stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("think", "gpt-4.1", true), + ); + expect(res.status).toBe(200); + expect(hasReasoningStreamEvents(res.body)).toBe(true); + expect(warnSpy).toHaveBeenCalled(); + }); + + it("non-reasoning model, strict ON (header): suppresses reasoning (non-stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("think", "gpt-4.1", false), + { "X-AIMock-Strict": "true" }, + ); + expect(res.status).toBe(200); + expect(hasReasoningOutputItem(res.body)).toBe(false); + }); + + it("non-reasoning model, strict ON (header): suppresses reasoning (stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("think", "gpt-4.1", true), + { "X-AIMock-Strict": "true" }, + ); + expect(res.status).toBe(200); + expect(hasReasoningStreamEvents(res.body)).toBe(false); + }); + + it("non-reasoning model, server --strict: suppresses reasoning (non-stream)", async () => { + instance = await createServer(allFixtures, { port: 0, strict: true }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("think", "gpt-4.1", false), + ); + expect(res.status).toBe(200); + expect(hasReasoningOutputItem(res.body)).toBe(false); + }); + + it("unknown model emits reasoning, no warn (non-stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("think", "future-x", false), + ); + expect(res.status).toBe(200); + expect(hasReasoningOutputItem(res.body)).toBe(true); + expect(warnSpy).not.toHaveBeenCalled(); + }); + + it("no fixture reasoning: no-op, no warn even for non-reasoning model", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("plain", "gpt-4.1", false), + ); + expect(res.status).toBe(200); + expect(hasReasoningOutputItem(res.body)).toBe(false); + expect(warnSpy).not.toHaveBeenCalled(); + }); +}); + +describe("Responses API reasoning capability gating — content+toolCalls response", () => { + it("reasoning-capable model emits reasoning (non-stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("tool-think", "o3", false), + ); + expect(res.status).toBe(200); + expect(hasReasoningOutputItem(res.body)).toBe(true); + }); + + it("non-reasoning model, strict ON: suppresses reasoning (non-stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("tool-think", "gpt-4.1", false), + { "X-AIMock-Strict": "true" }, + ); + expect(res.status).toBe(200); + expect(hasReasoningOutputItem(res.body)).toBe(false); + }); + + it("non-reasoning model, strict ON: suppresses reasoning (stream)", async () => { + instance = await createServer(allFixtures, { port: 0 }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("tool-think", "gpt-4.1", true), + { "X-AIMock-Strict": "true" }, + ); + expect(res.status).toBe(200); + expect(hasReasoningStreamEvents(res.body)).toBe(false); + }); + + it("non-reasoning model, strict OFF: emits reasoning + warns (non-stream)", async () => { + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + instance = await createServer(allFixtures, { port: 0, logLevel: "warn" }); + const res = await httpPost( + `${instance.url}/v1/responses`, + responsesBody("tool-think", "gpt-4.1", false), + ); + expect(res.status).toBe(200); + expect(hasReasoningOutputItem(res.body)).toBe(true); + expect(warnSpy).toHaveBeenCalled(); + }); +}); diff --git a/src/__tests__/ws-responses.test.ts b/src/__tests__/ws-responses.test.ts index 0edaebd0..6da5e675 100644 --- a/src/__tests__/ws-responses.test.ts +++ b/src/__tests__/ws-responses.test.ts @@ -30,7 +30,21 @@ const reasoningFixture: Fixture = { response: { content: "The answer.", reasoning: "Let me reason about this." }, }; -const allFixtures: Fixture[] = [textFixture, toolFixture, errorFixture, reasoningFixture]; +// Reasoning fixture used by the capability-gating tests. Distinct match key so it +// can be exercised against different requested models without colliding with the +// `gpt-4`-defaulting "think" fixture above. +const capabilityReasoningFixture: Fixture = { + match: { userMessage: "reason-please" }, + response: { content: "The answer.", reasoning: "Let me reason about this." }, +}; + +const allFixtures: Fixture[] = [ + textFixture, + toolFixture, + errorFixture, + reasoningFixture, + capabilityReasoningFixture, +]; // --- tests --- @@ -471,3 +485,105 @@ describe("WebSocket /v1/responses", () => { expect(reasoningIdx).toBeLessThan(textIdx); }); }); + +// ─── Capability-aware reasoning gating: WebSocket /v1/responses ─────────────── + +describe("WebSocket /v1/responses reasoning capability gating", () => { + it("emits reasoning for a reasoning-capable model", async () => { + instance = await createServer(allFixtures); + const ws = await connectWebSocket(instance.url, "/v1/responses"); + + ws.send(responseCreateMsg("reason-please", "o3-mini")); + + const raw = await ws.waitForMessages(15); + const events = parseEvents(raw); + const types = events.map((e) => e.type); + + expect(types).toContain("response.reasoning_summary_text.delta"); + expect(types).toContain("response.output_text.delta"); + + ws.close(); + }); + + it("emits reasoning (warn-by-default) for a non-reasoning model when strict is off", async () => { + instance = await createServer(allFixtures); + const ws = await connectWebSocket(instance.url, "/v1/responses"); + + // gpt-4.1 is a known non-reasoning model; strict is off by default, so the + // reasoning channel is still emitted (a warning is logged at non-silent levels). + ws.send(responseCreateMsg("reason-please", "gpt-4.1")); + + const raw = await ws.waitForMessages(15); + const events = parseEvents(raw); + const types = events.map((e) => e.type); + + expect(types).toContain("response.reasoning_summary_text.delta"); + expect(types).toContain("response.output_text.delta"); + + ws.close(); + }); + + it("suppresses reasoning for a non-reasoning model when strict is on (upgrade header)", async () => { + instance = await createServer(allFixtures); + // Strict resolves from the connection upgrade headers on the WS path; pass + // X-AIMock-Strict via the upgrade handshake. + const ws = await connectWebSocket(instance.url, "/v1/responses", { + "X-AIMock-Strict": "true", + }); + + ws.send(responseCreateMsg("reason-please", "gpt-4.1")); + + // Without reasoning the text response is a shorter sequence; wait for the + // terminal response.completed rather than a reasoning-inflated count. + const raw = await ws.waitForMessages(9); + const events = parseEvents(raw); + const types = events.map((e) => e.type); + + expect(types).not.toContain("response.reasoning_summary_text.delta"); + expect(types).not.toContain("response.reasoning_summary_text.done"); + // Text emission is unaffected. + expect(types).toContain("response.output_text.delta"); + expect(types[types.length - 1]).toBe("response.completed"); + + ws.close(); + }); + + it("still emits reasoning for a reasoning-capable model under strict mode", async () => { + instance = await createServer(allFixtures); + const ws = await connectWebSocket(instance.url, "/v1/responses", { + "X-AIMock-Strict": "true", + }); + + ws.send(responseCreateMsg("reason-please", "o3-mini")); + + const raw = await ws.waitForMessages(15); + const events = parseEvents(raw); + const types = events.map((e) => e.type); + + expect(types).toContain("response.reasoning_summary_text.delta"); + expect(types).toContain("response.output_text.delta"); + + ws.close(); + }); + + it("no-op when the fixture carries no reasoning (non-reasoning model, strict on)", async () => { + instance = await createServer(allFixtures); + const ws = await connectWebSocket(instance.url, "/v1/responses", { + "X-AIMock-Strict": "true", + }); + + // textFixture has no reasoning; gating short-circuits before any model check. + ws.send(responseCreateMsg("hello", "gpt-4.1")); + + const raw = await ws.waitForMessages(9); + const events = parseEvents(raw); + const types = events.map((e) => e.type); + + expect(types).not.toContain("response.reasoning_summary_text.delta"); + expect(types).toContain("response.output_text.delta"); + const deltas = events.filter((e) => e.type === "response.output_text.delta"); + expect(deltas.map((d) => d.delta).join("")).toBe("Hi there!"); + + ws.close(); + }); +}); From 44b08a5d7ec74562aff50c5af94e3e13942a5d69 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Mon, 8 Jun 2026 16:31:25 -0700 Subject: [PATCH 4/4] docs: add changelog entry for model-capability-aware reasoning emission --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 45297c1f..eeb5de37 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Changed + +- **Reasoning emission** — replaying a reasoning channel is now gated on the requested model's capability. aimock no longer synthesizes a reasoning channel (chat `reasoning_content` / Responses `reasoning_summary_text` / Anthropic thinking / etc.) for models that would not emit reasoning against the real provider. A new `isReasoningModel` classifier and `resolveReasoningForModel` gate are applied across OpenAI chat + Responses, Anthropic, Ollama, Gemini, Cohere, Bedrock (invoke + Converse), and WebSocket Responses: a non-reasoning model paired with a reasoning fixture has its reasoning suppressed under strict mode, or warns-and-emits otherwise. The `AIMOCK_REASONING_MODELS` and `AIMOCK_NONREASONING_MODELS` env vars override the classifier. + ## [1.29.0] - 2026-06-04 ### Added