diff --git a/CHANGELOG.md b/CHANGELOG.md index eeb5de37..f723d1f7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ ## [Unreleased] +### Added + +- **Extended-thinking request invariants** — aimock now validates Anthropic extended-thinking continuations on the tool-use loop. When extended thinking is enabled, a continuation whose prior assistant turn drops the leading `thinking` block (or its `signature`, or a `redacted_thinking` block's `data`) is rejected with the real Anthropic `400`, instead of producing a false-green replay — under strict mode; otherwise the violation warns and replay proceeds. Emitted thinking blocks now carry a non-empty placeholder signature so record→replay round-trips stay green across text, content+tool, and tool-only response shapes. + ### Changed - **Reasoning emission** — replaying a reasoning channel is now gated on the requested model's capability. aimock no longer synthesizes a reasoning channel (chat `reasoning_content` / Responses `reasoning_summary_text` / Anthropic thinking / etc.) for models that would not emit reasoning against the real provider. A new `isReasoningModel` classifier and `resolveReasoningForModel` gate are applied across OpenAI chat + Responses, Anthropic, Ollama, Gemini, Cohere, Bedrock (invoke + Converse), and WebSocket Responses: a non-reasoning model paired with a reasoning fixture has its reasoning suppressed under strict mode, or warns-and-emits otherwise. The `AIMOCK_REASONING_MODELS` and `AIMOCK_NONREASONING_MODELS` env vars override the classifier. diff --git a/src/__tests__/drift/anthropic.drift.ts b/src/__tests__/drift/anthropic.drift.ts index 80d782d5..8c888599 100644 --- a/src/__tests__/drift/anthropic.drift.ts +++ b/src/__tests__/drift/anthropic.drift.ts @@ -212,7 +212,10 @@ describe("Anthropic Claude extended thinking shapes", () => { expect(mockBody.content.length).toBe(2); expect(mockBody.content[0].type).toBe("thinking"); expect(mockBody.content[0].thinking).toBe("I need to consider..."); - expect(mockBody.content[0].signature).toBe(""); + // Real Anthropic non-streaming returns a non-empty cryptographic signature + // on the assembled thinking block (assembled here from the signature_delta). + expect(typeof mockBody.content[0].signature).toBe("string"); + expect(mockBody.content[0].signature.length).toBeGreaterThan(0); expect(mockBody.content[1].type).toBe("text"); expect(mockBody.content[1].text).toBe("Hello!"); @@ -261,7 +264,11 @@ describe("Anthropic Claude extended thinking shapes", () => { (e) => e.type === "content_block_delta" && e.data?.delta?.type === "signature_delta", ); expect(signatureDeltas.length, "Missing signature_delta event").toBe(1); - expect(signatureDeltas[0].data.delta.signature).toBe(""); + // Real Anthropic delivers the non-empty cryptographic signature here (the + // `content_block_start` carried ""); an SDK assembles the block's signature + // from this delta. + expect(typeof signatureDeltas[0].data.delta.signature).toBe("string"); + expect(signatureDeltas[0].data.delta.signature.length).toBeGreaterThan(0); // Verify text block follows thinking block const textBlockStart = mockEvents.find( diff --git a/src/__tests__/helpers/mock-res.ts b/src/__tests__/helpers/mock-res.ts index 679f90a9..2013f629 100644 --- a/src/__tests__/helpers/mock-res.ts +++ b/src/__tests__/helpers/mock-res.ts @@ -53,6 +53,7 @@ export function createDefaults(overrides: Partial = {}): Handle return { latency: 0, chunkSize: 100, + replaySpeed: 1.0, logger: new Logger("silent"), ...overrides, }; diff --git a/src/__tests__/reasoning-capability-anthropic.test.ts b/src/__tests__/reasoning-capability-anthropic.test.ts index b0c75a81..b50e29c0 100644 --- a/src/__tests__/reasoning-capability-anthropic.test.ts +++ b/src/__tests__/reasoning-capability-anthropic.test.ts @@ -46,7 +46,20 @@ const contentWithToolsReasoningFixture: Fixture = { }, }; -const allFixtures: Fixture[] = [reasoningFixture, plainFixture, contentWithToolsReasoningFixture]; +const toolOnlyReasoningFixture: Fixture = { + match: { userMessage: "toolonly" }, + response: { + reasoning: REASONING_TEXT, + toolCalls: [{ name: "get_weather", arguments: '{"city":"NYC"}' }], + }, +}; + +const allFixtures: Fixture[] = [ + reasoningFixture, + plainFixture, + contentWithToolsReasoningFixture, + toolOnlyReasoningFixture, +]; /** Mock ServerResponse that captures everything written to the body. */ function createCapturingRes(): { res: http.ServerResponse; getBody: () => string } { @@ -493,3 +506,125 @@ describe("Anthropic /v1/messages reasoning gating — content+toolCalls, capable expect(error).not.toHaveBeenCalled(); }); }); + +// ─── tool-only branch → gating on the pure-tool-call path ──────────────────── +// +// aimock#253 newly synthesized a leading thinking block on the pure-tool-call +// dispatch (no text content) so a replayed tool-only turn under extended +// thinking does not self-trip `missing_thinking_first`. aimock#254's capability +// gate never covered that path (it didn't emit reasoning at #254's time). These +// cases lock in that the gate now applies there too: a non-reasoning model under +// strict SUPPRESSES the thinking block, a capable model EMITS it — while the +// tool_use block always survives. (Red without the `resolveReasoningForModel` +// routing: raw `response.reasoning` would emit the thinking block unconditionally.) + +describe("Anthropic /v1/messages reasoning gating — tool-only, strict ON suppresses", () => { + it("non-streaming: suppresses thinking, keeps tool_use", async () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + const body = await run( + { + model: "claude-3-5-sonnet-20241022", + max_tokens: 1024, + messages: [{ role: "user", content: "toolonly" }], + }, + makeDefaults(logger, true), + ); + + const blocks = contentBlocks(body); + expect(blocks.filter((b) => b.type === "thinking")).toHaveLength(0); + + const toolUse = blocks.find((b) => b.type === "tool_use"); + expect(toolUse?.name).toBe("get_weather"); + expect(toolUse?.input).toEqual({ city: "NYC" }); + + expect(error).toHaveBeenCalledTimes(1); + expect(error.mock.calls[0]?.join(" ")).toContain("claude-3-5-sonnet-20241022"); + expect(warn).not.toHaveBeenCalled(); + }); + + it("streaming: emits no thinking deltas, keeps tool_use block", async () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + const body = await run( + { + model: "claude-3-5-sonnet-20241022", + max_tokens: 1024, + stream: true, + messages: [{ role: "user", content: "toolonly" }], + }, + makeDefaults(logger, true), + ); + + expect(streamThinkingDeltas(body)).toHaveLength(0); + + const started = streamStartedBlocks(body); + expect(started.some((b) => b.type === "thinking")).toBe(false); + expect(started.map((b) => b.type)).toEqual(["tool_use"]); + expect(started.find((b) => b.type === "tool_use")?.name).toBe("get_weather"); + + expect(error).toHaveBeenCalledTimes(1); + expect(warn).not.toHaveBeenCalled(); + }); +}); + +describe("Anthropic /v1/messages reasoning gating — tool-only, capable model emits", () => { + it("non-streaming: emits thinking, keeps tool_use in order", async () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + const body = await run( + { + model: "claude-opus-4", + max_tokens: 1024, + messages: [{ role: "user", content: "toolonly" }], + }, + makeDefaults(logger), + ); + + const blocks = contentBlocks(body); + const thinking = blocks.filter((b) => b.type === "thinking"); + expect(thinking).toHaveLength(1); + expect(thinking[0].thinking).toBe(REASONING_TEXT); + + const toolUse = blocks.find((b) => b.type === "tool_use"); + expect(toolUse?.name).toBe("get_weather"); + expect(toolUse?.input).toEqual({ city: "NYC" }); + + // Order: thinking → tool_use. + expect(blocks.map((b) => b.type)).toEqual(["thinking", "tool_use"]); + + expect(warn).not.toHaveBeenCalled(); + expect(error).not.toHaveBeenCalled(); + }); + + it("streaming: emits thinking deltas, keeps tool_use block in order", async () => { + const logger = new Logger("warn"); + const warn = vi.spyOn(logger, "warn"); + const error = vi.spyOn(logger, "error"); + + const body = await run( + { + model: "claude-opus-4", + max_tokens: 1024, + stream: true, + messages: [{ role: "user", content: "toolonly" }], + }, + makeDefaults(logger), + ); + + expect(streamThinkingDeltas(body).join("")).toBe(REASONING_TEXT); + + const started = streamStartedBlocks(body); + expect(started.map((b) => b.type)).toEqual(["thinking", "tool_use"]); + expect(started.find((b) => b.type === "tool_use")?.name).toBe("get_weather"); + + expect(warn).not.toHaveBeenCalled(); + expect(error).not.toHaveBeenCalled(); + }); +}); diff --git a/src/__tests__/thinking-invariants.test.ts b/src/__tests__/thinking-invariants.test.ts new file mode 100644 index 00000000..62ab28b9 --- /dev/null +++ b/src/__tests__/thinking-invariants.test.ts @@ -0,0 +1,929 @@ +import { describe, it, expect, afterEach, vi } from "vitest"; +import * as http from "node:http"; +import type { Fixture } from "../types.js"; +import { createServer, type ServerInstance } from "../server.js"; +import { + validateThinkingInvariants, + handleMessages, + claudeToCompletionRequest, +} from "../messages.js"; +import { Journal } from "../journal.js"; +import { Logger } from "../logger.js"; +import { createMockRes, createDefaults } from "./helpers/mock-res.js"; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +function post( + url: string, + body: unknown, + headers?: Record, +): Promise<{ status: number; body: string }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify(body); + const parsed = new URL(url); + const req = http.request( + { + hostname: parsed.hostname, + port: parsed.port, + path: parsed.pathname, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => + resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString() }), + ); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +function parseClaudeSSEEvents(body: string): Array> { + const events: Array> = []; + for (const line of body.split("\n")) { + if (line.startsWith("data: ")) { + events.push(JSON.parse(line.slice(6)) as Record); + } + } + return events; +} + +// --------------------------------------------------------------------------- +// Request builders +// --------------------------------------------------------------------------- + +const ENABLED = { type: "enabled" as const, budget_tokens: 1024 }; + +// A well-formed in-scope continuation: assistant turn leads with a thinking +// block + signature, carries a tool_use, and the next user turn answers it. +function continuationRequest(opts: { + thinking?: unknown; + // The assistant turn's content blocks (the in-scope turn at index 1). + assistantContent: unknown; + // Whether to append a user tool_result turn answering tool_use id "tu_1". + answer?: boolean; +}): Record { + const messages: unknown[] = [ + { role: "user", content: "What is the weather?" }, + { role: "assistant", content: opts.assistantContent }, + ]; + if (opts.answer !== false) { + messages.push({ + role: "user", + content: [{ type: "tool_result", tool_use_id: "tu_1", content: "Sunny" }], + }); + } + return { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + ...(opts.thinking !== undefined ? { thinking: opts.thinking } : {}), + messages, + }; +} + +const validThinkingBlock = { type: "thinking", thinking: "Let me check.", signature: "sig-abc" }; +const toolUseBlock = { type: "tool_use", id: "tu_1", name: "get_weather", input: { city: "NYC" } }; + +// --------------------------------------------------------------------------- +// 8.1 Pure-helper unit tests +// --------------------------------------------------------------------------- + +describe("validateThinkingInvariants (pure helper)", () => { + it("U1: thinking disabled (no thinking field) → null", () => { + const req = continuationRequest({ assistantContent: [toolUseBlock] }); + expect(validateThinkingInvariants(req as never)).toBeNull(); + }); + + it("U2: thinking.type disabled → null", () => { + const req = continuationRequest({ + thinking: { type: "disabled" }, + assistantContent: [toolUseBlock], + }); + expect(validateThinkingInvariants(req as never)).toBeNull(); + }); + + it("U3: malformed thinking (true) → null", () => { + const req = continuationRequest({ thinking: true, assistantContent: [toolUseBlock] }); + expect(validateThinkingInvariants(req as never)).toBeNull(); + }); + + it("U4: malformed thinking ({}) → null", () => { + const req = continuationRequest({ thinking: {}, assistantContent: [toolUseBlock] }); + expect(validateThinkingInvariants(req as never)).toBeNull(); + }); + + it("U5: enabled, single user turn only → null", () => { + const req = { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + thinking: ENABLED, + messages: [{ role: "user", content: "hello" }], + }; + expect(validateThinkingInvariants(req as never)).toBeNull(); + }); + + it("U6: enabled, in-scope turn leads with valid thinking + signature → null", () => { + const req = continuationRequest({ + thinking: ENABLED, + assistantContent: [validThinkingBlock, toolUseBlock], + }); + expect(validateThinkingInvariants(req as never)).toBeNull(); + }); + + it("U7: enabled, text-only assistant turn (no tool_use) → null (exempt)", () => { + const req = continuationRequest({ + thinking: ENABLED, + assistantContent: [{ type: "text", text: "Final answer." }], + answer: false, + }); + expect(validateThinkingInvariants(req as never)).toBeNull(); + }); + + it("U8: enabled, tool_use turn answered, leads with text → missing_thinking_first", () => { + const req = continuationRequest({ + thinking: ENABLED, + assistantContent: [{ type: "text", text: "thinking..." }, toolUseBlock], + }); + expect(validateThinkingInvariants(req as never)).toEqual({ + kind: "missing_thinking_first", + messageIndex: 1, + observedFirstBlockType: "text", + }); + }); + + it("U9: enabled, first block is tool_use (reasoning dropped) → missing_thinking_first", () => { + const req = continuationRequest({ + thinking: ENABLED, + assistantContent: [toolUseBlock], + }); + expect(validateThinkingInvariants(req as never)).toEqual({ + kind: "missing_thinking_first", + messageIndex: 1, + observedFirstBlockType: "tool_use", + }); + }); + + it("U10: enabled, trailing unanswered tool_use (no tool_result) → null (out of scope)", () => { + const req = continuationRequest({ + thinking: ENABLED, + assistantContent: [toolUseBlock], + answer: false, + }); + expect(validateThinkingInvariants(req as never)).toBeNull(); + }); + + it("U11: enabled, leading thinking with empty signature → missing_signature", () => { + const req = continuationRequest({ + thinking: ENABLED, + assistantContent: [{ type: "thinking", thinking: "x", signature: "" }, toolUseBlock], + }); + expect(validateThinkingInvariants(req as never)).toEqual({ + kind: "missing_signature", + messageIndex: 1, + }); + }); + + it("U12: enabled, leading thinking with missing signature → missing_signature", () => { + const req = continuationRequest({ + thinking: ENABLED, + assistantContent: [{ type: "thinking", thinking: "x" }, toolUseBlock], + }); + expect(validateThinkingInvariants(req as never)).toEqual({ + kind: "missing_signature", + messageIndex: 1, + }); + }); + + it("U13: enabled, leading redacted_thinking with non-empty data → null", () => { + const req = continuationRequest({ + thinking: ENABLED, + assistantContent: [{ type: "redacted_thinking", data: "encrypted" }, toolUseBlock], + }); + expect(validateThinkingInvariants(req as never)).toBeNull(); + }); + + it("U14: enabled, in-scope leading redacted_thinking with missing data → dropped_redacted_thinking", () => { + const req = continuationRequest({ + thinking: ENABLED, + assistantContent: [{ type: "redacted_thinking" }, toolUseBlock], + }); + expect(validateThinkingInvariants(req as never)).toEqual({ + kind: "dropped_redacted_thinking", + messageIndex: 1, + }); + }); + + it("U14b: enabled, out-of-scope redacted_thinking with empty data → null (boundary guard)", () => { + const req = continuationRequest({ + thinking: ENABLED, + assistantContent: [{ type: "redacted_thinking", data: "" }, toolUseBlock], + answer: false, + }); + expect(validateThinkingInvariants(req as never)).toBeNull(); + }); + + it("U14c: enabled, in-scope leading redacted_thinking with empty-string data → dropped_redacted_thinking", () => { + // Exercises the empty-string `data` case of invariant (c) on an in-scope + // turn. U14 covers `data` undefined (missing); this covers the empty-string + // case. Both the current `length === 0` guard and a `!first.data` guard + // satisfy this — it documents the empty-string intent, it does not + // discriminate between those implementations. + const req = continuationRequest({ + thinking: ENABLED, + assistantContent: [{ type: "redacted_thinking", data: "" }, toolUseBlock], + }); + expect(validateThinkingInvariants(req as never)).toEqual({ + kind: "dropped_redacted_thinking", + messageIndex: 1, + }); + }); + + it("U15: enabled, two in-scope turns, first valid, second drops thinking → reports second", () => { + const req = { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + thinking: ENABLED, + messages: [ + { role: "user", content: "go" }, + { role: "assistant", content: [validThinkingBlock, { ...toolUseBlock, id: "tu_1" }] }, + { role: "user", content: [{ type: "tool_result", tool_use_id: "tu_1", content: "ok" }] }, + { role: "assistant", content: [{ type: "tool_use", id: "tu_2", name: "f", input: {} }] }, + { role: "user", content: [{ type: "tool_result", tool_use_id: "tu_2", content: "ok" }] }, + ], + }; + expect(validateThinkingInvariants(req as never)).toEqual({ + kind: "missing_thinking_first", + messageIndex: 3, + observedFirstBlockType: "tool_use", + }); + }); + + it("U16: enabled, two in-scope turns, first drops thinking, second valid → reports first (bug #2)", () => { + const req = { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + thinking: ENABLED, + messages: [ + { role: "user", content: "go" }, + { role: "assistant", content: [{ type: "tool_use", id: "tu_1", name: "f", input: {} }] }, + { role: "user", content: [{ type: "tool_result", tool_use_id: "tu_1", content: "ok" }] }, + { + role: "assistant", + content: [validThinkingBlock, { type: "tool_use", id: "tu_2", name: "f", input: {} }], + }, + { role: "user", content: [{ type: "tool_result", tool_use_id: "tu_2", content: "ok" }] }, + ], + }; + expect(validateThinkingInvariants(req as never)).toEqual({ + kind: "missing_thinking_first", + messageIndex: 1, + observedFirstBlockType: "tool_use", + }); + }); + + it("U17: enabled, assistant turn with string content → null (exempt)", () => { + const req = continuationRequest({ + thinking: ENABLED, + assistantContent: "I will look that up.", + }); + expect(validateThinkingInvariants(req as never)).toBeNull(); + }); + + it("U18: enabled, assistant turn with empty array content → null (exempt)", () => { + const req = continuationRequest({ thinking: ENABLED, assistantContent: [] }); + expect(validateThinkingInvariants(req as never)).toBeNull(); + }); + + it("U19: enabled, tool_result on user turn, assistant turns valid → null", () => { + const req = { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + thinking: ENABLED, + messages: [ + { role: "user", content: "go" }, + { role: "assistant", content: [validThinkingBlock, { ...toolUseBlock, id: "tu_1" }] }, + { role: "user", content: [{ type: "tool_result", tool_use_id: "tu_1", content: "ok" }] }, + ], + }; + expect(validateThinkingInvariants(req as never)).toBeNull(); + }); + + // Malformed-input crash guards: `req.messages` comes from untrusted + // JSON.parse, so message entries and content blocks may be null / non-object. + // The validator must not throw on them. + + it("U20: enabled, messages contains a null entry → null (no throw)", () => { + const req = { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + thinking: ENABLED, + messages: [null], + }; + expect(() => validateThinkingInvariants(req as never)).not.toThrow(); + expect(validateThinkingInvariants(req as never)).toBeNull(); + }); + + it("U21: enabled, messages contains a non-object (string) entry → null (no throw)", () => { + const req = { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + thinking: ENABLED, + messages: ["foo"], + }; + expect(() => validateThinkingInvariants(req as never)).not.toThrow(); + expect(validateThinkingInvariants(req as never)).toBeNull(); + }); + + it("U22: enabled, in-scope assistant turn whose content leads with null → missing_thinking_first (no throw)", () => { + const req = continuationRequest({ + thinking: ENABLED, + assistantContent: [null, toolUseBlock], + }); + expect(() => validateThinkingInvariants(req as never)).not.toThrow(); + expect(validateThinkingInvariants(req as never)).toEqual({ + kind: "missing_thinking_first", + messageIndex: 1, + observedFirstBlockType: undefined, + }); + }); + + it("U23: enabled, in-scope assistant turn whose content leads with a string block → missing_thinking_first (no throw)", () => { + const req = continuationRequest({ + thinking: ENABLED, + assistantContent: ["x", toolUseBlock], + }); + expect(() => validateThinkingInvariants(req as never)).not.toThrow(); + expect(validateThinkingInvariants(req as never)).toEqual({ + kind: "missing_thinking_first", + messageIndex: 1, + observedFirstBlockType: undefined, + }); + }); +}); + +// --------------------------------------------------------------------------- +// 8.2 Integration tests — strict ON +// --------------------------------------------------------------------------- + +describe("thinking invariants — strict ON integration", () => { + let server: ServerInstance; + // A fixture that answers any continuation (matches the tool_result turn). + const continuationFixture: Fixture = { + match: { toolCallId: "tu_1" }, + response: { content: "It is sunny." }, + }; + const textFixture: Fixture = { + match: { userMessage: "Final answer please" }, + response: { content: "Done." }, + }; + + afterEach(async () => { + if (server) await new Promise((r) => server.server.close(() => r())); + }); + + it("I1: valid thinking continuation + matching fixture → 200", async () => { + server = await createServer([continuationFixture], { port: 0, strict: true }); + const res = await post( + `${server.url}/v1/messages`, + continuationRequest({ + thinking: ENABLED, + assistantContent: [validThinkingBlock, toolUseBlock], + }), + ); + expect(res.status).toBe(200); + }); + + it("I2: text-only assistant turn (exempt) + matching fixture → 200, no false 400", async () => { + server = await createServer([textFixture], { port: 0, strict: true }); + const res = await post(`${server.url}/v1/messages`, { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + thinking: ENABLED, + messages: [ + { role: "user", content: "go" }, + { role: "assistant", content: [{ type: "text", text: "answer" }] }, + { role: "user", content: "Final answer please" }, + ], + }); + expect(res.status).toBe(200); + }); + + it("I3: in-scope continuation turn missing thinking block → 400 invalid_request_error", async () => { + server = await createServer([continuationFixture], { port: 0, strict: true }); + const res = await post( + `${server.url}/v1/messages`, + continuationRequest({ thinking: ENABLED, assistantContent: [toolUseBlock] }), + ); + expect(res.status).toBe(400); + const body = JSON.parse(res.body); + expect(body.type).toBe("error"); + expect(body.error.type).toBe("invalid_request_error"); + expect(body.error.message).toMatch(/content\.0.*must begin with a `thinking` block/); + const entries = server.journal.getAll(); + expect(entries[entries.length - 1].response.status).toBe(400); + expect(entries[entries.length - 1].response.fixture).toBeNull(); + }); + + it("I4: leading thinking with empty signature → 400 mentioning signature", async () => { + server = await createServer([continuationFixture], { port: 0, strict: true }); + const res = await post( + `${server.url}/v1/messages`, + continuationRequest({ + thinking: ENABLED, + assistantContent: [{ type: "thinking", thinking: "x", signature: "" }, toolUseBlock], + }), + ); + expect(res.status).toBe(400); + expect(JSON.parse(res.body).error.message).toMatch(/signature/); + }); + + it("I5: leading redacted_thinking with dropped data → 400 mentioning redacted_thinking/data", async () => { + server = await createServer([continuationFixture], { port: 0, strict: true }); + const res = await post( + `${server.url}/v1/messages`, + continuationRequest({ + thinking: ENABLED, + assistantContent: [{ type: "redacted_thinking" }, toolUseBlock], + }), + ); + expect(res.status).toBe(400); + expect(JSON.parse(res.body).error.message).toMatch(/redacted_thinking.*data/); + }); + + it("I6: strict via X-AIMock-Strict header (server default off) → 400 + strictOverride", async () => { + server = await createServer([continuationFixture], { port: 0 }); + const res = await post( + `${server.url}/v1/messages`, + continuationRequest({ thinking: ENABLED, assistantContent: [toolUseBlock] }), + { "X-AIMock-Strict": "true" }, + ); + expect(res.status).toBe(400); + const entries = server.journal.getAll(); + expect(entries[entries.length - 1].response.strictOverride).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// 8.3 Integration tests — strict OFF (backward compat) + warn-spy +// --------------------------------------------------------------------------- + +describe("thinking invariants — strict OFF backward compat", () => { + const continuationFixture: Fixture = { + match: { toolCallId: "tu_1" }, + response: { content: "It is sunny." }, + }; + + it("B1: strict OFF + violation + matching fixture → 200 replay + warn fires", async () => { + const journal = new Journal(); + const logger = new Logger("warn"); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + const defaults = createDefaults({ chunkSize: 10, logger, strict: false }); + const mockReq = { + method: "POST", + url: "/v1/messages", + headers: {}, + } as unknown as http.IncomingMessage; + const mockRes = createMockRes(); + await handleMessages( + mockReq, + mockRes, + JSON.stringify( + continuationRequest({ thinking: ENABLED, assistantContent: [toolUseBlock] }), + ), + [continuationFixture], + journal, + defaults, + () => {}, + ); + expect(journal.getLast()!.response.status).toBe(200); + expect(warnSpy).toHaveBeenCalled(); + const warned = warnSpy.mock.calls.map((c) => c.join(" ")).join("\n"); + expect(warned).toMatch(/THINKING/); + expect(warned).toMatch(/strict off/); + } finally { + warnSpy.mockRestore(); + } + }); + + it("B2: strict OFF + thinking disabled → no warn (validator short-circuits)", async () => { + const journal = new Journal(); + const logger = new Logger("warn"); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + const defaults = createDefaults({ chunkSize: 10, logger, strict: false }); + const mockReq = { + method: "POST", + url: "/v1/messages", + headers: {}, + } as unknown as http.IncomingMessage; + const mockRes = createMockRes(); + await handleMessages( + mockReq, + mockRes, + JSON.stringify(continuationRequest({ assistantContent: [toolUseBlock] })), + [continuationFixture], + journal, + defaults, + () => {}, + ); + expect(journal.getLast()!.response.status).toBe(200); + const warned = warnSpy.mock.calls.map((c) => c.join(" ")).join("\n"); + expect(warned).not.toMatch(/THINKING/); + } finally { + warnSpy.mockRestore(); + } + }); + + it("B3: header X-AIMock-Strict false overriding server strict:true + violation → 200 + warn", async () => { + const journal = new Journal(); + const logger = new Logger("warn"); + const warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + try { + const defaults = createDefaults({ chunkSize: 10, logger, strict: true }); + const mockReq = { + method: "POST", + url: "/v1/messages", + headers: { "x-aimock-strict": "false" }, + } as unknown as http.IncomingMessage; + const mockRes = createMockRes(); + await handleMessages( + mockReq, + mockRes, + JSON.stringify( + continuationRequest({ thinking: ENABLED, assistantContent: [toolUseBlock] }), + ), + [continuationFixture], + journal, + defaults, + () => {}, + ); + expect(journal.getLast()!.response.status).toBe(200); + const warned = warnSpy.mock.calls.map((c) => c.join(" ")).join("\n"); + expect(warned).toMatch(/THINKING/); + } finally { + warnSpy.mockRestore(); + } + }); +}); + +// --------------------------------------------------------------------------- +// 8.4 Emit-side round-trip regression (OQ3) +// --------------------------------------------------------------------------- + +describe("emit-side placeholder signature round-trip", () => { + let server: ServerInstance; + // Fixture that emits a thinking block (reasoning present) on the first turn. + const reasoningFixture: Fixture = { + match: { userMessage: "What is the weather?" }, + response: { content: "Let me check.", reasoning: "I should call the weather tool." }, + }; + const continuationFixture: Fixture = { + match: { toolCallId: "tu_1" }, + response: { content: "It is sunny." }, + }; + + afterEach(async () => { + if (server) await new Promise((r) => server.server.close(() => r())); + }); + + it("RT1: emitted placeholder signature replays into a strict continuation → 200", async () => { + server = await createServer([reasoningFixture, continuationFixture], { port: 0, strict: true }); + + // First turn: streaming, captures the emitted thinking block's signature. + const first = await post(`${server.url}/v1/messages`, { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + thinking: ENABLED, + stream: true, + messages: [{ role: "user", content: "What is the weather?" }], + }); + expect(first.status).toBe(200); + const events = parseClaudeSSEEvents(first.body); + // The thinking `content_block_start` carries an empty signature (mirrors + // real Anthropic); the signature arrives only via `signature_delta`. + const thinkingStart = events.find( + (e) => + e.type === "content_block_start" && + (e.content_block as { type?: string } | undefined)?.type === "thinking", + ); + expect((thinkingStart?.content_block as { signature?: string } | undefined)?.signature).toBe( + "", + ); + // Assembled signature is the signature_delta value, not the start placeholder. + const sigDelta = events.find( + (e) => + e.type === "content_block_delta" && + (e.delta as { type?: string } | undefined)?.type === "signature_delta", + ); + const signature = (sigDelta?.delta as { signature?: string } | undefined)?.signature; + expect(signature).toBe("aimock-placeholder-signature"); + + // Replay the captured thinking block back into a strict continuation. + const cont = await post(`${server.url}/v1/messages`, { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + thinking: ENABLED, + messages: [ + { role: "user", content: "What is the weather?" }, + { + role: "assistant", + content: [ + { type: "thinking", thinking: "I should call the weather tool.", signature }, + toolUseBlock, + ], + }, + { role: "user", content: [{ type: "tool_result", tool_use_id: "tu_1", content: "Sunny" }] }, + ], + }); + expect(cont.status).toBe(200); + }); + + it("RT2: non-streaming assembled thinking block carries the non-empty placeholder", async () => { + server = await createServer([reasoningFixture], { port: 0 }); + const res = await post(`${server.url}/v1/messages`, { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + thinking: ENABLED, + messages: [{ role: "user", content: "What is the weather?" }], + }); + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as { content: Array<{ type: string; signature?: string }> }; + const thinkingBlock = body.content.find((b) => b.type === "thinking"); + expect(thinkingBlock?.signature).toBe("aimock-placeholder-signature"); + }); +}); + +// --------------------------------------------------------------------------- +// 8.5 Tool-call-only thinking emission (parity with content+tool path) +// +// A pure-tool-call fixture WITH `reasoning` must emit a leading thinking block +// exactly as the content+tool path does — streaming (content_block_start +// thinking signature "", signature_delta = placeholder) and non-streaming +// (content[0] is the thinking block). Without it, replaying aimock's own +// tool-only-with-thinking turn into a strict continuation self-trips +// `missing_thinking_first`. +// --------------------------------------------------------------------------- + +describe("tool-call-only thinking emission", () => { + let server: ServerInstance; + // Pure tool-call fixture WITH reasoning — no `content` field, so this routes + // through the isToolCallResponse branch (not content+tool). + const toolOnlyReasoningFixture: Fixture = { + match: { userMessage: "What is the weather?" }, + response: { + toolCalls: [{ id: "tu_1", name: "get_weather", arguments: '{"city":"NYC"}' }], + reasoning: "I should call the weather tool.", + }, + }; + const continuationFixture: Fixture = { + match: { toolCallId: "tu_1" }, + response: { content: "It is sunny." }, + }; + + afterEach(async () => { + if (server) await new Promise((r) => server.server.close(() => r())); + }); + + it("T1: streaming tool-only + reasoning → leading thinking block, tool_use shifted to index 1", async () => { + server = await createServer([toolOnlyReasoningFixture], { port: 0 }); + const res = await post(`${server.url}/v1/messages`, { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + thinking: ENABLED, + stream: true, + messages: [{ role: "user", content: "What is the weather?" }], + }); + expect(res.status).toBe(200); + const events = parseClaudeSSEEvents(res.body); + + // Leading thinking block: content_block_start at index 0 with empty signature. + const thinkingStart = events.find( + (e) => + e.type === "content_block_start" && + (e.content_block as { type?: string } | undefined)?.type === "thinking", + ); + expect(thinkingStart).toBeTruthy(); + expect(thinkingStart!.index).toBe(0); + expect((thinkingStart!.content_block as { signature?: string }).signature).toBe(""); + + // signature_delta carries the placeholder. + const sigDelta = events.find( + (e) => + e.type === "content_block_delta" && + (e.delta as { type?: string } | undefined)?.type === "signature_delta", + ); + expect((sigDelta?.delta as { signature?: string } | undefined)?.signature).toBe( + "aimock-placeholder-signature", + ); + + // tool_use block intact, shifted to index 1 by the prepended thinking block. + const toolStart = events.find( + (e) => + e.type === "content_block_start" && + (e.content_block as { type?: string } | undefined)?.type === "tool_use", + ); + expect(toolStart).toBeTruthy(); + expect(toolStart!.index).toBe(1); + expect((toolStart!.content_block as { name?: string }).name).toBe("get_weather"); + }); + + it("T2: non-streaming tool-only + reasoning → content[0] thinking block, tool_use at content[1]", async () => { + server = await createServer([toolOnlyReasoningFixture], { port: 0 }); + const res = await post(`${server.url}/v1/messages`, { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + thinking: ENABLED, + messages: [{ role: "user", content: "What is the weather?" }], + }); + expect(res.status).toBe(200); + const body = JSON.parse(res.body) as { + content: Array<{ type: string; signature?: string; name?: string }>; + }; + expect(body.content[0].type).toBe("thinking"); + expect(body.content[0].signature).toBe("aimock-placeholder-signature"); + expect(body.content[1].type).toBe("tool_use"); + expect(body.content[1].name).toBe("get_weather"); + }); + + it("T3: aimock's emitted tool-only-with-thinking turn replays into a strict continuation → 200", async () => { + server = await createServer([toolOnlyReasoningFixture, continuationFixture], { + port: 0, + strict: true, + }); + + // First turn (streaming): capture the emitted signature. + const first = await post(`${server.url}/v1/messages`, { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + thinking: ENABLED, + stream: true, + messages: [{ role: "user", content: "What is the weather?" }], + }); + expect(first.status).toBe(200); + const events = parseClaudeSSEEvents(first.body); + const sigDelta = events.find( + (e) => + e.type === "content_block_delta" && + (e.delta as { type?: string } | undefined)?.type === "signature_delta", + ); + const signature = (sigDelta?.delta as { signature?: string } | undefined)?.signature; + expect(signature).toBe("aimock-placeholder-signature"); + + // Replay the emitted tool-only-with-thinking turn into a strict continuation. + const cont = await post(`${server.url}/v1/messages`, { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + thinking: ENABLED, + messages: [ + { role: "user", content: "What is the weather?" }, + { + role: "assistant", + content: [ + { type: "thinking", thinking: "I should call the weather tool.", signature }, + toolUseBlock, + ], + }, + { role: "user", content: [{ type: "tool_result", tool_use_id: "tu_1", content: "Sunny" }] }, + ], + }); + expect(cont.status).toBe(200); + }); +}); + +// --------------------------------------------------------------------------- +// claudeToCompletionRequest — malformed-input crash hardening +// +// `req.messages` and every content-block array come from untrusted JSON.parse, +// so message entries and content blocks may be null / non-object / primitive. +// The conversion path runs on every request right after validation; it must +// skip such entries rather than throw an unhandled TypeError (which the real +// Anthropic API answers with a 400, not a 500/hang). +// --------------------------------------------------------------------------- + +describe("claudeToCompletionRequest — malformed-input crash hardening", () => { + it("C1: user turn with a null content block → does not throw", () => { + const req = { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + messages: [{ role: "user", content: [null] }], + }; + expect(() => claudeToCompletionRequest(req as never)).not.toThrow(); + }); + + it("C2: assistant turn with a string + null content block → does not throw", () => { + const req = { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + messages: [{ role: "assistant", content: ["x", null] }], + }; + expect(() => claudeToCompletionRequest(req as never)).not.toThrow(); + }); + + it("C3: user turn with a primitive (number) content block → does not throw", () => { + const req = { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + messages: [{ role: "user", content: [42] }], + }; + expect(() => claudeToCompletionRequest(req as never)).not.toThrow(); + }); + + it("C4: tool_result with a null inner content block → does not throw", () => { + const req = { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + messages: [ + { role: "user", content: [{ type: "tool_result", tool_use_id: "tu_1", content: [null] }] }, + ], + }; + expect(() => claudeToCompletionRequest(req as never)).not.toThrow(); + }); + + it("C5: messages array contains a null / non-object entry → does not throw", () => { + const req = { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + messages: [null, "foo"], + }; + expect(() => claudeToCompletionRequest(req as never)).not.toThrow(); + }); + + it("C6: system field array with a null block → does not throw", () => { + const req = { + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + system: [null, { type: "text", text: "sys" }], + messages: [{ role: "user", content: "hi" }], + }; + expect(() => claudeToCompletionRequest(req as never)).not.toThrow(); + }); +}); + +// --------------------------------------------------------------------------- +// handleMessages — end-to-end no-crash on malformed content blocks +// +// Drives the full request path (the same place claudeToCompletionRequest runs) +// with malformed content arrays and asserts a clean Anthropic-shaped response, +// never an unhandled exception (500/hang). +// --------------------------------------------------------------------------- + +describe("handleMessages — malformed content blocks do not crash", () => { + const anyFixture: Fixture = { + match: {}, + response: { content: "ok" }, + }; + + async function run(body: unknown): Promise<{ status: number }> { + const journal = new Journal(); + const logger = new Logger("silent"); + const defaults = createDefaults({ chunkSize: 10, logger, strict: false }); + const mockReq = { + method: "POST", + url: "/v1/messages", + headers: {}, + } as unknown as http.IncomingMessage; + const mockRes = createMockRes(); + await handleMessages( + mockReq, + mockRes, + JSON.stringify(body), + [anyFixture], + journal, + defaults, + () => {}, + ); + return { status: journal.getLast()!.response.status }; + } + + it("M1: user content [null] → no throw, clean status", async () => { + await expect( + run({ + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + messages: [{ role: "user", content: [null] }], + }), + ).resolves.toEqual({ status: 200 }); + }); + + it("M2: assistant content ['x', null] → no throw, clean status", async () => { + await expect( + run({ + model: "claude-3-7-sonnet-20250219", + max_tokens: 1024, + messages: [{ role: "assistant", content: ["x", null] }], + }), + ).resolves.toEqual({ status: 200 }); + }); +}); diff --git a/src/messages.ts b/src/messages.ts index f47b4bfd..0fa30b2b 100644 --- a/src/messages.ts +++ b/src/messages.ts @@ -42,10 +42,29 @@ import type { Logger } from "./logger.js"; import { applyChaos } from "./chaos.js"; import { proxyAndRecord } from "./recorder.js"; +/** + * Non-empty placeholder signature written into emitted `thinking` blocks. + * + * The real Anthropic signature is a cryptographic value aimock cannot + * reproduce, but extended-thinking invariant (b) requires a non-empty + * `signature` on the leading thinking block of a tool-loop continuation turn. + * Emitting "" would make a record→replay round-trip of an aimock thinking turn + * self-trip that invariant under strict mode. A non-empty placeholder keeps + * round-trips green; the invariant only checks for non-emptiness, not value. + */ +const PLACEHOLDER_SIGNATURE = "aimock-placeholder-signature"; + // ─── Claude Messages API request types ────────────────────────────────────── interface ClaudeContentBlock { - type: "text" | "tool_use" | "tool_result" | "image" | "document"; + type: + | "text" + | "tool_use" + | "tool_result" + | "image" + | "document" + | "thinking" + | "redacted_thinking"; text?: string; id?: string; name?: string; @@ -53,6 +72,12 @@ interface ClaudeContentBlock { tool_use_id?: string; content?: string | ClaudeContentBlock[]; is_error?: boolean; + // Extended-thinking fields (Anthropic): `thinking` blocks carry the reasoning + // text plus a cryptographic `signature`; `redacted_thinking` blocks carry an + // opaque `data` payload instead. + thinking?: string; + signature?: string; + data?: string; } interface ClaudeMessage { @@ -75,15 +100,33 @@ interface ClaudeRequest { stream?: boolean; max_tokens: number; temperature?: number; + // Extended-thinking config. Explicitly modeled so it is no longer swallowed + // by the index signature below; read defensively (may be a non-object). + thinking?: { type?: "enabled" | "disabled"; budget_tokens?: number }; [key: string]: unknown; } +// ─── Extended-thinking request invariants (Anthropic) ─────────────────────── + +/** + * A detected violation of the Anthropic extended-thinking request invariants on + * a tool-loop continuation turn. `kind` distinguishes the three failure modes + * so callers can template a per-kind error message and tests can assert without + * string matching. + */ +interface ThinkingViolation { + kind: "missing_thinking_first" | "missing_signature" | "dropped_redacted_thinking"; + messageIndex: number; + observedFirstBlockType?: string; +} + // ─── Input conversion: Claude → ChatCompletions messages ──────────────────── function extractClaudeTextContent(content: string | ClaudeContentBlock[]): string { if (typeof content === "string") return content; + if (!Array.isArray(content)) return ""; return content - .filter((b) => b.type === "text") + .filter((b) => b != null && typeof b === "object" && b.type === "text") .map((b) => b.text ?? "") .join(""); } @@ -96,21 +139,28 @@ export function claudeToCompletionRequest(req: ClaudeRequest): ChatCompletionReq const systemText = typeof req.system === "string" ? req.system - : req.system - .filter((b) => b.type === "text") - .map((b) => b.text ?? "") - .join(""); + : Array.isArray(req.system) + ? req.system + .filter((b) => b != null && typeof b === "object" && b.type === "text") + .map((b) => b.text ?? "") + .join("") + : ""; if (systemText) { messages.push({ role: "system", content: systemText }); } } - for (const msg of req.messages) { + const reqMessages = Array.isArray(req.messages) ? req.messages : []; + for (const msg of reqMessages) { + // `req.messages` is untrusted JSON; entries may be null / non-object. + if (!msg || typeof msg !== "object") continue; if (msg.role === "user") { // Check for tool_result blocks if (typeof msg.content !== "string" && Array.isArray(msg.content)) { - const toolResults = msg.content.filter((b) => b.type === "tool_result"); - const textBlocks = msg.content.filter((b) => b.type === "text"); + // Content blocks are equally untrusted; skip null / non-object entries. + const blocks = msg.content.filter((b) => b != null && typeof b === "object"); + const toolResults = blocks.filter((b) => b.type === "tool_result"); + const textBlocks = blocks.filter((b) => b.type === "text"); if (toolResults.length > 0) { // Each tool_result → tool message @@ -120,7 +170,7 @@ export function claudeToCompletionRequest(req: ClaudeRequest): ChatCompletionReq ? tr.content : Array.isArray(tr.content) ? tr.content - .filter((b) => b.type === "text") + .filter((b) => b != null && typeof b === "object" && b.type === "text") .map((b) => b.text ?? "") .join("") : ""; @@ -149,7 +199,12 @@ export function claudeToCompletionRequest(req: ClaudeRequest): ChatCompletionReq if (typeof msg.content === "string") { messages.push({ role: "assistant", content: msg.content }); } else if (Array.isArray(msg.content)) { - const toolUseBlocks = msg.content.filter((b) => b.type === "tool_use"); + const toolUseBlocks = msg.content.filter( + (b) => b != null && typeof b === "object" && b.type === "tool_use", + ); + // Only `text` blocks feed fixture matching; `thinking` / + // `redacted_thinking` blocks are intentionally excluded from the + // matchable content. const textContent = extractClaudeTextContent(msg.content); if (toolUseBlocks.length > 0) { @@ -199,6 +254,125 @@ export function claudeToCompletionRequest(req: ClaudeRequest): ChatCompletionReq }; } +// ─── Extended-thinking invariant validation ───────────────────────────────── + +/** True iff `req.thinking` is an object with `type === "enabled"`. */ +function isThinkingEnabled(req: ClaudeRequest): boolean { + const t = req.thinking; + return typeof t === "object" && t !== null && (t as { type?: unknown }).type === "enabled"; +} + +/** + * True iff the user turn at `req.messages[userIndex]` carries a `tool_result` + * referencing one of `toolUseIds` — i.e. it answers the preceding assistant + * turn's `tool_use` block(s), making that assistant turn a tool-loop + * continuation point. + */ +function userTurnAnswersToolUse(msg: ClaudeMessage | undefined, toolUseIds: Set): boolean { + if (!msg || msg.role !== "user" || !Array.isArray(msg.content)) return false; + return msg.content.some( + (b) => + b != null && + typeof b === "object" && + b.type === "tool_result" && + typeof b.tool_use_id === "string" && + toolUseIds.has(b.tool_use_id), + ); +} + +/** + * Validate the Anthropic extended-thinking request invariants on tool-loop + * continuation turns. Pure: returns the first detected `ThinkingViolation` or + * `null` when thinking is disabled or every in-scope turn is well-formed. + * + * Scope: an assistant turn is in-scope only when it (a) is array content + * carrying at least one `tool_use` block and (b) is followed by a matching + * `tool_result` on the next user turn. Text-only / `end_turn` turns, string or + * empty turns, and trailing unanswered `tool_use` turns are all exempt. + * + * Scope detection assumes well-formed Anthropic transcripts: `tool_use` ids are + * unique, and a tool_use turn is answered by the immediately-following user + * turn's `tool_result` (adjacency). Malformed shapes — non-adjacent answers, + * idless `tool_use` blocks, or a `tool_result` separated from its `tool_use` by + * intervening turns — are intentionally treated as out-of-scope (and therefore + * not 400'd) rather than validated, since the real Anthropic API would never + * have produced them. + */ +export function validateThinkingInvariants(req: ClaudeRequest): ThinkingViolation | null { + if (!isThinkingEnabled(req)) return null; + + const messages = Array.isArray(req.messages) ? req.messages : []; + for (let i = 0; i < messages.length; i++) { + const msg = messages[i]; + // `req.messages` is untrusted JSON; entries may be null / non-object. + if (!msg || typeof msg !== "object") continue; + if (msg.role !== "assistant" || !Array.isArray(msg.content) || msg.content.length === 0) { + continue; + } + + const toolUseIds = new Set(); + for (const b of msg.content) { + // Content blocks are equally untrusted; skip null / non-object entries. + if (!b || typeof b !== "object") continue; + if (b.type === "tool_use" && typeof b.id === "string") toolUseIds.add(b.id); + } + // Not a tool_use-bearing turn → not a reasoning-bearing continuation turn. + if (toolUseIds.size === 0) continue; + // Trailing/unanswered tool_use (no matching tool_result follows) → out of scope. + if (!userTurnAnswersToolUse(messages[i + 1], toolUseIds)) continue; + + const first = msg.content[0]; + + // A null / non-object leading block cannot be a thinking block → it + // violates invariant (a) just as a wrong-typed block would. + if (!first || typeof first !== "object") { + return { + kind: "missing_thinking_first", + messageIndex: i, + observedFirstBlockType: undefined, + }; + } + + // (a) The in-scope turn must lead with a thinking / redacted_thinking block. + if (first.type !== "thinking" && first.type !== "redacted_thinking") { + return { + kind: "missing_thinking_first", + messageIndex: i, + observedFirstBlockType: first.type, + }; + } + + // (b) A leading `thinking` block must carry a non-empty string `signature`. + if (first.type === "thinking") { + if (typeof first.signature !== "string" || first.signature.length === 0) { + return { kind: "missing_signature", messageIndex: i }; + } + } + + // (c) A leading `redacted_thinking` block must preserve a non-empty `data`. + if (first.type === "redacted_thinking") { + if (typeof first.data !== "string" || first.data.length === 0) { + return { kind: "dropped_redacted_thinking", messageIndex: i }; + } + } + } + + return null; +} + +/** Render the Anthropic-shaped 400 error message for a thinking violation. */ +function thinkingViolationMessage(v: ThinkingViolation): string { + const prefix = `messages.${v.messageIndex}.content.0`; + switch (v.kind) { + case "missing_thinking_first": + return `${prefix}: when \`thinking\` is enabled, a tool-loop continuation assistant turn must begin with a \`thinking\` block; got \`${v.observedFirstBlockType ?? "unknown"}\`.`; + case "missing_signature": + return `${prefix}: the leading \`thinking\` block is missing a non-empty \`signature\`.`; + case "dropped_redacted_thinking": + return `${prefix}: the leading \`redacted_thinking\` block must preserve its \`data\`.`; + } +} + // ─── Response building: fixture → Claude Messages API format ──────────────── function claudeStopReason(finishReason: string | undefined, defaultReason: string): string { @@ -255,6 +429,9 @@ function buildClaudeTextStreamEvents( // Thinking block (emitted before text when reasoning is present) if (reasoning) { + // Real Anthropic emits an empty `signature` on the thinking + // `content_block_start`; the cryptographic signature arrives only via the + // trailing `signature_delta`. Mirror that wire shape here. events.push({ type: "content_block_start", index: blockIndex, @@ -273,7 +450,7 @@ function buildClaudeTextStreamEvents( events.push({ type: "content_block_delta", index: blockIndex, - delta: { type: "signature_delta", signature: "" }, + delta: { type: "signature_delta", signature: PLACEHOLDER_SIGNATURE }, }); events.push({ @@ -328,6 +505,7 @@ function buildClaudeToolCallStreamEvents( model: string, chunkSize: number, logger: Logger, + reasoning?: string, overrides?: ResponseOverrides, ): ClaudeSSEEvent[] { const msgId = overrides?.id ?? generateMessageId(); @@ -349,8 +527,47 @@ function buildClaudeToolCallStreamEvents( }, }); - for (let idx = 0; idx < toolCalls.length; idx++) { - const tc = toolCalls[idx]; + let blockIndex = 0; + + // Optional thinking block (emitted before the tool_use blocks when reasoning + // is present). Mirrors buildClaudeContentWithToolCallsStreamEvents exactly so + // a pure-tool-call turn under extended thinking emits a leading thinking + // block — without it, replaying the emitted turn under strict self-trips + // `missing_thinking_first`. + if (reasoning) { + // Real Anthropic emits an empty `signature` on the thinking + // `content_block_start`; the cryptographic signature arrives only via the + // trailing `signature_delta`. Mirror that wire shape here. + events.push({ + type: "content_block_start", + index: blockIndex, + content_block: { type: "thinking", thinking: "", signature: "" }, + }); + + for (let i = 0; i < reasoning.length; i += chunkSize) { + const slice = reasoning.slice(i, i + chunkSize); + events.push({ + type: "content_block_delta", + index: blockIndex, + delta: { type: "thinking_delta", thinking: slice }, + }); + } + + events.push({ + type: "content_block_delta", + index: blockIndex, + delta: { type: "signature_delta", signature: PLACEHOLDER_SIGNATURE }, + }); + + events.push({ + type: "content_block_stop", + index: blockIndex, + }); + + blockIndex++; + } + + for (const tc of toolCalls) { const toolUseId = tc.id || generateToolUseId(); // Parse arguments to JSON object (Claude uses objects, not strings) @@ -368,7 +585,7 @@ function buildClaudeToolCallStreamEvents( // content_block_start events.push({ type: "content_block_start", - index: idx, + index: blockIndex, content_block: { type: "tool_use", id: toolUseId, @@ -382,7 +599,7 @@ function buildClaudeToolCallStreamEvents( const slice = argsJson.slice(i, i + chunkSize); events.push({ type: "content_block_delta", - index: idx, + index: blockIndex, delta: { type: "input_json_delta", partial_json: slice }, }); } @@ -390,8 +607,10 @@ function buildClaudeToolCallStreamEvents( // content_block_stop events.push({ type: "content_block_stop", - index: idx, + index: blockIndex, }); + + blockIndex++; } // message_delta @@ -421,7 +640,7 @@ function buildClaudeTextResponse( const contentBlocks: object[] = []; if (reasoning) { - contentBlocks.push({ type: "thinking", thinking: reasoning, signature: "" }); + contentBlocks.push({ type: "thinking", thinking: reasoning, signature: PLACEHOLDER_SIGNATURE }); } contentBlocks.push({ type: "text", text: content }); @@ -442,29 +661,41 @@ function buildClaudeToolCallResponse( toolCalls: ToolCall[], model: string, logger: Logger, + reasoning?: string, overrides?: ResponseOverrides, ): object { + const contentBlocks: object[] = []; + + // Leading thinking block when reasoning is present — mirrors + // buildClaudeContentWithToolCallsResponse so a pure-tool-call turn under + // extended thinking carries the same leading thinking block. + if (reasoning) { + contentBlocks.push({ type: "thinking", thinking: reasoning, signature: PLACEHOLDER_SIGNATURE }); + } + + for (const tc of toolCalls) { + let argsObj: unknown; + try { + argsObj = JSON.parse(tc.arguments || "{}"); + } catch { + logger.warn( + `Malformed JSON in fixture tool call arguments for "${tc.name}": ${tc.arguments}`, + ); + argsObj = {}; + } + contentBlocks.push({ + type: "tool_use", + id: tc.id || generateToolUseId(), + name: tc.name, + input: argsObj, + }); + } + return { id: overrides?.id ?? generateMessageId(), type: "message", role: overrides?.role ?? "assistant", - content: toolCalls.map((tc) => { - let argsObj: unknown; - try { - argsObj = JSON.parse(tc.arguments || "{}"); - } catch { - logger.warn( - `Malformed JSON in fixture tool call arguments for "${tc.name}": ${tc.arguments}`, - ); - argsObj = {}; - } - return { - type: "tool_use", - id: tc.id || generateToolUseId(), - name: tc.name, - input: argsObj, - }; - }), + content: contentBlocks, model: overrides?.model ?? model, stop_reason: claudeStopReason(overrides?.finishReason, "tool_use"), stop_sequence: null, @@ -504,6 +735,9 @@ function buildClaudeContentWithToolCallsStreamEvents( // Optional thinking block if (reasoning) { + // Real Anthropic emits an empty `signature` on the thinking + // `content_block_start`; the cryptographic signature arrives only via the + // trailing `signature_delta`. Mirror that wire shape here. events.push({ type: "content_block_start", index: blockIndex, @@ -522,7 +756,7 @@ function buildClaudeContentWithToolCallsStreamEvents( events.push({ type: "content_block_delta", index: blockIndex, - delta: { type: "signature_delta", signature: "" }, + delta: { type: "signature_delta", signature: PLACEHOLDER_SIGNATURE }, }); events.push({ @@ -626,7 +860,7 @@ function buildClaudeContentWithToolCallsResponse( const contentBlocks: object[] = []; if (reasoning) { - contentBlocks.push({ type: "thinking", thinking: reasoning, signature: "" }); + contentBlocks.push({ type: "thinking", thinking: reasoning, signature: PLACEHOLDER_SIGNATURE }); } contentBlocks.push({ type: "text", text: content }); @@ -747,6 +981,44 @@ export async function handleMessages( return; } + // Extended-thinking invariant validation. The validator runs whenever + // thinking is enabled (it self-short-circuits to null otherwise). On a + // detected violation: strict ON → 400, strict OFF → warn + replay. Mirrors + // the real Anthropic API, which 400s on these. + const thinkingViolation = validateThinkingInvariants(claudeReq); + if (thinkingViolation) { + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const violationMessage = thinkingViolationMessage(thinkingViolation); + if (effectiveStrict) { + logger.error(`THINKING: ${violationMessage}`); + journal.add({ + method: req.method ?? "POST", + path: req.url ?? "/v1/messages", + headers: flattenHeaders(req.headers), + body: null, + response: { + status: 400, + fixture: null, + ...strictOverrideField(defaults.strict, req.headers), + }, + }); + writeErrorResponse( + res, + 400, + JSON.stringify({ + type: "error", + error: { + type: "invalid_request_error", + message: violationMessage, + }, + }), + ); + return; + } + logger.warn(`THINKING: ${violationMessage} (strict off — replaying anyway)`); + // Fall through to existing match/replay behavior. + } + // Convert to ChatCompletionRequest for fixture matching const completionReq = claudeToCompletionRequest(claudeReq); completionReq._context = getContext(req); @@ -1025,6 +1297,13 @@ export async function handleMessages( ); } const overrides = extractOverrides(response); + const effectiveStrict = resolveStrictMode(defaults.strict, req.headers); + const effReasoning = resolveReasoningForModel( + response.reasoning, + completionReq.model, + effectiveStrict, + defaults.logger, + ); const journalEntry = journal.add({ method: req.method ?? "POST", path: req.url ?? "/v1/messages", @@ -1037,6 +1316,7 @@ export async function handleMessages( response.toolCalls, completionReq.model, logger, + effReasoning, overrides, ); res.writeHead(200, { "Content-Type": "application/json" }); @@ -1047,6 +1327,7 @@ export async function handleMessages( completionReq.model, chunkSize, logger, + effReasoning, overrides, ); const interruption = createInterruptionSignal(fixture); diff --git a/src/types.ts b/src/types.ts index bb60d308..7d7371a1 100644 --- a/src/types.ts +++ b/src/types.ts @@ -160,6 +160,7 @@ export interface ToolCall { export interface ToolCallResponse extends ResponseOverrides { toolCalls: ToolCall[]; + reasoning?: string; webSearches?: string[]; } @@ -357,6 +358,7 @@ export interface FixtureFileToolCall { export interface FixtureFileToolCallResponse extends ResponseOverrides { toolCalls: FixtureFileToolCall[]; + reasoning?: string; webSearches?: string[]; }