Skip to content

Commit 5aa901f

Browse files
marthakellyclaude
marthakelly
and
claude
committed
test(agent): strengthen reasoning lifecycle test coverage
- Add cardinality assertions (toHaveLength(1)) to all 5 auto-close tests so duplicate REASONING_MESSAGE_END / REASONING_END events are caught - Add missing RUN_FINISHED final-event assertion to abort auto-close test - Add test for consecutive reasoning blocks with no reasoning-end between them (exercises closeReasoningIfOpen inside reasoning-start) - Add test for exception thrown mid-stream (exercises catch block path) - Add test for stream exhaustion without terminal event while reasoning is open (exercises !terminalEventEmitted fallback) - Expand changeset description to cover all three behavioral fixes Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
1 parent f78bb6d commit 5aa901f

2 files changed

Lines changed: 206 additions & 1 deletion

File tree

‎.changeset/fix-reasoning-stall.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,4 +2,10 @@
22
"@copilotkit/runtime": patch
33
---
44

5-
fix(agent): skip empty reasoning deltas and auto-close reasoning lifecycle when SDK omits reasoning-end
5+
fix(agent): harden BuiltInAgent reasoning lifecycle
6+
7+
- Skip empty reasoning deltas (violates @ag-ui/core schema)
8+
- Auto-close reasoning lifecycle when SDK omits reasoning-end (on consecutive-start, phase transitions, abort, error, and fallback paths)
9+
- Make reasoning-end idempotent to prevent duplicate close events when auto-close already fired
10+
- Regenerate reasoningMessageId for consecutive reasoning blocks when SDK provides no id
11+
- Close reasoning in outer catch block so exceptions mid-reasoning emit proper lifecycle events

‎packages/runtime/src/agent/__tests__/basic-agent.test.ts‎

Lines changed: 199 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1181,6 +1181,14 @@ describe("BasicAgent", () => {
11811181
expect(reasoningEndIdx).toBeGreaterThan(reasoningMsgEndIdx);
11821182
expect(reasoningEndIdx).toBeLessThan(toolCallStartIdx);
11831183

1184+
// Each close event must appear exactly once (guard against double-emit)
1185+
expect(
1186+
eventTypes.filter((t) => t === EventType.REASONING_MESSAGE_END),
1187+
).toHaveLength(1);
1188+
expect(
1189+
eventTypes.filter((t) => t === EventType.REASONING_END),
1190+
).toHaveLength(1);
1191+
11841192
// Stream still completes with RUN_FINISHED
11851193
expect(eventTypes[eventTypes.length - 1]).toBe(EventType.RUN_FINISHED);
11861194
});
@@ -1223,6 +1231,14 @@ describe("BasicAgent", () => {
12231231
expect(reasoningEndIdx).toBeGreaterThan(reasoningMsgEndIdx);
12241232
expect(reasoningEndIdx).toBeLessThan(textChunkIdx);
12251233

1234+
// Each close event must appear exactly once (guard against double-emit)
1235+
expect(
1236+
eventTypes.filter((t) => t === EventType.REASONING_MESSAGE_END),
1237+
).toHaveLength(1);
1238+
expect(
1239+
eventTypes.filter((t) => t === EventType.REASONING_END),
1240+
).toHaveLength(1);
1241+
12261242
// Stream still completes with RUN_FINISHED
12271243
expect(eventTypes[eventTypes.length - 1]).toBe(EventType.RUN_FINISHED);
12281244
});
@@ -1261,6 +1277,14 @@ describe("BasicAgent", () => {
12611277
expect(reasoningMsgEndIdx).toBeGreaterThan(0);
12621278
expect(reasoningEndIdx).toBeGreaterThan(reasoningMsgEndIdx);
12631279

1280+
// Each close event must appear exactly once (guard against double-emit)
1281+
expect(
1282+
eventTypes.filter((t) => t === EventType.REASONING_MESSAGE_END),
1283+
).toHaveLength(1);
1284+
expect(
1285+
eventTypes.filter((t) => t === EventType.REASONING_END),
1286+
).toHaveLength(1);
1287+
12641288
// Stream still completes with RUN_FINISHED
12651289
expect(eventTypes[eventTypes.length - 1]).toBe(EventType.RUN_FINISHED);
12661290
});
@@ -1300,6 +1324,17 @@ describe("BasicAgent", () => {
13001324
expect(reasoningMsgEndIdx).toBeGreaterThan(0);
13011325
expect(reasoningEndIdx).toBeGreaterThan(reasoningMsgEndIdx);
13021326
expect(runFinishedIdx).toBeGreaterThan(reasoningEndIdx);
1327+
1328+
// Each close event must appear exactly once (guard against double-emit)
1329+
expect(
1330+
eventTypes.filter((t) => t === EventType.REASONING_MESSAGE_END),
1331+
).toHaveLength(1);
1332+
expect(
1333+
eventTypes.filter((t) => t === EventType.REASONING_END),
1334+
).toHaveLength(1);
1335+
1336+
// Stream still completes with RUN_FINISHED
1337+
expect(eventTypes[eventTypes.length - 1]).toBe(EventType.RUN_FINISHED);
13031338
});
13041339

13051340
it("should auto-close reasoning when stream errors mid-reasoning", async () => {
@@ -1346,6 +1381,170 @@ describe("BasicAgent", () => {
13461381
expect(reasoningMsgEndIdx).toBeGreaterThan(0);
13471382
expect(reasoningEndIdx).toBeGreaterThan(reasoningMsgEndIdx);
13481383
expect(runErrorIdx).toBeGreaterThan(reasoningEndIdx);
1384+
1385+
// Each close event must appear exactly once (guard against double-emit)
1386+
expect(
1387+
eventTypes.filter((t) => t === EventType.REASONING_MESSAGE_END),
1388+
).toHaveLength(1);
1389+
expect(
1390+
eventTypes.filter((t) => t === EventType.REASONING_END),
1391+
).toHaveLength(1);
1392+
});
1393+
1394+
it("should auto-close reasoning for consecutive blocks with no reasoning-end between them", async () => {
1395+
const agent = new BasicAgent({
1396+
model: "openai/gpt-4o",
1397+
});
1398+
1399+
vi.mocked(streamText).mockReturnValue(
1400+
mockStreamTextResponse([
1401+
reasoningStart(),
1402+
reasoningDelta("First thought"),
1403+
// NO reasoningEnd() — second block starts immediately
1404+
reasoningStart(),
1405+
reasoningDelta("Second thought"),
1406+
reasoningEnd(),
1407+
finish(),
1408+
]) as any,
1409+
);
1410+
1411+
const input: RunAgentInput = {
1412+
threadId: "thread1",
1413+
runId: "run1",
1414+
messages: [],
1415+
tools: [],
1416+
context: [],
1417+
state: {},
1418+
};
1419+
1420+
const events = await collectEvents(agent["run"](input));
1421+
const eventTypes = events.map((e: any) => e.type);
1422+
1423+
// Both reasoning blocks must be properly closed — two complete lifecycles
1424+
expect(
1425+
eventTypes.filter((t) => t === EventType.REASONING_MESSAGE_END),
1426+
).toHaveLength(2);
1427+
expect(
1428+
eventTypes.filter((t) => t === EventType.REASONING_END),
1429+
).toHaveLength(2);
1430+
1431+
// First block's REASONING_END must appear before second block's REASONING_START
1432+
const firstReasoningEndIdx = eventTypes.indexOf(EventType.REASONING_END);
1433+
const secondReasoningStartIdx = eventTypes.lastIndexOf(
1434+
EventType.REASONING_START,
1435+
);
1436+
expect(firstReasoningEndIdx).toBeLessThan(secondReasoningStartIdx);
1437+
1438+
// The two blocks must use distinct messageIds
1439+
const startEvents = events.filter(
1440+
(e: any) => e.type === EventType.REASONING_START,
1441+
);
1442+
expect(startEvents).toHaveLength(2);
1443+
expect((startEvents[0] as any).messageId).not.toBe(
1444+
(startEvents[1] as any).messageId,
1445+
);
1446+
1447+
expect(eventTypes[eventTypes.length - 1]).toBe(EventType.RUN_FINISHED);
1448+
});
1449+
1450+
it("should close reasoning when an exception is thrown mid-stream", async () => {
1451+
const agent = new BasicAgent({
1452+
model: "openai/gpt-4o",
1453+
});
1454+
1455+
// Simulate the fullStream generator throwing mid-iteration (not a stream error event)
1456+
const throwingStream = {
1457+
fullStream: (async function* () {
1458+
yield reasoningStart();
1459+
yield reasoningDelta("Thinking...");
1460+
throw new Error("unexpected network failure");
1461+
})(),
1462+
};
1463+
vi.mocked(streamText).mockReturnValue(throwingStream as any);
1464+
1465+
const input: RunAgentInput = {
1466+
threadId: "thread1",
1467+
runId: "run1",
1468+
messages: [],
1469+
tools: [],
1470+
context: [],
1471+
state: {},
1472+
};
1473+
1474+
// subscriber.error() causes collectEvents to reject, so collect manually
1475+
const events: any[] = [];
1476+
await new Promise<void>((resolve) => {
1477+
agent["run"](input).subscribe({
1478+
next: (e: any) => events.push(e),
1479+
error: () => resolve(), // error is expected
1480+
complete: () => resolve(),
1481+
});
1482+
});
1483+
1484+
const eventTypes = events.map((e: any) => e.type);
1485+
1486+
// Reasoning must be closed before RUN_ERROR despite the exception path
1487+
const reasoningMsgEndIdx = eventTypes.indexOf(
1488+
EventType.REASONING_MESSAGE_END,
1489+
);
1490+
const reasoningEndIdx = eventTypes.indexOf(EventType.REASONING_END);
1491+
const runErrorIdx = eventTypes.indexOf(EventType.RUN_ERROR);
1492+
expect(reasoningMsgEndIdx).toBeGreaterThan(0);
1493+
expect(reasoningEndIdx).toBeGreaterThan(reasoningMsgEndIdx);
1494+
expect(runErrorIdx).toBeGreaterThan(reasoningEndIdx);
1495+
1496+
expect(
1497+
eventTypes.filter((t) => t === EventType.REASONING_MESSAGE_END),
1498+
).toHaveLength(1);
1499+
expect(
1500+
eventTypes.filter((t) => t === EventType.REASONING_END),
1501+
).toHaveLength(1);
1502+
});
1503+
1504+
it("should close reasoning and emit RUN_FINISHED when stream exhausts without terminal event", async () => {
1505+
const agent = new BasicAgent({
1506+
model: "openai/gpt-4o",
1507+
});
1508+
1509+
// Stream ends with no finish/abort/error — exercises !terminalEventEmitted fallback
1510+
vi.mocked(streamText).mockReturnValue(
1511+
mockStreamTextResponse([
1512+
reasoningStart(),
1513+
reasoningDelta("Thinking..."),
1514+
// deliberate: no finish(), no abort(), no error()
1515+
]) as any,
1516+
);
1517+
1518+
const input: RunAgentInput = {
1519+
threadId: "thread1",
1520+
runId: "run1",
1521+
messages: [],
1522+
tools: [],
1523+
context: [],
1524+
state: {},
1525+
};
1526+
1527+
const events = await collectEvents(agent["run"](input));
1528+
const eventTypes = events.map((e: any) => e.type);
1529+
1530+
// Reasoning must be closed before RUN_FINISHED via fallback
1531+
const reasoningMsgEndIdx = eventTypes.indexOf(
1532+
EventType.REASONING_MESSAGE_END,
1533+
);
1534+
const reasoningEndIdx = eventTypes.indexOf(EventType.REASONING_END);
1535+
const runFinishedIdx = eventTypes.indexOf(EventType.RUN_FINISHED);
1536+
expect(reasoningMsgEndIdx).toBeGreaterThan(0);
1537+
expect(reasoningEndIdx).toBeGreaterThan(reasoningMsgEndIdx);
1538+
expect(runFinishedIdx).toBeGreaterThan(reasoningEndIdx);
1539+
1540+
expect(
1541+
eventTypes.filter((t) => t === EventType.REASONING_MESSAGE_END),
1542+
).toHaveLength(1);
1543+
expect(
1544+
eventTypes.filter((t) => t === EventType.REASONING_END),
1545+
).toHaveLength(1);
1546+
1547+
expect(eventTypes[eventTypes.length - 1]).toBe(EventType.RUN_FINISHED);
13491548
});
13501549

13511550
it("should handle reasoning interleaved with tool calls", async () => {

0 commit comments

Comments
 (0)