tool-rendering-default-catchall.spec.ts

import { test, expect } from "@playwright/test";

// QA reference: qa/tool-rendering-default-catchall.md
// Demo source: src/app/demos/tool-rendering-default-catchall/page.tsx
//
// This cell registers ZERO custom render hooks. The runtime falls back
// to the framework's built-in DefaultToolCallRenderer, which paints
// every tool call with a stable `[data-testid="copilot-tool-render"]`
// wrapper plus a `data-tool-name="<name>"` attribute. We assert on the
// built-in contract — branded testids from sibling cells stay at zero.

const SUGGESTION_TIMEOUT = 15000;
const TOOL_TIMEOUT = 60000;

const PILLS = ["Weather in SF", "Find flights", "Roll a d20", "Chain tools"];

test.describe("Tool Rendering — Default Catch-all", () => {
  test.beforeEach(async ({ page }) => {
    await page.goto("/demos/tool-rendering-default-catchall");
    await expect(page.getByPlaceholder("Type a message")).toBeVisible({
      timeout: SUGGESTION_TIMEOUT,
    });
  });

  test("page loads with composer and 4 suggestion pills", async ({ page }) => {
    const suggestions = page.locator('[data-testid="copilot-suggestion"]');
    for (const title of PILLS) {
      await expect(suggestions.filter({ hasText: title }).first()).toBeVisible({
        timeout: SUGGESTION_TIMEOUT,
      });
    }

    // Sanity: branded sibling-cell testids stay at zero on this cell.
    await expect(page.locator('[data-testid="weather-card"]')).toHaveCount(0);
    await expect(page.locator('[data-testid="flights-card"]')).toHaveCount(0);
    await expect(page.locator('[data-testid="stock-card"]')).toHaveCount(0);
    await expect(page.locator('[data-testid="d20-card"]')).toHaveCount(0);
    await expect(
      page.locator('[data-testid="custom-wildcard-card"]'),
    ).toHaveCount(0);
  });

  test("Weather in SF pill paints the built-in default card for get_weather", async ({
    page,
  }) => {
    await page
      .locator('[data-testid="copilot-suggestion"]')
      .filter({ hasText: "Weather in SF" })
      .first()
      .click();

    const card = page
      .locator(
        '[data-testid="copilot-tool-render"][data-tool-name="get_weather"]',
      )
      .first();
    await expect(card).toBeVisible({ timeout: TOOL_TIMEOUT });

    // Args are pinned to San Francisco (verbatim pill prompt → fixture).
    await expect
      .poll(async () => card.getAttribute("data-args"), {
        timeout: TOOL_TIMEOUT,
      })
      .toContain("San Francisco");

    // No branded sibling-cell card mounted.
    await expect(page.locator('[data-testid="weather-card"]')).toHaveCount(0);
    await expect(
      page.locator('[data-testid="custom-wildcard-card"]'),
    ).toHaveCount(0);
  });

  test("Find flights pill paints the built-in default card for search_flights", async ({
    page,
  }) => {
    await page
      .locator('[data-testid="copilot-suggestion"]')
      .filter({ hasText: "Find flights" })
      .first()
      .click();

    const card = page
      .locator(
        '[data-testid="copilot-tool-render"][data-tool-name="search_flights"]',
      )
      .first();
    await expect(card).toBeVisible({ timeout: TOOL_TIMEOUT });

    // Result attribute carries the deterministic fixture flights (NOT
    // the a2ui beautiful-chat shape).
    await expect
      .poll(async () => card.getAttribute("data-result"), {
        timeout: TOOL_TIMEOUT,
      })
      .toMatch(/United|Delta|JetBlue/);

    await expect(page.locator('[data-testid="flights-card"]')).toHaveCount(0);
  });

  test("Roll a d20 pill paints exactly 5 default cards for roll_d20", async ({
    page,
  }) => {
    await page
      .locator('[data-testid="copilot-suggestion"]')
      .filter({ hasText: "Roll a d20" })
      .first()
      .click();

    const cards = page.locator(
      '[data-testid="copilot-tool-render"][data-tool-name="roll_d20"]',
    );

    await expect
      .poll(async () => cards.count(), { timeout: TOOL_TIMEOUT })
      .toBe(5);

    // 5th card's result must contain "20" (the final scripted roll).
    const lastResult = await cards.nth(4).getAttribute("data-result");
    expect(lastResult ?? "").toMatch(/"value":\s*20|"result":\s*20/);

    // First 4 results are not-20.
    for (let i = 0; i < 4; i++) {
      const r = (await cards.nth(i).getAttribute("data-result")) ?? "";
      expect(r).not.toMatch(/"value":\s*20|"result":\s*20/);
    }

    await expect(page.locator('[data-testid="d20-card"]')).toHaveCount(0);
  });

  test("Chain tools pill paints 3 default cards (weather + flights + d20)", async ({
    page,
  }) => {
    await page
      .locator('[data-testid="copilot-suggestion"]')
      .filter({ hasText: "Chain tools" })
      .first()
      .click();

    await expect(
      page
        .locator(
          '[data-testid="copilot-tool-render"][data-tool-name="get_weather"]',
        )
        .first(),
    ).toBeVisible({ timeout: TOOL_TIMEOUT });
    await expect(
      page
        .locator(
          '[data-testid="copilot-tool-render"][data-tool-name="search_flights"]',
        )
        .first(),
    ).toBeVisible({ timeout: TOOL_TIMEOUT });
    await expect(
      page
        .locator(
          '[data-testid="copilot-tool-render"][data-tool-name="roll_d20"]',
        )
        .first(),
    ).toBeVisible({ timeout: TOOL_TIMEOUT });
  });

  // Regression for the aimock multi-pill bug:
  // The d20 and Chain-tools fixtures used `turnIndex` + `hasToolResult` to
  // disambiguate sequential iterations of the same prompt. Those gates
  // count *global* thread state: clicking Find flights first left two
  // assistant messages and one tool message behind, so the d20 loop
  // entered at `turnIndex=2` (skipping rolls 7 and 14, hence only 3
  // cards), and the Chain-tools tool-emitting fixture was skipped
  // entirely (`hasToolResult: false` failed) so the pill went straight to
  // the "Done — Tokyo is sunny…" content with no tool cards. Fix: chain
  // all follow-ups via `toolCallId`, drop the global gates. This test
  // drives the three offending pills in a single thread and asserts the
  // expected card counts for each.
  test("sequential pills in one thread render full card sequences for each", async ({
    page,
  }) => {
    // Three sequential pills × multi-tool chains × LLM-mock latency easily
    // exceeds Playwright's 30s default. Bumped to cover the worst case.
    test.setTimeout(240_000);

    await page
      .locator('[data-testid="copilot-suggestion"]')
      .filter({ hasText: "Find flights" })
      .first()
      .click();
    const flights = page.locator(
      '[data-testid="copilot-tool-render"][data-tool-name="search_flights"]',
    );
    await expect(flights).toHaveCount(1, { timeout: TOOL_TIMEOUT });

    await page
      .locator('[data-testid="copilot-suggestion"]')
      .filter({ hasText: "Roll a d20" })
      .first()
      .click();
    const d20 = page.locator(
      '[data-testid="copilot-tool-render"][data-tool-name="roll_d20"]',
    );
    await expect
      .poll(async () => d20.count(), { timeout: TOOL_TIMEOUT })
      .toBe(5);
    // Final scripted roll lands the 20 — proves the chain advanced through
    // all 5 fixtures, not just the first two before bailing to content.
    await expect
      .poll(async () => d20.nth(4).getAttribute("data-result"), {
        timeout: TOOL_TIMEOUT,
      })
      .toMatch(/"value":\s*20|"result":\s*20/);
    await expect(page.getByText("Rolled the d20 five times")).toBeVisible({
      timeout: TOOL_TIMEOUT,
    });
  });

  test("every rendered card matches the built-in default-renderer DOM signature", async ({
    page,
  }) => {
    // Drive a single pill that produces a single card so the assertions
    // here are scoped to the exact DOM the framework's default renderer
    // produces.
    await page
      .locator('[data-testid="copilot-suggestion"]')
      .filter({ hasText: "Weather in SF" })
      .first()
      .click();

    const card = page.locator('[data-testid="copilot-tool-render"]').first();
    await expect(card).toBeVisible({ timeout: TOOL_TIMEOUT });

    // The built-in default renderer always exposes name + status pill.
    await expect(
      card.locator('[data-testid="copilot-tool-render-name"]'),
    ).toBeVisible({ timeout: TOOL_TIMEOUT });
    await expect(
      card.locator('[data-testid="copilot-tool-render-status"]'),
    ).toBeVisible({ timeout: TOOL_TIMEOUT });

    // Every card on the page shares the same wrapper testid count as
    // the inner-name and inner-status testids — proves the built-in
    // shell is what's painting (no per-tool shells).
    const total = await page
      .locator('[data-testid="copilot-tool-render"]')
      .count();
    await expect(
      page.locator('[data-testid="copilot-tool-render-name"]'),
    ).toHaveCount(total);
    await expect(
      page.locator('[data-testid="copilot-tool-render-status"]'),
    ).toHaveCount(total);
  });
});