tool-rendering-custom-catchall.spec.ts

import { test, expect } from "@playwright/test";

// QA reference: qa/tool-rendering-custom-catchall.md
// Demo source: src/app/demos/tool-rendering-custom-catchall/page.tsx
// Renderer source: src/app/demos/tool-rendering-custom-catchall/custom-catchall-renderer.tsx
//
// This cell registers a SINGLE branded wildcard renderer via
// `useDefaultRenderTool`. Every tool call must paint via the same
// `[data-testid="custom-wildcard-card"]` shell — no per-tool
// specialization. Test 6 is the load-bearing assertion: every card on
// the page after each pill click shares the same testid signature.

const SUGGESTION_TIMEOUT = 15000;
const TOOL_TIMEOUT = 60000;

const PILLS = ["Weather in SF", "Find flights", "Roll a d20", "Chain tools"];

test.describe("Tool Rendering — Custom Catch-all (branded wildcard)", () => {
  test.beforeEach(async ({ page }) => {
    await page.goto("/demos/tool-rendering-custom-catchall");
    await expect(page.getByPlaceholder("Type a message")).toBeVisible({
      timeout: SUGGESTION_TIMEOUT,
    });
  });

  test("page loads with composer and 4 suggestion pills", async ({ page }) => {
    const suggestions = page.locator('[data-testid="copilot-suggestion"]');
    for (const title of PILLS) {
      await expect(suggestions.filter({ hasText: title }).first()).toBeVisible({
        timeout: SUGGESTION_TIMEOUT,
      });
    }

    // Sanity: per-tool branded testids from sibling cells stay at zero.
    await expect(page.locator('[data-testid="weather-card"]')).toHaveCount(0);
    await expect(page.locator('[data-testid="flights-card"]')).toHaveCount(0);
    await expect(page.locator('[data-testid="stock-card"]')).toHaveCount(0);
    await expect(page.locator('[data-testid="d20-card"]')).toHaveCount(0);
    // Sanity: the OOTB default-renderer testid does NOT appear here.
    await expect(
      page.locator('[data-testid="copilot-tool-render"]'),
    ).toHaveCount(0);
  });

  test("Weather in SF pill paints the branded wildcard card for get_weather", async ({
    page,
  }) => {
    await page
      .locator('[data-testid="copilot-suggestion"]')
      .filter({ hasText: "Weather in SF" })
      .first()
      .click();

    const card = page
      .locator(
        '[data-testid="custom-wildcard-card"][data-tool-name="get_weather"]',
      )
      .first();
    await expect(card).toBeVisible({ timeout: TOOL_TIMEOUT });
    await expect(
      card.locator('[data-testid="custom-wildcard-tool-name"]'),
    ).toHaveText("get_weather");
    await expect(
      card.locator('[data-testid="custom-wildcard-args"]'),
    ).toContainText("San Francisco", { timeout: TOOL_TIMEOUT });
  });

  test("Find flights pill paints the SAME branded wildcard card for search_flights", async ({
    page,
  }) => {
    await page
      .locator('[data-testid="copilot-suggestion"]')
      .filter({ hasText: "Find flights" })
      .first()
      .click();

    const card = page
      .locator(
        '[data-testid="custom-wildcard-card"][data-tool-name="search_flights"]',
      )
      .first();
    await expect(card).toBeVisible({ timeout: TOOL_TIMEOUT });
    await expect(
      card.locator('[data-testid="custom-wildcard-tool-name"]'),
    ).toHaveText("search_flights");

    // Result block surfaces the deterministic flights from our fixture
    // (NOT the a2ui beautiful-chat boilerplate).
    await expect(
      card.locator('[data-testid="custom-wildcard-result"]'),
    ).toContainText(/United|Delta|JetBlue/, { timeout: TOOL_TIMEOUT });
  });

  test("Roll a d20 pill paints exactly 5 wildcard cards, last result is 20", async ({
    page,
  }) => {
    await page
      .locator('[data-testid="copilot-suggestion"]')
      .filter({ hasText: "Roll a d20" })
      .first()
      .click();

    const cards = page.locator(
      '[data-testid="custom-wildcard-card"][data-tool-name="roll_d20"]',
    );
    await expect
      .poll(async () => cards.count(), { timeout: TOOL_TIMEOUT })
      .toBe(5);

    // 5th card's result is 20.
    await expect(
      cards.nth(4).locator('[data-testid="custom-wildcard-result"]'),
    ).toContainText(/"value":\s*20|"result":\s*20/, { timeout: TOOL_TIMEOUT });

    // First 4 are non-20.
    for (let i = 0; i < 4; i++) {
      const txt = await cards
        .nth(i)
        .locator('[data-testid="custom-wildcard-result"]')
        .innerText();
      expect(txt).not.toMatch(/"value":\s*20|"result":\s*20/);
    }
  });

  test("Chain tools pill paints 3 wildcard cards (one per tool)", async ({
    page,
  }) => {
    await page
      .locator('[data-testid="copilot-suggestion"]')
      .filter({ hasText: "Chain tools" })
      .first()
      .click();

    await expect(
      page
        .locator(
          '[data-testid="custom-wildcard-card"][data-tool-name="get_weather"]',
        )
        .first(),
    ).toBeVisible({ timeout: TOOL_TIMEOUT });
    await expect(
      page
        .locator(
          '[data-testid="custom-wildcard-card"][data-tool-name="search_flights"]',
        )
        .first(),
    ).toBeVisible({ timeout: TOOL_TIMEOUT });
    await expect(
      page
        .locator(
          '[data-testid="custom-wildcard-card"][data-tool-name="roll_d20"]',
        )
        .first(),
    ).toBeVisible({ timeout: TOOL_TIMEOUT });
  });

  test("every rendered card shares the same wildcard testid signature", async ({
    page,
  }) => {
    // Cross-tool sanity: drive Chain tools (3 distinct tools → 3
    // cards) and assert every card matches the same wildcard shell.
    await page
      .locator('[data-testid="copilot-suggestion"]')
      .filter({ hasText: "Chain tools" })
      .first()
      .click();

    const cards = page.locator('[data-testid="custom-wildcard-card"]');
    await expect
      .poll(async () => cards.count(), { timeout: TOOL_TIMEOUT })
      .toBeGreaterThanOrEqual(3);

    const total = await cards.count();
    await expect(
      page.locator('[data-testid="custom-wildcard-tool-name"]'),
    ).toHaveCount(total);
    await expect(
      page.locator('[data-testid="custom-wildcard-args"]'),
    ).toHaveCount(total);

    // All cards expose distinct tool names but the SAME shell.
    const toolNames = await cards.evaluateAll((nodes) =>
      nodes.map((n) => n.getAttribute("data-tool-name")),
    );
    const uniqueNames = new Set(toolNames);
    expect(uniqueNames.size).toBeGreaterThanOrEqual(3);
    for (const name of toolNames) {
      expect(["get_weather", "search_flights", "roll_d20"]).toContain(name);
    }

    // The OOTB default-renderer testid stays at zero — proves the
    // single custom wildcard is what painted, not the framework
    // fallback.
    await expect(
      page.locator('[data-testid="copilot-tool-render"]'),
    ).toHaveCount(0);
  });

  // Regression for the aimock multi-pill bug:
  // The d20 and Chain-tools fixtures used global thread state
  // (`turnIndex`, `hasToolResult`) to drive sequencing. After clicking
  // Find flights, the d20 loop entered at turnIndex=2 (rendering only 3
  // cards instead of 5) and the Chain-tools tool-emitting fixture was
  // skipped entirely (no tool cards, just the final "Done — Tokyo is
  // sunny…" content). Fix: chain all follow-ups via `toolCallId`. This
  // test drives Find flights → Roll a d20 → Chain tools in one thread
  // and asserts the wildcard renderer paints the full card sequence for
  // every pill (1 + 5 + 3 = 9 cards).
  test("sequential pills in one thread render full card sequences for each", async ({
    page,
  }) => {
    // Three sequential pills × multi-tool chains × LLM-mock latency easily
    // exceeds Playwright's 30s default. Bumped to cover the worst case.
    test.setTimeout(240_000);

    await page
      .locator('[data-testid="copilot-suggestion"]')
      .filter({ hasText: "Find flights" })
      .first()
      .click();
    const cards = page.locator('[data-testid="custom-wildcard-card"]');
    await expect
      .poll(async () => cards.count(), { timeout: TOOL_TIMEOUT })
      .toBe(1);

    await page
      .locator('[data-testid="copilot-suggestion"]')
      .filter({ hasText: "Roll a d20" })
      .first()
      .click();
    // 1 (flights) + 5 (d20) = 6 cards once d20 chain finishes.
    await expect
      .poll(async () => cards.count(), { timeout: TOOL_TIMEOUT })
      .toBe(6);
    await expect(page.getByText("Rolled the d20 five times")).toBeVisible({
      timeout: TOOL_TIMEOUT,
    });

    await page
      .locator('[data-testid="copilot-suggestion"]')
      .filter({ hasText: "Chain tools" })
      .first()
      .click();
    // 1 + 5 + 3 = 9 once chain tools mounts get_weather + search_flights +
    // roll_d20 cards.
    await expect
      .poll(async () => cards.count(), { timeout: TOOL_TIMEOUT })
      .toBe(9);
    await expect(page.getByText("Done — Tokyo is sunny")).toBeVisible({
      timeout: TOOL_TIMEOUT,
    });
  });
});