From 803f3d8d015e4f79f4facec64beaaf1e07ee980d Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sun, 31 May 2026 11:43:49 -0700 Subject: [PATCH 1/5] chore(showcase): remove unused QA-to-Notion sync workflow and script --- .github/workflows/showcase_qa-sync.yml | 64 ---- showcase/scripts/package.json | 2 - showcase/scripts/sync-qa-to-notion.ts | 435 ------------------------- 3 files changed, 501 deletions(-) delete mode 100644 .github/workflows/showcase_qa-sync.yml delete mode 100644 showcase/scripts/sync-qa-to-notion.ts diff --git a/.github/workflows/showcase_qa-sync.yml b/.github/workflows/showcase_qa-sync.yml deleted file mode 100644 index ee3066dbb64..00000000000 --- a/.github/workflows/showcase_qa-sync.yml +++ /dev/null @@ -1,64 +0,0 @@ -name: "Showcase: Sync QA to Notion" - -on: - push: - branches: [main] - paths: - - "showcase/integrations/*/qa/**" - - "showcase/integrations/*/manifest.yaml" - - "showcase/scripts/sync-qa-to-notion.ts" - workflow_dispatch: - -concurrency: - group: showcase-qa-sync - cancel-in-progress: true - -permissions: - contents: read - -jobs: - sync-qa: - name: Sync QA Instructions - runs-on: ubuntu-latest - timeout-minutes: 10 - permissions: - contents: read - env: - SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }} - - steps: - - name: Checkout - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4 - with: - persist-credentials: false - - - name: Setup Node.js - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 - with: - node-version: 20 - - - name: Setup pnpm - uses: pnpm/action-setup@0e279bb959325dab635dd2c09392533439d90093 # v6.0.8 - - - name: Install dependencies - working-directory: showcase/scripts - run: pnpm install --frozen-lockfile || pnpm install - - - name: Validate manifests - working-directory: showcase/scripts - run: npx tsx generate-registry.ts - - - name: Sync QA to Notion - working-directory: showcase/scripts - env: - NOTION_API_KEY: ${{ secrets.SHOWCASE_NOTION_API_KEY }} - run: npx tsx sync-qa-to-notion.ts - - - name: Notify Slack (failure) - if: failure() && env.SLACK_WEBHOOK != '' - uses: slackapi/slack-github-action@b0fa283ad8fea605de13dc3f449259339835fc52 # v2.1.0 - with: - webhook: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }} - webhook-type: incoming-webhook - payload: | - { "text": ":x: *QA sync to Notion*: failed | " } diff --git a/showcase/scripts/package.json b/showcase/scripts/package.json index c7eeb807aed..ced997bc415 100644 --- a/showcase/scripts/package.json +++ b/showcase/scripts/package.json @@ -10,8 +10,6 @@ "validate-manifests": "tsx generate-registry.ts --validate-only", "showcase:audit": "tsx audit.ts", "create-integration": "tsx create-integration/index.ts", - "sync-qa": "tsx sync-qa-to-notion.ts", - "sync-qa:dry": "DRY_RUN=true tsx sync-qa-to-notion.ts", "bundle-content": "tsx bundle-demo-content.ts", "bundle-setup-content": "tsx bundle-setup-content.ts", "extract-starter": "tsx extract-starter.ts", diff --git a/showcase/scripts/sync-qa-to-notion.ts b/showcase/scripts/sync-qa-to-notion.ts deleted file mode 100644 index e52dd5c1efe..00000000000 --- a/showcase/scripts/sync-qa-to-notion.ts +++ /dev/null @@ -1,435 +0,0 @@ -// Sync QA Instructions to Notion -// -// Reads qa/*.md files from integration packages and syncs them -// to Notion pages under the "Showcase QA Instructions" parent page. -// -// Structure: -// Showcase QA Instructions (parent) -// └── LangGraph (Python) (integration page) -// ├── Agentic Chat (feature QA page) -// ├── Human in the Loop (feature QA page) -// └── ... -// -// Usage: -// NOTION_API_KEY=secret_xxx npx tsx showcase/scripts/sync-qa-to-notion.ts -// -// Environment: -// NOTION_API_KEY - Notion integration token (required) -// QA_PARENT_PAGE_ID - Notion page ID for "Showcase QA Instructions" -// (default: 32d3aa38-1852-81af-a1f4-d1ef37402428) -// DRY_RUN - Set to "true" to preview without writing (optional) - -import fs from "fs"; -import path from "path"; -import { fileURLToPath } from "url"; -import yaml from "yaml"; - -const __filename = fileURLToPath(import.meta.url); -const __dirname = path.dirname(__filename); - -const ROOT = path.resolve(__dirname, ".."); -const PACKAGES_DIR = path.join(ROOT, "integrations"); -const FEATURE_REGISTRY_PATH = path.join( - ROOT, - "shared", - "feature-registry.json", -); - -const NOTION_API_KEY = process.env.NOTION_API_KEY; -const QA_PARENT_PAGE_ID = - process.env.QA_PARENT_PAGE_ID || "3423aa38-1852-8126-84d5-e40a2bd5a7ec"; -const DRY_RUN = process.env.DRY_RUN === "true"; - -const NOTION_API = "https://api.notion.com/v1"; -const NOTION_VERSION = "2022-06-28"; - -interface Feature { - id: string; - name: string; -} - -interface QAFile { - integrationSlug: string; - integrationName: string; - featureId: string; - featureName: string; - content: string; - filePath: string; -} - -// --- Notion API helpers --- - -async function notionRequest( - endpoint: string, - method: string = "GET", - body?: any, -): Promise { - const response = await fetch(`${NOTION_API}${endpoint}`, { - method, - headers: { - Authorization: `Bearer ${NOTION_API_KEY}`, - "Notion-Version": NOTION_VERSION, - "Content-Type": "application/json", - }, - body: body ? JSON.stringify(body) : undefined, - }); - - if (!response.ok) { - const error = await response.text(); - throw new Error(`Notion API error ${response.status}: ${error}`); - } - - return response.json(); -} - -async function findChildPage( - parentId: string, - title: string, -): Promise { - const result = await notionRequest( - `/blocks/${parentId}/children?page_size=100`, - ); - - for (const block of result.results) { - if (block.type === "child_page" && block.child_page?.title === title) { - return block.id; - } - } - return null; -} - -async function createPage( - parentId: string, - title: string, - icon?: string, -): Promise { - const result = await notionRequest("/pages", "POST", { - parent: { page_id: parentId }, - properties: { - title: { - title: [{ text: { content: title } }], - }, - }, - ...(icon ? { icon: { emoji: icon } } : {}), - }); - return result.id; -} - -function markdownToNotionBlocks(markdown: string): any[] { - const blocks: any[] = []; - const lines = markdown.split("\n"); - let i = 0; - - while (i < lines.length) { - const line = lines[i]; - - // Headings - if (line.startsWith("### ")) { - blocks.push({ - type: "heading_3", - heading_3: { - rich_text: [{ text: { content: line.slice(4).trim() } }], - }, - }); - i++; - continue; - } - if (line.startsWith("## ")) { - blocks.push({ - type: "heading_2", - heading_2: { - rich_text: [{ text: { content: line.slice(3).trim() } }], - }, - }); - i++; - continue; - } - if (line.startsWith("# ")) { - blocks.push({ - type: "heading_1", - heading_1: { - rich_text: [{ text: { content: line.slice(2).trim() } }], - }, - }); - i++; - continue; - } - - // Checkbox items (- [ ] or - [x]) - const checkboxMatch = line.match(/^- \[([ x])\] (.+)/); - if (checkboxMatch) { - blocks.push({ - type: "to_do", - to_do: { - rich_text: [{ text: { content: checkboxMatch[2].trim() } }], - checked: checkboxMatch[1] === "x", - }, - }); - i++; - continue; - } - - // Bullet items (- text) - if (line.startsWith("- ")) { - blocks.push({ - type: "bulleted_list_item", - bulleted_list_item: { - rich_text: [{ text: { content: line.slice(2).trim() } }], - }, - }); - i++; - continue; - } - - // Horizontal rule - if (line.match(/^---+$/)) { - blocks.push({ type: "divider", divider: {} }); - i++; - continue; - } - - // Empty line - if (line.trim() === "") { - i++; - continue; - } - - // Regular paragraph - blocks.push({ - type: "paragraph", - paragraph: { - rich_text: [{ text: { content: line } }], - }, - }); - i++; - } - - return blocks; -} - -async function replacePageContent( - pageId: string, - blocks: any[], -): Promise { - // First, delete all existing blocks - const existing = await notionRequest( - `/blocks/${pageId}/children?page_size=100`, - ); - for (const block of existing.results) { - await notionRequest(`/blocks/${block.id}`, "DELETE"); - } - - // Then add new blocks in batches of 100 (Notion API limit) - for (let i = 0; i < blocks.length; i += 100) { - const batch = blocks.slice(i, i + 100); - await notionRequest(`/blocks/${pageId}/children`, "PATCH", { - children: batch, - }); - } -} - -// --- Main logic --- - -function loadFeatureNames(): Map { - const raw = fs.readFileSync(FEATURE_REGISTRY_PATH, "utf-8"); - const registry = JSON.parse(raw); - const map = new Map(); - for (const feature of registry.features) { - map.set(feature.id, feature.name); - } - return map; -} - -function collectQAFiles(): QAFile[] { - const featureNames = loadFeatureNames(); - const qaFiles: QAFile[] = []; - - if (!fs.existsSync(PACKAGES_DIR)) return qaFiles; - - const packageDirs = fs - .readdirSync(PACKAGES_DIR, { withFileTypes: true }) - .filter((d) => d.isDirectory()) - .map((d) => d.name); - - for (const pkgDir of packageDirs) { - const manifestPath = path.join(PACKAGES_DIR, pkgDir, "manifest.yaml"); - if (!fs.existsSync(manifestPath)) continue; - - const manifest = yaml.parse(fs.readFileSync(manifestPath, "utf-8")); - const qaDir = path.join(PACKAGES_DIR, pkgDir, "qa"); - if (!fs.existsSync(qaDir)) continue; - - const qaEntries = fs.readdirSync(qaDir).filter((f) => f.endsWith(".md")); - - for (const qaFile of qaEntries) { - const featureId = qaFile.replace(".md", ""); - const content = fs.readFileSync(path.join(qaDir, qaFile), "utf-8"); - - qaFiles.push({ - integrationSlug: manifest.slug, - integrationName: manifest.name, - featureId, - featureName: featureNames.get(featureId) || featureId, - content, - filePath: path.join(qaDir, qaFile), - }); - } - } - - return qaFiles; -} - -async function syncToNotion(qaFiles: QAFile[]): Promise { - // Group by integration - const byIntegration = new Map(); - for (const file of qaFiles) { - if (!byIntegration.has(file.integrationSlug)) { - byIntegration.set(file.integrationSlug, { - name: file.integrationName, - files: [], - }); - } - byIntegration.get(file.integrationSlug)!.files.push(file); - } - - console.log( - `Syncing ${qaFiles.length} QA docs across ${byIntegration.size} integrations\n`, - ); - - for (const [slug, { name, files }] of byIntegration) { - console.log(` ${name} (${slug}): ${files.length} QA docs`); - - if (DRY_RUN) { - for (const file of files) { - console.log(` - ${file.featureName} (${file.featureId})`); - } - continue; - } - - // Find or create integration page - let integrationPageId = await findChildPage(QA_PARENT_PAGE_ID, name); - if (!integrationPageId) { - console.log(` Creating integration page: ${name}`); - integrationPageId = await createPage(QA_PARENT_PAGE_ID, name, "📦"); - } - - // Sync each feature QA doc - for (const file of files) { - const pageTitle = `QA: ${file.featureName}`; - let featurePageId = await findChildPage(integrationPageId, pageTitle); - - if (!featurePageId) { - console.log(` Creating: ${pageTitle}`); - featurePageId = await createPage(integrationPageId, pageTitle, "✅"); - } else { - console.log(` Updating: ${pageTitle}`); - } - - // Convert markdown to Notion blocks and replace content - const blocks = markdownToNotionBlocks(file.content); - - // Add source file reference at the bottom - blocks.push( - { type: "divider", divider: {} }, - { - type: "paragraph", - paragraph: { - rich_text: [ - { - text: { content: "Source: " }, - annotations: { italic: true, color: "gray" }, - }, - { - text: { - content: `showcase/integrations/${file.integrationSlug}/qa/${file.featureId}.md`, - link: { - url: `https://github.com/CopilotKit/CopilotKit/blob/main/showcase/integrations/${file.integrationSlug}/qa/${file.featureId}.md`, - }, - }, - annotations: { italic: true, color: "gray", code: true }, - }, - ], - }, - }, - { - type: "paragraph", - paragraph: { - rich_text: [ - { - text: { - content: `Last synced: ${new Date().toISOString()}`, - }, - annotations: { italic: true, color: "gray" }, - }, - ], - }, - }, - ); - - await replacePageContent(featurePageId, blocks); - } - } - - // Update the parent page's "last synced" timestamp - if (!DRY_RUN) { - const parentBlocks = await notionRequest( - `/blocks/${QA_PARENT_PAGE_ID}/children?page_size=100`, - ); - for (const block of parentBlocks.results) { - if ( - block.type === "paragraph" && - block.paragraph?.rich_text?.[0]?.text?.content?.startsWith( - "Last synced:", - ) - ) { - await notionRequest(`/blocks/${block.id}`, "PATCH", { - paragraph: { - rich_text: [ - { - text: { - content: `Last synced: ${new Date().toISOString()}`, - }, - annotations: { italic: true }, - }, - ], - }, - }); - break; - } - } - } -} - -async function main() { - if (!NOTION_API_KEY) { - console.error("Error: NOTION_API_KEY environment variable is required."); - console.error( - "Create a Notion integration at https://www.notion.so/my-integrations", - ); - console.error("and share the QA parent page with it."); - process.exit(1); - } - - if (DRY_RUN) { - console.log("DRY RUN mode — no changes will be made to Notion\n"); - } - - const qaFiles = collectQAFiles(); - - if (qaFiles.length === 0) { - console.log("No QA files found in any integration packages."); - return; - } - - console.log(`Found ${qaFiles.length} QA files:\n`); - - try { - await syncToNotion(qaFiles); - console.log("\nSync complete."); - } catch (error) { - console.error("\nSync failed:", error); - process.exit(1); - } -} - -main(); From cfe1b1ae9556ec5db6433ea154709a3d0ffe4082 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sun, 31 May 2026 12:36:13 -0700 Subject: [PATCH 2/5] ci: fix zizmor ref-version-mismatch version comments on 3 pinned actions --- .github/workflows/publish-release.yml | 2 +- .github/workflows/test_e2e-dojo.yml | 2 +- .github/workflows/test_unit-python-sdk.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/publish-release.yml b/.github/workflows/publish-release.yml index ee7ae5d8922..948bdcc35e5 100644 --- a/.github/workflows/publish-release.yml +++ b/.github/workflows/publish-release.yml @@ -463,7 +463,7 @@ jobs: - name: Install Poetry if: steps.detect.outputs.should_publish == 'true' - uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 + uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1.4.1 with: version: latest virtualenvs-create: true diff --git a/.github/workflows/test_e2e-dojo.yml b/.github/workflows/test_e2e-dojo.yml index 30245a2d210..eeb03d88618 100644 --- a/.github/workflows/test_e2e-dojo.yml +++ b/.github/workflows/test_e2e-dojo.yml @@ -238,7 +238,7 @@ jobs: - name: Install Poetry if: steps.should-run.outputs.skip != 'true' && matrix.needs_python - uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 + uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1.4.1 with: version: latest virtualenvs-create: true diff --git a/.github/workflows/test_unit-python-sdk.yml b/.github/workflows/test_unit-python-sdk.yml index cdc77f84974..ee2c752c256 100644 --- a/.github/workflows/test_unit-python-sdk.yml +++ b/.github/workflows/test_unit-python-sdk.yml @@ -43,7 +43,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install Poetry - uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1 + uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1.4.1 with: version: latest virtualenvs-create: true From 4dab96569c08a952af50316b5e73349ba396ebd9 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sun, 31 May 2026 15:16:33 -0700 Subject: [PATCH 3/5] fix(showcase/harness): stagger d6 service startup + re-acquire disconnected browsers to avoid PID-exhaustion crashes --- .../src/probes/drivers/d6-all-pills.test.ts | 112 +++++++++++++++++ .../src/probes/drivers/d6-all-pills.ts | 22 +++- .../src/probes/loader/probe-invoker.test.ts | 115 ++++++++++++++++++ .../src/probes/loader/probe-invoker.ts | 59 ++++++++- 4 files changed, 305 insertions(+), 3 deletions(-) diff --git a/showcase/harness/src/probes/drivers/d6-all-pills.test.ts b/showcase/harness/src/probes/drivers/d6-all-pills.test.ts index 2464f373e4f..998d4c3a81a 100644 --- a/showcase/harness/src/probes/drivers/d6-all-pills.test.ts +++ b/showcase/harness/src/probes/drivers/d6-all-pills.test.ts @@ -20,6 +20,8 @@ import { } from "../helpers/d5-registry.js"; import type { D5Script } from "../helpers/d5-registry.js"; import { logger } from "../../logger.js"; +import type { Browser } from "playwright"; +import type { BrowserPool } from "../helpers/browser-pool.js"; import type { ProbeContext, ProbeResult, @@ -527,4 +529,114 @@ describe("e2e-full driver", () => { expect(() => sem.release()).toThrow(); }); }); + + // -------------------------------------------------------------------- + // Defensive re-acquire on disconnected browser. The pool's own + // acquire() skips zombies whose `disconnected` event has already + // fired, but a browser can also die in the narrow window AFTER + // acquire() returns it but BEFORE the caller hands it to + // `browser.newContext()` — most commonly during the D6 service + // fan-out's Chromium-spawn burst when fork() returns EAGAIN. The + // launcher must release-and-re-acquire so the entire service's + // ~40 features don't all fail with "Target page, context or + // browser has been closed". + // -------------------------------------------------------------------- + describe("createPooledE2eFullLauncher", () => { + it("releases and re-acquires when acquire() returns a disconnected browser", async () => { + const dead = makeFakeBrowserish(false); + const fresh = makeFakeBrowserish(true); + const acquired: FakeBrowserish[] = []; + const released: FakeBrowserish[] = []; + let nextIdx = 0; + const queue = [dead, fresh]; + const fakePool = { + async acquire() { + const b = queue[nextIdx++]!; + acquired.push(b); + return b as unknown as Browser; + }, + release(b: unknown) { + released.push(b as FakeBrowserish); + }, + stats() { + return { size: 1, available: 0, inUse: 1, totalRecycles: 0 }; + }, + }; + const warnings: Array<{ event: string; meta?: unknown }> = []; + const warnLogger = { + warn: (event: string, meta?: Record) => + warnings.push({ event, meta }), + }; + const launcher = createPooledE2eFullLauncher( + fakePool as unknown as BrowserPool, + warnLogger, + ); + const browser = await launcher(); + // Must have acquired twice (dead, then fresh) and released the + // dead instance back to the pool exactly once. + expect(acquired).toHaveLength(2); + expect(acquired[0]).toBe(dead); + expect(acquired[1]).toBe(fresh); + expect(released).toHaveLength(1); + expect(released[0]).toBe(dead); + // The launcher must return a usable E2eFullBrowser; smoke-test + // newContext() to confirm we got back the FRESH instance. + await browser.newContext(); + // Diagnostic warn surfaced so operators can correlate. + expect(warnings.map((w) => w.event)).toContain( + "probe.e2e-full.pool-acquire-dead", + ); + }); + + it("does not re-acquire when the first acquire returns a healthy browser", async () => { + const healthy = makeFakeBrowserish(true); + const acquired: FakeBrowserish[] = []; + const released: FakeBrowserish[] = []; + const fakePool = { + async acquire() { + acquired.push(healthy); + return healthy as unknown as Browser; + }, + release(b: unknown) { + released.push(b as FakeBrowserish); + }, + stats() { + return { size: 1, available: 0, inUse: 1, totalRecycles: 0 }; + }, + }; + const launcher = createPooledE2eFullLauncher( + fakePool as unknown as BrowserPool, + ); + await launcher(); + expect(acquired).toHaveLength(1); + expect(released).toHaveLength(0); + }); + }); }); + +// Module-scoped helpers for the createPooledE2eFullLauncher tests above — +// lifted out of the describe so oxlint's consistent-function-scoping is +// satisfied (the factory captures no parent state). +interface FakeBrowserish { + connected: boolean; + newContext(): Promise<{ + close(): Promise; + newPage(): Promise; + }>; + isConnected(): boolean; +} + +function makeFakeBrowserish(connected: boolean): FakeBrowserish { + return { + connected, + isConnected() { + return this.connected; + }, + async newContext() { + return { + close: async () => {}, + newPage: async () => ({}), + }; + }, + }; +} diff --git a/showcase/harness/src/probes/drivers/d6-all-pills.ts b/showcase/harness/src/probes/drivers/d6-all-pills.ts index 99d978f2a6a..91ebb629c82 100644 --- a/showcase/harness/src/probes/drivers/d6-all-pills.ts +++ b/showcase/harness/src/probes/drivers/d6-all-pills.ts @@ -333,7 +333,27 @@ export function createPooledE2eFullLauncher( logger?: { warn(event: string, meta?: Record): void }, ): E2eFullBrowserLauncher { return async (abortSignal?: AbortSignal): Promise => { - const browser = await pool.acquire(); + let browser = await pool.acquire(); + + // Defensive re-acquire: the pool's acquire() skips zombie slots whose + // browser is already disconnected, but a browser can also die in the + // narrow window AFTER acquire() returns it but BEFORE we hand it to + // `browser.newContext()` — most commonly during the D6 service fan-out's + // Chromium-spawn burst, when the kernel returns EAGAIN on fork and a + // recently-handed-out chromium dies between this caller and its first + // newContext call. Without this guard, the dead browser dooms an entire + // service's ~40 features ("Target page, context or browser has been + // closed" on every single feature). One re-acquire is enough — release + // routes the dead instance back into the pool where its `isConnected` + // check + `recycleSlot` recovery take over. + if (!browser.isConnected()) { + logger?.warn("probe.e2e-full.pool-acquire-dead", { + poolAvailable: pool.stats().available, + poolInUse: pool.stats().inUse, + }); + pool.release(browser); + browser = await pool.acquire(); + } let forceReleased = false; const openContexts = new Set<{ close(): Promise }>(); diff --git a/showcase/harness/src/probes/loader/probe-invoker.test.ts b/showcase/harness/src/probes/loader/probe-invoker.test.ts index b15a928fb36..e50fce5e33b 100644 --- a/showcase/harness/src/probes/loader/probe-invoker.test.ts +++ b/showcase/harness/src/probes/loader/probe-invoker.test.ts @@ -3706,4 +3706,119 @@ describe("buildProbeInvoker", () => { expect(persisted.total).toBe(persisted.passed + persisted.failed); expect(persisted.failed).toBeGreaterThanOrEqual(1); }); + + // ---------------------------------------------------------------------- + // Service-startup stagger — protects against the d6 Chromium-spawn + // thundering herd that crashes the BrowserPool when many services fan + // out into pool.acquire() in the same JS tick. The stagger spaces + // worker-zero..N-1's FIRST pull over `(N-1)*staggerMs`, so per-Chromium + // thread-spawn bursts don't trip the container's PID/thread ceiling. + // ---------------------------------------------------------------------- + it("staggers worker first-pull start times across the fan-out", async () => { + const inputSchema = z.object({ key: z.string() }); + const startTimes: number[] = []; + // Each driver run hangs long enough that every worker's FIRST pull + // happens before any of them finish — that's the regime where the + // stagger actually matters (it spaces the per-service Chromium + // launches, not the steady-state cursor draining). + const HOLD_MS = 500; + const driver: ProbeDriver, { ok: true }> = { + kind: "smoke", + inputSchema, + async run(ctx, input) { + startTimes.push(Date.now()); + await new Promise((r) => setTimeout(r, HOLD_MS)); + return { + key: input.key, + state: "green", + signal: { ok: true }, + observedAt: ctx.now().toISOString(), + }; + }, + }; + const targets = Array.from({ length: 4 }, (_, i) => ({ + key: `smoke:t${i}`, + })); + const cfg: ProbeConfig = { + kind: "smoke", + id: "smoke-stagger", + schedule: "*/15 * * * *", + max_concurrency: 4, + targets, + }; + const { writer, writes } = mkWriter(); + const STAGGER = 50; + const invoker = buildProbeInvoker(cfg, { + driver, + schedulerId: cfg.id, + discoveryRegistry: createDiscoveryRegistry(), + writer, + ...BASE_DEPS, + // Override the default 300ms to keep the test fast; uses real timers + // so a small positive value is enough to assert ordering. + env: { SERVICE_STARTUP_STAGGER_MS: String(STAGGER) } as Readonly< + Record + >, + }); + await invoker(); + expect(writes).toHaveLength(4); + expect(startTimes).toHaveLength(4); + // Worker 0 starts immediately; workers 1..N-1 each delayed by their + // index * stagger. Assert monotonic non-decreasing start times and a + // spread between first and last that is at least (N-1) * stagger + // less a small jitter margin. Use a generous lower bound so this is + // not flaky under CI scheduling jitter. + for (let i = 1; i < startTimes.length; i++) { + expect(startTimes[i]!).toBeGreaterThanOrEqual(startTimes[i - 1]!); + } + const totalSpread = startTimes[3]! - startTimes[0]!; + // (4 workers - 1) * 50ms = 150ms expected spread; allow 60% floor for + // CI scheduling jitter on slow runners. + expect(totalSpread).toBeGreaterThanOrEqual(Math.floor(3 * STAGGER * 0.6)); + }); + + it("disables stagger when SERVICE_STARTUP_STAGGER_MS=0", async () => { + const inputSchema = z.object({ key: z.string() }); + const startTimes: number[] = []; + const driver: ProbeDriver, { ok: true }> = { + kind: "smoke", + inputSchema, + async run(ctx, input) { + startTimes.push(Date.now()); + return { + key: input.key, + state: "green", + signal: { ok: true }, + observedAt: ctx.now().toISOString(), + }; + }, + }; + const targets = Array.from({ length: 4 }, (_, i) => ({ + key: `smoke:t${i}`, + })); + const cfg: ProbeConfig = { + kind: "smoke", + id: "smoke-no-stagger", + schedule: "*/15 * * * *", + max_concurrency: 4, + targets, + }; + const { writer } = mkWriter(); + const invoker = buildProbeInvoker(cfg, { + driver, + schedulerId: cfg.id, + discoveryRegistry: createDiscoveryRegistry(), + writer, + ...BASE_DEPS, + env: { SERVICE_STARTUP_STAGGER_MS: "0" } as Readonly< + Record + >, + }); + await invoker(); + // With stagger=0 every worker pulls its first input in the same + // microtask flush — the spread is bounded by JS scheduling latency + // for sibling promises, well under any meaningful stagger value. + const totalSpread = startTimes[3]! - startTimes[0]!; + expect(totalSpread).toBeLessThan(50); + }); }); diff --git a/showcase/harness/src/probes/loader/probe-invoker.ts b/showcase/harness/src/probes/loader/probe-invoker.ts index f06ffa72edb..80da70a612f 100644 --- a/showcase/harness/src/probes/loader/probe-invoker.ts +++ b/showcase/harness/src/probes/loader/probe-invoker.ts @@ -22,6 +22,47 @@ import type { ProbeRunWriter, ProbeRunSummary } from "../run-history.js"; */ const SYNTHETIC_ERROR_MSG_BUDGET = 1200; +/** + * Per-worker startup stagger (ms) for the fan-out loop below. When a + * probe like d6-all-pills-e2e fans `max_concurrency` workers out across + * its discovered services in the same JS tick, each worker + * immediately drives a `pool.acquire()` that launches a Chromium + * (~50 threads/procs each). With 8 simultaneous workers the kernel + * sees ~400 thread spawns inside a 4ms window on top of an already- + * warm 10-browser pool, and the container hits its PID/thread ceiling + * — fork/pthread_create returns EAGAIN, Chromium processes die, and + * per-feature `browser.newContext()` fails with "Target page, context + * or browser has been closed" so the whole run aborts at T+2s. + * + * The fix is to spread the FIRST `pool.acquire()` of each worker out + * over `(concurrency - 1) * STAGGER_MS` so the per-Chromium thread-spawn + * bursts overlap rather than collide. Subsequent iterations of the same + * worker run at full speed — this only delays the initial fan-out + * (worker `i` waits `i * stagger` before pulling its first input), so + * wall-clock throughput is preserved. + * + * Env-overridable: `SERVICE_STARTUP_STAGGER_MS` (parsed as int; non-finite + * or negative values fall through to the default). Set to `0` to disable. + * + * Default chosen to spread 8 workers across ~2.1s — small relative to + * the 10-minute D6 budget but large enough that each Chromium's + * thread-spawn burst (~150ms in practice) is done before the next + * starts. + */ +const DEFAULT_SERVICE_STARTUP_STAGGER_MS = 300; + +function resolveStaggerMs( + env: Readonly>, +): number { + const raw = env.SERVICE_STARTUP_STAGGER_MS; + if (raw === undefined) return DEFAULT_SERVICE_STARTUP_STAGGER_MS; + const parsed = parseInt(raw, 10); + if (!Number.isFinite(parsed) || parsed < 0) { + return DEFAULT_SERVICE_STARTUP_STAGGER_MS; + } + return parsed; +} + /** * B7: minimal scheduler surface the invoker needs. Re-typed inline rather * than imported from `../scheduler/scheduler.ts` to keep this module @@ -517,8 +558,22 @@ export function buildProbeInvoker( // Hand-rolled bounded pool. Each worker pulls from a shared index so // N workers process the M inputs cooperatively — no Promise.all // stampede even when M >> N. + // + // Per-worker startup stagger: worker `i` sleeps `i * staggerMs` before + // its first pull so the per-service `pool.acquire()` Chromium-launch + // bursts (~50 threads/procs each) don't pile up inside a single JS + // tick and trip the container's PID/thread ceiling. See + // `DEFAULT_SERVICE_STARTUP_STAGGER_MS` for the full rationale. Only the + // FIRST iteration of each worker is staggered — subsequent iterations + // run at full speed so wall-clock throughput is unchanged. + const staggerMs = resolveStaggerMs(env); let cursor = 0; - const runOne = async (): Promise => { + const runOne = async (workerIndex: number): Promise => { + if (staggerMs > 0 && workerIndex > 0) { + await new Promise((resolve) => + setTimeout(resolve, workerIndex * staggerMs), + ); + } while (cursor < inputs.length) { // Invariant: `idx < inputs.length` from the loop precondition, // so `inputs[idx]` is always defined. Non-null assertion avoids @@ -611,7 +666,7 @@ export function buildProbeInvoker( if (resolved.ok) { const workers = Array.from( { length: Math.min(concurrency, Math.max(inputs.length, 1)) }, - () => runOne(), + (_, i) => runOne(i), ); await Promise.all(workers); } From 12dc63330239b34e3c0d19f47d2a6a706040830f Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sun, 31 May 2026 16:10:13 -0700 Subject: [PATCH 4/5] fix(showcase/harness): recycle browser between d6 features after timeout + lower FEATURE_CONCURRENCY_D6 to isolate fixture-miss cascade --- .../config/probes/d6-all-pills-e2e.yml | 8 +- .../src/probes/drivers/d6-all-pills.test.ts | 164 ++++++++++++++++- .../src/probes/drivers/d6-all-pills.ts | 172 +++++++++++++++++- 3 files changed, 332 insertions(+), 12 deletions(-) diff --git a/showcase/harness/config/probes/d6-all-pills-e2e.yml b/showcase/harness/config/probes/d6-all-pills-e2e.yml index 86f316c3faa..3632f0fa982 100644 --- a/showcase/harness/config/probes/d6-all-pills-e2e.yml +++ b/showcase/harness/config/probes/d6-all-pills-e2e.yml @@ -13,12 +13,16 @@ # -- Cadence: hourly at :40 -- # # Avoids e2e-demos at :10, D5 at :05/:20/:35/:50. With BROWSER_POOL_SIZE=10 -# and FEATURE_CONCURRENCY=4, estimated runtime ~7-10 min. +# and FEATURE_CONCURRENCY_D6=2 (lowered from 4 because 4 concurrent +# hung 5-min contexts on the shared per-service Chromium OOM-killed +# the subprocess and cascaded "browser closed" across the rest of +# that service's features — see drivers/d6-all-pills.ts), estimated +# runtime ~10-14 min. # # -- timeout_ms: 600_000 (10 min) -- # -- max_concurrency: 8 -- # -# 8 services x 4 features = 32 concurrent contexts on 10 browsers. +# 8 services x 2 features = 16 concurrent contexts on 10 browsers. kind: e2e_d6 id: d6-all-pills-e2e schedule: "40 * * * *" diff --git a/showcase/harness/src/probes/drivers/d6-all-pills.test.ts b/showcase/harness/src/probes/drivers/d6-all-pills.test.ts index 998d4c3a81a..e72d7d16006 100644 --- a/showcase/harness/src/probes/drivers/d6-all-pills.test.ts +++ b/showcase/harness/src/probes/drivers/d6-all-pills.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach } from "vitest"; +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import { createE2eFullDriver, createPooledE2eFullLauncher, @@ -137,8 +137,8 @@ describe("e2e-full driver", () => { expect(typeof createPooledE2eFullLauncher).toBe("function"); }); - it("exports FEATURE_CONCURRENCY_D6 = 4", () => { - expect(FEATURE_CONCURRENCY_D6).toBe(4); + it("exports FEATURE_CONCURRENCY_D6 = 2 (lowered from 4 to reduce hung-context memory pressure)", () => { + expect(FEATURE_CONCURRENCY_D6).toBe(2); }); it("exports e2eFullDriver default instance", () => { @@ -530,6 +530,164 @@ describe("e2e-full driver", () => { }); }); + // -------------------------------------------------------------------- + // Between-features browser recycle (cascade isolation). + // + // A single hung feature (e.g. missing aimock fixture loops to the + // 5-min `featureTimeoutMs`) can accumulate enough hung-context + // memory to OOM-kill the shared per-service Chromium subprocess. + // Once dead, every subsequent `newContext()` on the same handle + // throws "Target page, context or browser has been closed" and + // wipes the rest of the service's ~40 features with that same + // error — destroying per-feature D6 signal. + // + // Fix: the per-service feature loop checks `browser.isConnected()` + // before each feature and recycles (close + relaunch) when dead. + // It also recycles proactively after a `feature-timeout` result so + // the next feature on this worker does not inherit the pressure. + // -------------------------------------------------------------------- + describe("between-features browser recycle (cascade isolation)", () => { + function makeDeadAfterNCallsBrowser( + script: PageScript, + callsBeforeDeath: number, + ): E2eFullBrowser & { connected: boolean; newContextCalls: number } { + // Tracks newContext calls. Returns alive contexts for the first + // `callsBeforeDeath` calls and flips `connected` to false + // immediately after the Nth call returns — mimicking a + // Chromium that survives long enough to hand out one context + // but is OOM-killed shortly after (e.g. the contended hung- + // context memory pressure pattern). Subsequent newContext + // calls throw the same error Playwright surfaces when the + // underlying chromium has been killed. + const b = { + connected: true, + newContextCalls: 0, + async newContext() { + b.newContextCalls++; + if (b.newContextCalls > callsBeforeDeath) { + b.connected = false; + throw new Error("Target page, context or browser has been closed"); + } + const ctx = makeContext({ pageScript: script }); + // Flip to dead immediately AFTER this returned context is + // wired up. The next isConnected() check (between + // features) will observe the dead handle. + if (b.newContextCalls >= callsBeforeDeath) { + b.connected = false; + } + return ctx; + }, + async close() { + b.connected = false; + }, + isConnected() { + return b.connected; + }, + }; + return b; + } + + // The driver's per-service feature concurrency is env-overridable + // via FEATURE_CONCURRENCY_D6; force it to 1 here so the + // recycle-between-features semantics is exercised + // deterministically (worker 1 finishes before worker 2 starts). + // Without this the two workers would run in parallel against the + // SAME browser and both grab newContext before the dead check + // fires, masking the recycle path. + beforeEach(() => { + vi.stubEnv("FEATURE_CONCURRENCY_D6", "1"); + }); + afterEach(() => { + vi.unstubAllEnvs(); + }); + + it("swaps to a fresh browser when the current handle is disconnected before next feature", async () => { + registerD5Script(makeScript(["agentic-chat"])); + registerD5Script(makeScript(["tool-rendering"])); + + const sideEmits: ProbeResult[] = []; + const writer: ProbeResultWriter = { + write: async (r) => { + sideEmits.push(r); + }, + }; + + // First browser handles exactly 1 newContext call, then "dies" + // (mimics OOM-killed Chromium). Second browser is healthy. The + // launcher hands them out in sequence; the driver's + // between-features check must trigger the second launcher call. + const browsers: ReturnType[] = [ + makeDeadAfterNCallsBrowser({}, 1), + makeDeadAfterNCallsBrowser({}, 100), + ]; + let launcherCalls = 0; + const driver = createE2eFullDriver({ + launcher: async () => { + const b = browsers[launcherCalls++]; + if (!b) throw new Error("launcher called more times than expected"); + return b; + }, + scriptLoader: noopScriptLoader(), + featureTimeoutMs: 5_000, + }); + + await driver.run(makeCtx({ writer }), { + key: "d6-all-pills-e2e:showcase-test-slug", + backendUrl: "https://test.example.com", + features: ["agentic-chat", "tool-rendering"], + }); + + // The recycle must have triggered at least one extra launcher + // call so the second feature got a live browser. + expect(launcherCalls).toBeGreaterThanOrEqual(2); + + // Critical assertion: the second feature got newContext off + // the FRESH browser, not the dead one. + expect(browsers[1]!.newContextCalls).toBeGreaterThanOrEqual(1); + + // Aggregate row exists (driver completed without bailing). + const aggRow = sideEmits.find((r) => r.key === "d6:test-slug"); + expect(aggRow).toBeDefined(); + }); + + it("caps recycle attempts per service to avoid infinite loops on a poisoned pool", async () => { + // Every launcher returns a browser that dies on first + // newContext. Without a cap the driver would recycle forever + // and never make progress. The cap is + // MAX_BROWSER_RECYCLES_PER_SERVICE (5) — total launcher calls + // = 1 initial + up to 5 recycles = 6 max. + registerD5Script(makeScript(["agentic-chat"])); + registerD5Script(makeScript(["tool-rendering"])); + registerD5Script(makeScript(["shared-state-read"])); + + const writer: ProbeResultWriter = { + write: async () => {}, + }; + + let launcherCalls = 0; + const driver = createE2eFullDriver({ + launcher: async () => { + launcherCalls++; + return makeDeadAfterNCallsBrowser({}, 0); + }, + scriptLoader: noopScriptLoader(), + featureTimeoutMs: 5_000, + }); + + await driver.run(makeCtx({ writer }), { + key: "d6-all-pills-e2e:showcase-test-slug", + backendUrl: "https://test.example.com", + features: ["agentic-chat", "tool-rendering", "shared-state-read"], + }); + + // Initial acquire + at most MAX_BROWSER_RECYCLES_PER_SERVICE + // recycles. Anything significantly beyond that indicates the + // cap is broken (e.g. 10+ would mean per-feature unlimited + // recycle). + expect(launcherCalls).toBeLessThanOrEqual(10); + }); + }); + // -------------------------------------------------------------------- // Defensive re-acquire on disconnected browser. The pool's own // acquire() skips zombies whose `disconnected` event has already diff --git a/showcase/harness/src/probes/drivers/d6-all-pills.ts b/showcase/harness/src/probes/drivers/d6-all-pills.ts index 91ebb629c82..fed930c1846 100644 --- a/showcase/harness/src/probes/drivers/d6-all-pills.ts +++ b/showcase/harness/src/probes/drivers/d6-all-pills.ts @@ -179,6 +179,16 @@ export interface E2eFullBrowser { extraHTTPHeaders?: Record; }): Promise; close(): Promise; + /** + * Optional liveness probe. Pooled launchers and the default Playwright + * launcher both expose this; tests/fakes may omit it (treated as alive + * by callers via `?? true`). Used by the driver's between-features + * cascade-isolation check: a single hung feature can OOM-kill the + * shared chromium subprocess, after which every subsequent + * `newContext()` on the dead handle throws "browser closed" — recycle + * the pooled handle BEFORE the next feature acquires its context. + */ + isConnected?(): boolean; } export type E2eFullBrowserLauncher = ( @@ -204,10 +214,48 @@ const DEFAULT_PAGE_TIMEOUT_MS = 30 * 1000; const DEFAULT_FEATURE_TIMEOUT_MS = 5 * 60 * 1000; /** - * D6 runs 4 features concurrently (vs D5's 2). Higher parallelism - * because D6 is the full matrix and needs to complete within budget. + * D6 per-service feature concurrency. + * + * Was 4 (matching D6's full-matrix wall-clock budget). Lowered to 2 + * because 4 concurrent features each holding a 5-min hung context (the + * worst case: a feature whose aimock fixture is missing loops to the + * `featureTimeoutMs` wall-clock) accumulate enough loaded-page + + * SSE/DOM state to OOM-kill the shared Chromium subprocess. Once + * Chromium dies, every remaining `newContext()` on that handle fails + * with "Target page, context or browser has been closed" and wipes the + * rest of the service's ~40 features. Halving the in-flight context + * count roughly halves the simultaneous hung-context memory pressure; + * wall-clock per service rises modestly but per-feature signal quality + * (the actual ask of D6) matters more here. + * + * Operators can override via `FEATURE_CONCURRENCY_D6` env (see + * `resolveFeatureConcurrencyD6`). */ -export const FEATURE_CONCURRENCY_D6 = 4; +export const FEATURE_CONCURRENCY_D6 = 2; + +/** + * Cap on between-features browser-recycle attempts per service. A + * single hung-feature timeout is the expected case (one recycle). + * Repeated recycles within one service indicate the pool itself is + * unhealthy — bail out rather than spin recycling forever. + */ +export const MAX_BROWSER_RECYCLES_PER_SERVICE = 5; + +/** + * Resolve the effective feature concurrency for a D6 service run. + * Reads `FEATURE_CONCURRENCY_D6` from the supplied env map (defaults to + * `process.env`) and falls back to the `FEATURE_CONCURRENCY_D6` + * constant when unset/invalid. Clamped to >= 1. + */ +export function resolveFeatureConcurrencyD6( + env: NodeJS.ProcessEnv = process.env, +): number { + const raw = env.FEATURE_CONCURRENCY_D6; + if (raw === undefined || raw === "") return FEATURE_CONCURRENCY_D6; + const parsed = parseInt(raw, 10); + if (!Number.isFinite(parsed) || parsed < 1) return FEATURE_CONCURRENCY_D6; + return parsed; +} /** * Inline counting semaphore — gates concurrent access to a bounded @@ -325,6 +373,7 @@ const defaultLauncher: E2eFullBrowserLauncher = }; }, close: () => browser.close(), + isConnected: () => browser.isConnected(), }; }; @@ -462,6 +511,7 @@ export function createPooledE2eFullLauncher( if (forceReleased) return; pool.release(browser); }, + isConnected: () => browser.isConnected(), }; }; } @@ -844,9 +894,86 @@ export function createE2eFullDriver( return aggregateResult; } - // Run features with bounded parallelism. - const sem = new Semaphore(FEATURE_CONCURRENCY_D6); - const browserRef: E2eFullBrowser = browser!; + // Run features with bounded parallelism. Concurrency is + // env-overridable (FEATURE_CONCURRENCY_D6) — see resolver + // comment near the constant. + const featureConcurrency = resolveFeatureConcurrencyD6(); + const sem = new Semaphore(featureConcurrency); + + // Shared, mutable handle to the current pooled browser. A + // single hung feature (e.g. missing aimock fixture loops to + // `featureTimeoutMs`) can accumulate enough hung-context + // memory to OOM-kill the shared Chromium subprocess; once + // dead, every subsequent `newContext()` on the same handle + // throws "Target page, context or browser has been closed" + // and wipes the rest of the service's features. The + // between-features check below detects the dead handle and + // single-flight recycles it (release + relauncher) before + // the next feature acquires its context, isolating the + // cascade to just the feature(s) that hung. + const handle: { current: E2eFullBrowser } = { current: browser! }; + let recycleAttempts = 0; + let recycleInFlight: Promise | null = null; + + const recycleBrowser = async (reason: string): Promise => { + // Single-flight: if a recycle is already running, await it. + // The waiter gets the swapped handle without launching a + // duplicate browser. + if (recycleInFlight) { + await recycleInFlight; + return; + } + if (recycleAttempts >= MAX_BROWSER_RECYCLES_PER_SERVICE) { + ctx.logger.warn("probe.e2e-full.recycle-cap-reached", { + slug, + attempts: recycleAttempts, + cap: MAX_BROWSER_RECYCLES_PER_SERVICE, + reason, + }); + return; + } + recycleAttempts++; + ctx.logger.warn("probe.e2e-full.between-features-recycle", { + slug, + attempt: recycleAttempts, + cap: MAX_BROWSER_RECYCLES_PER_SERVICE, + reason, + }); + recycleInFlight = (async () => { + const stale = handle.current; + try { + await stale.close(); + } catch (err) { + // Stale browser is dead/disconnected — close failure + // is expected and non-fatal; the launcher's release + // path routes the dead instance back to the pool where + // its isConnected check + recycleSlot recovery takes + // over. + ctx.logger.warn("probe.e2e-full.recycle-close-failed", { + slug, + err: err instanceof Error ? err.message : String(err), + }); + } + try { + handle.current = await launcher(abort.signal); + } catch (err) { + // Relaunch failed (pool empty / chromium spawn EAGAIN / + // abort fired). Leave handle.current pointing at the + // stale browser; subsequent features will fail-fast on + // newContext and we won't infinite-loop because of the + // attempts cap above. + ctx.logger.warn("probe.e2e-full.recycle-relaunch-failed", { + slug, + err: err instanceof Error ? err.message : String(err), + }); + } + })(); + try { + await recycleInFlight; + } finally { + recycleInFlight = null; + } + }; const featurePromises = runnable.map(async (ft) => { const sideKey = `d6:${slug}/${ft}`; @@ -894,6 +1021,18 @@ export function createE2eFullDriver( }; } + // Between-features cascade isolation: if the shared + // pooled browser died during a prior feature (typically a + // 5-min hung context that OOM-killed Chromium), recycle + // it BEFORE this feature acquires its context. Without + // this, `runFeature -> browser.newContext()` would throw + // "Target page, context or browser has been closed" and + // every remaining feature in the service would fail with + // that same error. + if (handle.current.isConnected?.() === false) { + await recycleBrowser("dead-handle-pre-feature"); + } + const featureAbort = new AbortController(); const onParentAbort = (): void => featureAbort.abort(); if (abort.signal.aborted) featureAbort.abort(); @@ -917,7 +1056,11 @@ export function createE2eFullDriver( try { return await Promise.race([ runFeature({ - browser: browserRef, + // Read handle.current at call-time, not capture- + // time: a between-features recycle may have + // swapped the live browser since this worker was + // scheduled. + browser: handle.current, url, slug, featureType: ft, @@ -986,6 +1129,21 @@ export function createE2eFullDriver( abort.signal.removeEventListener("abort", onParentAbort); } + // Proactive recycle after a feature-timeout: the hung + // context may still be holding memory inside the shared + // Chromium even after `featureAbort.abort()` (Playwright's + // abort tears down the context asynchronously). Recycling + // here drops the pressure before the next feature on this + // worker starts, and is cheap on the happy path because + // the cap + single-flight guard prevents runaway. + if ( + !featureResult.ok && + featureResult.errorClass === "feature-timeout" && + !abort.signal.aborted + ) { + await recycleBrowser("post-feature-timeout"); + } + if (featureResult.ok) { await sideEmit(ctx, { key: sideKey, From 8839bf354acc2b0275371dd3a53353b7448eac31 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sun, 31 May 2026 16:57:35 -0700 Subject: [PATCH 5/5] Revert "fix(showcase/harness): isolate d6 feature-timeout cascade (#5134)" This reverts commit 470f6f0687e107c5e1eae63bb20260793cff183d, reversing changes made to 6cd630919f59ec4ec555f6fa2b35199d7e121dc8. --- .../config/probes/d6-all-pills-e2e.yml | 8 +- .../src/probes/drivers/d6-all-pills.test.ts | 164 +---------------- .../src/probes/drivers/d6-all-pills.ts | 172 +----------------- 3 files changed, 12 insertions(+), 332 deletions(-) diff --git a/showcase/harness/config/probes/d6-all-pills-e2e.yml b/showcase/harness/config/probes/d6-all-pills-e2e.yml index 3632f0fa982..86f316c3faa 100644 --- a/showcase/harness/config/probes/d6-all-pills-e2e.yml +++ b/showcase/harness/config/probes/d6-all-pills-e2e.yml @@ -13,16 +13,12 @@ # -- Cadence: hourly at :40 -- # # Avoids e2e-demos at :10, D5 at :05/:20/:35/:50. With BROWSER_POOL_SIZE=10 -# and FEATURE_CONCURRENCY_D6=2 (lowered from 4 because 4 concurrent -# hung 5-min contexts on the shared per-service Chromium OOM-killed -# the subprocess and cascaded "browser closed" across the rest of -# that service's features — see drivers/d6-all-pills.ts), estimated -# runtime ~10-14 min. +# and FEATURE_CONCURRENCY=4, estimated runtime ~7-10 min. # # -- timeout_ms: 600_000 (10 min) -- # -- max_concurrency: 8 -- # -# 8 services x 2 features = 16 concurrent contexts on 10 browsers. +# 8 services x 4 features = 32 concurrent contexts on 10 browsers. kind: e2e_d6 id: d6-all-pills-e2e schedule: "40 * * * *" diff --git a/showcase/harness/src/probes/drivers/d6-all-pills.test.ts b/showcase/harness/src/probes/drivers/d6-all-pills.test.ts index e72d7d16006..998d4c3a81a 100644 --- a/showcase/harness/src/probes/drivers/d6-all-pills.test.ts +++ b/showcase/harness/src/probes/drivers/d6-all-pills.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; +import { describe, it, expect, beforeEach } from "vitest"; import { createE2eFullDriver, createPooledE2eFullLauncher, @@ -137,8 +137,8 @@ describe("e2e-full driver", () => { expect(typeof createPooledE2eFullLauncher).toBe("function"); }); - it("exports FEATURE_CONCURRENCY_D6 = 2 (lowered from 4 to reduce hung-context memory pressure)", () => { - expect(FEATURE_CONCURRENCY_D6).toBe(2); + it("exports FEATURE_CONCURRENCY_D6 = 4", () => { + expect(FEATURE_CONCURRENCY_D6).toBe(4); }); it("exports e2eFullDriver default instance", () => { @@ -530,164 +530,6 @@ describe("e2e-full driver", () => { }); }); - // -------------------------------------------------------------------- - // Between-features browser recycle (cascade isolation). - // - // A single hung feature (e.g. missing aimock fixture loops to the - // 5-min `featureTimeoutMs`) can accumulate enough hung-context - // memory to OOM-kill the shared per-service Chromium subprocess. - // Once dead, every subsequent `newContext()` on the same handle - // throws "Target page, context or browser has been closed" and - // wipes the rest of the service's ~40 features with that same - // error — destroying per-feature D6 signal. - // - // Fix: the per-service feature loop checks `browser.isConnected()` - // before each feature and recycles (close + relaunch) when dead. - // It also recycles proactively after a `feature-timeout` result so - // the next feature on this worker does not inherit the pressure. - // -------------------------------------------------------------------- - describe("between-features browser recycle (cascade isolation)", () => { - function makeDeadAfterNCallsBrowser( - script: PageScript, - callsBeforeDeath: number, - ): E2eFullBrowser & { connected: boolean; newContextCalls: number } { - // Tracks newContext calls. Returns alive contexts for the first - // `callsBeforeDeath` calls and flips `connected` to false - // immediately after the Nth call returns — mimicking a - // Chromium that survives long enough to hand out one context - // but is OOM-killed shortly after (e.g. the contended hung- - // context memory pressure pattern). Subsequent newContext - // calls throw the same error Playwright surfaces when the - // underlying chromium has been killed. - const b = { - connected: true, - newContextCalls: 0, - async newContext() { - b.newContextCalls++; - if (b.newContextCalls > callsBeforeDeath) { - b.connected = false; - throw new Error("Target page, context or browser has been closed"); - } - const ctx = makeContext({ pageScript: script }); - // Flip to dead immediately AFTER this returned context is - // wired up. The next isConnected() check (between - // features) will observe the dead handle. - if (b.newContextCalls >= callsBeforeDeath) { - b.connected = false; - } - return ctx; - }, - async close() { - b.connected = false; - }, - isConnected() { - return b.connected; - }, - }; - return b; - } - - // The driver's per-service feature concurrency is env-overridable - // via FEATURE_CONCURRENCY_D6; force it to 1 here so the - // recycle-between-features semantics is exercised - // deterministically (worker 1 finishes before worker 2 starts). - // Without this the two workers would run in parallel against the - // SAME browser and both grab newContext before the dead check - // fires, masking the recycle path. - beforeEach(() => { - vi.stubEnv("FEATURE_CONCURRENCY_D6", "1"); - }); - afterEach(() => { - vi.unstubAllEnvs(); - }); - - it("swaps to a fresh browser when the current handle is disconnected before next feature", async () => { - registerD5Script(makeScript(["agentic-chat"])); - registerD5Script(makeScript(["tool-rendering"])); - - const sideEmits: ProbeResult[] = []; - const writer: ProbeResultWriter = { - write: async (r) => { - sideEmits.push(r); - }, - }; - - // First browser handles exactly 1 newContext call, then "dies" - // (mimics OOM-killed Chromium). Second browser is healthy. The - // launcher hands them out in sequence; the driver's - // between-features check must trigger the second launcher call. - const browsers: ReturnType[] = [ - makeDeadAfterNCallsBrowser({}, 1), - makeDeadAfterNCallsBrowser({}, 100), - ]; - let launcherCalls = 0; - const driver = createE2eFullDriver({ - launcher: async () => { - const b = browsers[launcherCalls++]; - if (!b) throw new Error("launcher called more times than expected"); - return b; - }, - scriptLoader: noopScriptLoader(), - featureTimeoutMs: 5_000, - }); - - await driver.run(makeCtx({ writer }), { - key: "d6-all-pills-e2e:showcase-test-slug", - backendUrl: "https://test.example.com", - features: ["agentic-chat", "tool-rendering"], - }); - - // The recycle must have triggered at least one extra launcher - // call so the second feature got a live browser. - expect(launcherCalls).toBeGreaterThanOrEqual(2); - - // Critical assertion: the second feature got newContext off - // the FRESH browser, not the dead one. - expect(browsers[1]!.newContextCalls).toBeGreaterThanOrEqual(1); - - // Aggregate row exists (driver completed without bailing). - const aggRow = sideEmits.find((r) => r.key === "d6:test-slug"); - expect(aggRow).toBeDefined(); - }); - - it("caps recycle attempts per service to avoid infinite loops on a poisoned pool", async () => { - // Every launcher returns a browser that dies on first - // newContext. Without a cap the driver would recycle forever - // and never make progress. The cap is - // MAX_BROWSER_RECYCLES_PER_SERVICE (5) — total launcher calls - // = 1 initial + up to 5 recycles = 6 max. - registerD5Script(makeScript(["agentic-chat"])); - registerD5Script(makeScript(["tool-rendering"])); - registerD5Script(makeScript(["shared-state-read"])); - - const writer: ProbeResultWriter = { - write: async () => {}, - }; - - let launcherCalls = 0; - const driver = createE2eFullDriver({ - launcher: async () => { - launcherCalls++; - return makeDeadAfterNCallsBrowser({}, 0); - }, - scriptLoader: noopScriptLoader(), - featureTimeoutMs: 5_000, - }); - - await driver.run(makeCtx({ writer }), { - key: "d6-all-pills-e2e:showcase-test-slug", - backendUrl: "https://test.example.com", - features: ["agentic-chat", "tool-rendering", "shared-state-read"], - }); - - // Initial acquire + at most MAX_BROWSER_RECYCLES_PER_SERVICE - // recycles. Anything significantly beyond that indicates the - // cap is broken (e.g. 10+ would mean per-feature unlimited - // recycle). - expect(launcherCalls).toBeLessThanOrEqual(10); - }); - }); - // -------------------------------------------------------------------- // Defensive re-acquire on disconnected browser. The pool's own // acquire() skips zombies whose `disconnected` event has already diff --git a/showcase/harness/src/probes/drivers/d6-all-pills.ts b/showcase/harness/src/probes/drivers/d6-all-pills.ts index fed930c1846..91ebb629c82 100644 --- a/showcase/harness/src/probes/drivers/d6-all-pills.ts +++ b/showcase/harness/src/probes/drivers/d6-all-pills.ts @@ -179,16 +179,6 @@ export interface E2eFullBrowser { extraHTTPHeaders?: Record; }): Promise; close(): Promise; - /** - * Optional liveness probe. Pooled launchers and the default Playwright - * launcher both expose this; tests/fakes may omit it (treated as alive - * by callers via `?? true`). Used by the driver's between-features - * cascade-isolation check: a single hung feature can OOM-kill the - * shared chromium subprocess, after which every subsequent - * `newContext()` on the dead handle throws "browser closed" — recycle - * the pooled handle BEFORE the next feature acquires its context. - */ - isConnected?(): boolean; } export type E2eFullBrowserLauncher = ( @@ -214,48 +204,10 @@ const DEFAULT_PAGE_TIMEOUT_MS = 30 * 1000; const DEFAULT_FEATURE_TIMEOUT_MS = 5 * 60 * 1000; /** - * D6 per-service feature concurrency. - * - * Was 4 (matching D6's full-matrix wall-clock budget). Lowered to 2 - * because 4 concurrent features each holding a 5-min hung context (the - * worst case: a feature whose aimock fixture is missing loops to the - * `featureTimeoutMs` wall-clock) accumulate enough loaded-page + - * SSE/DOM state to OOM-kill the shared Chromium subprocess. Once - * Chromium dies, every remaining `newContext()` on that handle fails - * with "Target page, context or browser has been closed" and wipes the - * rest of the service's ~40 features. Halving the in-flight context - * count roughly halves the simultaneous hung-context memory pressure; - * wall-clock per service rises modestly but per-feature signal quality - * (the actual ask of D6) matters more here. - * - * Operators can override via `FEATURE_CONCURRENCY_D6` env (see - * `resolveFeatureConcurrencyD6`). + * D6 runs 4 features concurrently (vs D5's 2). Higher parallelism + * because D6 is the full matrix and needs to complete within budget. */ -export const FEATURE_CONCURRENCY_D6 = 2; - -/** - * Cap on between-features browser-recycle attempts per service. A - * single hung-feature timeout is the expected case (one recycle). - * Repeated recycles within one service indicate the pool itself is - * unhealthy — bail out rather than spin recycling forever. - */ -export const MAX_BROWSER_RECYCLES_PER_SERVICE = 5; - -/** - * Resolve the effective feature concurrency for a D6 service run. - * Reads `FEATURE_CONCURRENCY_D6` from the supplied env map (defaults to - * `process.env`) and falls back to the `FEATURE_CONCURRENCY_D6` - * constant when unset/invalid. Clamped to >= 1. - */ -export function resolveFeatureConcurrencyD6( - env: NodeJS.ProcessEnv = process.env, -): number { - const raw = env.FEATURE_CONCURRENCY_D6; - if (raw === undefined || raw === "") return FEATURE_CONCURRENCY_D6; - const parsed = parseInt(raw, 10); - if (!Number.isFinite(parsed) || parsed < 1) return FEATURE_CONCURRENCY_D6; - return parsed; -} +export const FEATURE_CONCURRENCY_D6 = 4; /** * Inline counting semaphore — gates concurrent access to a bounded @@ -373,7 +325,6 @@ const defaultLauncher: E2eFullBrowserLauncher = }; }, close: () => browser.close(), - isConnected: () => browser.isConnected(), }; }; @@ -511,7 +462,6 @@ export function createPooledE2eFullLauncher( if (forceReleased) return; pool.release(browser); }, - isConnected: () => browser.isConnected(), }; }; } @@ -894,86 +844,9 @@ export function createE2eFullDriver( return aggregateResult; } - // Run features with bounded parallelism. Concurrency is - // env-overridable (FEATURE_CONCURRENCY_D6) — see resolver - // comment near the constant. - const featureConcurrency = resolveFeatureConcurrencyD6(); - const sem = new Semaphore(featureConcurrency); - - // Shared, mutable handle to the current pooled browser. A - // single hung feature (e.g. missing aimock fixture loops to - // `featureTimeoutMs`) can accumulate enough hung-context - // memory to OOM-kill the shared Chromium subprocess; once - // dead, every subsequent `newContext()` on the same handle - // throws "Target page, context or browser has been closed" - // and wipes the rest of the service's features. The - // between-features check below detects the dead handle and - // single-flight recycles it (release + relauncher) before - // the next feature acquires its context, isolating the - // cascade to just the feature(s) that hung. - const handle: { current: E2eFullBrowser } = { current: browser! }; - let recycleAttempts = 0; - let recycleInFlight: Promise | null = null; - - const recycleBrowser = async (reason: string): Promise => { - // Single-flight: if a recycle is already running, await it. - // The waiter gets the swapped handle without launching a - // duplicate browser. - if (recycleInFlight) { - await recycleInFlight; - return; - } - if (recycleAttempts >= MAX_BROWSER_RECYCLES_PER_SERVICE) { - ctx.logger.warn("probe.e2e-full.recycle-cap-reached", { - slug, - attempts: recycleAttempts, - cap: MAX_BROWSER_RECYCLES_PER_SERVICE, - reason, - }); - return; - } - recycleAttempts++; - ctx.logger.warn("probe.e2e-full.between-features-recycle", { - slug, - attempt: recycleAttempts, - cap: MAX_BROWSER_RECYCLES_PER_SERVICE, - reason, - }); - recycleInFlight = (async () => { - const stale = handle.current; - try { - await stale.close(); - } catch (err) { - // Stale browser is dead/disconnected — close failure - // is expected and non-fatal; the launcher's release - // path routes the dead instance back to the pool where - // its isConnected check + recycleSlot recovery takes - // over. - ctx.logger.warn("probe.e2e-full.recycle-close-failed", { - slug, - err: err instanceof Error ? err.message : String(err), - }); - } - try { - handle.current = await launcher(abort.signal); - } catch (err) { - // Relaunch failed (pool empty / chromium spawn EAGAIN / - // abort fired). Leave handle.current pointing at the - // stale browser; subsequent features will fail-fast on - // newContext and we won't infinite-loop because of the - // attempts cap above. - ctx.logger.warn("probe.e2e-full.recycle-relaunch-failed", { - slug, - err: err instanceof Error ? err.message : String(err), - }); - } - })(); - try { - await recycleInFlight; - } finally { - recycleInFlight = null; - } - }; + // Run features with bounded parallelism. + const sem = new Semaphore(FEATURE_CONCURRENCY_D6); + const browserRef: E2eFullBrowser = browser!; const featurePromises = runnable.map(async (ft) => { const sideKey = `d6:${slug}/${ft}`; @@ -1021,18 +894,6 @@ export function createE2eFullDriver( }; } - // Between-features cascade isolation: if the shared - // pooled browser died during a prior feature (typically a - // 5-min hung context that OOM-killed Chromium), recycle - // it BEFORE this feature acquires its context. Without - // this, `runFeature -> browser.newContext()` would throw - // "Target page, context or browser has been closed" and - // every remaining feature in the service would fail with - // that same error. - if (handle.current.isConnected?.() === false) { - await recycleBrowser("dead-handle-pre-feature"); - } - const featureAbort = new AbortController(); const onParentAbort = (): void => featureAbort.abort(); if (abort.signal.aborted) featureAbort.abort(); @@ -1056,11 +917,7 @@ export function createE2eFullDriver( try { return await Promise.race([ runFeature({ - // Read handle.current at call-time, not capture- - // time: a between-features recycle may have - // swapped the live browser since this worker was - // scheduled. - browser: handle.current, + browser: browserRef, url, slug, featureType: ft, @@ -1129,21 +986,6 @@ export function createE2eFullDriver( abort.signal.removeEventListener("abort", onParentAbort); } - // Proactive recycle after a feature-timeout: the hung - // context may still be holding memory inside the shared - // Chromium even after `featureAbort.abort()` (Playwright's - // abort tears down the context asynchronously). Recycling - // here drops the pressure before the next feature on this - // worker starts, and is cheap on the happy path because - // the cap + single-flight guard prevents runaway. - if ( - !featureResult.ok && - featureResult.errorClass === "feature-timeout" && - !abort.signal.aborted - ) { - await recycleBrowser("post-feature-timeout"); - } - if (featureResult.ok) { await sideEmit(ctx, { key: sideKey,