[pull] main from CopilotKit:main

.github/workflows/publish-release.yml

@@ -463,7 +463,7 @@ jobs:
 
       - name: Install Poetry
         if: steps.detect.outputs.should_publish == 'true'
-        uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1
+        uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1.4.1
         with:
           version: latest
           virtualenvs-create: true

.github/workflows/test_e2e-dojo.yml

@@ -238,7 +238,7 @@ jobs:
 
       - name: Install Poetry
         if: steps.should-run.outputs.skip != 'true' && matrix.needs_python
-        uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1
+        uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1.4.1
         with:
           version: latest
           virtualenvs-create: true

.github/workflows/test_unit-python-sdk.yml

@@ -43,7 +43,7 @@ jobs:
           python-version: ${{ matrix.python-version }}
 
       - name: Install Poetry
-        uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1
+        uses: snok/install-poetry@76e04a911780d5b312d89783f7b1cd627778900a # v1.4.1
         with:
           version: latest
           virtualenvs-create: true

showcase/harness/src/probes/drivers/d6-all-pills.test.ts

@@ -20,6 +20,8 @@ import {
 } from "../helpers/d5-registry.js";
 import type { D5Script } from "../helpers/d5-registry.js";
 import { logger } from "../../logger.js";
+import type { Browser } from "playwright";
+import type { BrowserPool } from "../helpers/browser-pool.js";
 import type {
   ProbeContext,
   ProbeResult,
@@ -527,4 +529,114 @@ describe("e2e-full driver", () => {
       expect(() => sem.release()).toThrow();
     });
   });
+
+  // --------------------------------------------------------------------
+  // Defensive re-acquire on disconnected browser. The pool's own
+  // acquire() skips zombies whose `disconnected` event has already
+  // fired, but a browser can also die in the narrow window AFTER
+  // acquire() returns it but BEFORE the caller hands it to
+  // `browser.newContext()` — most commonly during the D6 service
+  // fan-out's Chromium-spawn burst when fork() returns EAGAIN. The
+  // launcher must release-and-re-acquire so the entire service's
+  // ~40 features don't all fail with "Target page, context or
+  // browser has been closed".
+  // --------------------------------------------------------------------
+  describe("createPooledE2eFullLauncher", () => {
+    it("releases and re-acquires when acquire() returns a disconnected browser", async () => {
+      const dead = makeFakeBrowserish(false);
+      const fresh = makeFakeBrowserish(true);
+      const acquired: FakeBrowserish[] = [];
+      const released: FakeBrowserish[] = [];
+      let nextIdx = 0;
+      const queue = [dead, fresh];
+      const fakePool = {
+        async acquire() {
+          const b = queue[nextIdx++]!;
+          acquired.push(b);
+          return b as unknown as Browser;
+        },
+        release(b: unknown) {
+          released.push(b as FakeBrowserish);
+        },
+        stats() {
+          return { size: 1, available: 0, inUse: 1, totalRecycles: 0 };
+        },
+      };
+      const warnings: Array<{ event: string; meta?: unknown }> = [];
+      const warnLogger = {
+        warn: (event: string, meta?: Record<string, unknown>) =>
+          warnings.push({ event, meta }),
+      };
+      const launcher = createPooledE2eFullLauncher(
+        fakePool as unknown as BrowserPool,
+        warnLogger,
+      );
+      const browser = await launcher();
+      // Must have acquired twice (dead, then fresh) and released the
+      // dead instance back to the pool exactly once.
+      expect(acquired).toHaveLength(2);
+      expect(acquired[0]).toBe(dead);
+      expect(acquired[1]).toBe(fresh);
+      expect(released).toHaveLength(1);
+      expect(released[0]).toBe(dead);
+      // The launcher must return a usable E2eFullBrowser; smoke-test
+      // newContext() to confirm we got back the FRESH instance.
+      await browser.newContext();
+      // Diagnostic warn surfaced so operators can correlate.
+      expect(warnings.map((w) => w.event)).toContain(
+        "probe.e2e-full.pool-acquire-dead",
+      );
+    });
+
+    it("does not re-acquire when the first acquire returns a healthy browser", async () => {
+      const healthy = makeFakeBrowserish(true);
+      const acquired: FakeBrowserish[] = [];
+      const released: FakeBrowserish[] = [];
+      const fakePool = {
+        async acquire() {
+          acquired.push(healthy);
+          return healthy as unknown as Browser;
+        },
+        release(b: unknown) {
+          released.push(b as FakeBrowserish);
+        },
+        stats() {
+          return { size: 1, available: 0, inUse: 1, totalRecycles: 0 };
+        },
+      };
+      const launcher = createPooledE2eFullLauncher(
+        fakePool as unknown as BrowserPool,
+      );
+      await launcher();
+      expect(acquired).toHaveLength(1);
+      expect(released).toHaveLength(0);
+    });
+  });
 });
+
+// Module-scoped helpers for the createPooledE2eFullLauncher tests above —
+// lifted out of the describe so oxlint's consistent-function-scoping is
+// satisfied (the factory captures no parent state).
+interface FakeBrowserish {
+  connected: boolean;
+  newContext(): Promise<{
+    close(): Promise<void>;
+    newPage(): Promise<unknown>;
+  }>;
+  isConnected(): boolean;
+}
+
+function makeFakeBrowserish(connected: boolean): FakeBrowserish {
+  return {
+    connected,
+    isConnected() {
+      return this.connected;
+    },
+    async newContext() {
+      return {
+        close: async () => {},
+        newPage: async () => ({}),
+      };
+    },
+  };
+}

showcase/harness/src/probes/drivers/d6-all-pills.ts

@@ -333,7 +333,27 @@ export function createPooledE2eFullLauncher(
   logger?: { warn(event: string, meta?: Record<string, unknown>): void },
 ): E2eFullBrowserLauncher {
   return async (abortSignal?: AbortSignal): Promise<E2eFullBrowser> => {
-    const browser = await pool.acquire();
+    let browser = await pool.acquire();
+
+    // Defensive re-acquire: the pool's acquire() skips zombie slots whose
+    // browser is already disconnected, but a browser can also die in the
+    // narrow window AFTER acquire() returns it but BEFORE we hand it to
+    // `browser.newContext()` — most commonly during the D6 service fan-out's
+    // Chromium-spawn burst, when the kernel returns EAGAIN on fork and a
+    // recently-handed-out chromium dies between this caller and its first
+    // newContext call. Without this guard, the dead browser dooms an entire
+    // service's ~40 features ("Target page, context or browser has been
+    // closed" on every single feature). One re-acquire is enough — release
+    // routes the dead instance back into the pool where its `isConnected`
+    // check + `recycleSlot` recovery take over.
+    if (!browser.isConnected()) {
+      logger?.warn("probe.e2e-full.pool-acquire-dead", {
+        poolAvailable: pool.stats().available,
+        poolInUse: pool.stats().inUse,
+      });
+      pool.release(browser);
+      browser = await pool.acquire();
+    }
 
     let forceReleased = false;
     const openContexts = new Set<{ close(): Promise<void> }>();

showcase/harness/src/probes/loader/probe-invoker.test.ts

@@ -3706,4 +3706,119 @@ describe("buildProbeInvoker", () => {
     expect(persisted.total).toBe(persisted.passed + persisted.failed);
     expect(persisted.failed).toBeGreaterThanOrEqual(1);
   });
+
+  // ----------------------------------------------------------------------
+  // Service-startup stagger — protects against the d6 Chromium-spawn
+  // thundering herd that crashes the BrowserPool when many services fan
+  // out into pool.acquire() in the same JS tick. The stagger spaces
+  // worker-zero..N-1's FIRST pull over `(N-1)*staggerMs`, so per-Chromium
+  // thread-spawn bursts don't trip the container's PID/thread ceiling.
+  // ----------------------------------------------------------------------
+  it("staggers worker first-pull start times across the fan-out", async () => {
+    const inputSchema = z.object({ key: z.string() });
+    const startTimes: number[] = [];
+    // Each driver run hangs long enough that every worker's FIRST pull
+    // happens before any of them finish — that's the regime where the
+    // stagger actually matters (it spaces the per-service Chromium
+    // launches, not the steady-state cursor draining).
+    const HOLD_MS = 500;
+    const driver: ProbeDriver<z.infer<typeof inputSchema>, { ok: true }> = {
+      kind: "smoke",
+      inputSchema,
+      async run(ctx, input) {
+        startTimes.push(Date.now());
+        await new Promise((r) => setTimeout(r, HOLD_MS));
+        return {
+          key: input.key,
+          state: "green",
+          signal: { ok: true },
+          observedAt: ctx.now().toISOString(),
+        };
+      },
+    };
+    const targets = Array.from({ length: 4 }, (_, i) => ({
+      key: `smoke:t${i}`,
+    }));
+    const cfg: ProbeConfig = {
+      kind: "smoke",
+      id: "smoke-stagger",
+      schedule: "*/15 * * * *",
+      max_concurrency: 4,
+      targets,
+    };
+    const { writer, writes } = mkWriter();
+    const STAGGER = 50;
+    const invoker = buildProbeInvoker(cfg, {
+      driver,
+      schedulerId: cfg.id,
+      discoveryRegistry: createDiscoveryRegistry(),
+      writer,
+      ...BASE_DEPS,
+      // Override the default 300ms to keep the test fast; uses real timers
+      // so a small positive value is enough to assert ordering.
+      env: { SERVICE_STARTUP_STAGGER_MS: String(STAGGER) } as Readonly<
+        Record<string, string | undefined>
+      >,
+    });
+    await invoker();
+    expect(writes).toHaveLength(4);
+    expect(startTimes).toHaveLength(4);
+    // Worker 0 starts immediately; workers 1..N-1 each delayed by their
+    // index * stagger. Assert monotonic non-decreasing start times and a
+    // spread between first and last that is at least (N-1) * stagger
+    // less a small jitter margin. Use a generous lower bound so this is
+    // not flaky under CI scheduling jitter.
+    for (let i = 1; i < startTimes.length; i++) {
+      expect(startTimes[i]!).toBeGreaterThanOrEqual(startTimes[i - 1]!);
+    }
+    const totalSpread = startTimes[3]! - startTimes[0]!;
+    // (4 workers - 1) * 50ms = 150ms expected spread; allow 60% floor for
+    // CI scheduling jitter on slow runners.
+    expect(totalSpread).toBeGreaterThanOrEqual(Math.floor(3 * STAGGER * 0.6));
+  });
+
+  it("disables stagger when SERVICE_STARTUP_STAGGER_MS=0", async () => {
+    const inputSchema = z.object({ key: z.string() });
+    const startTimes: number[] = [];
+    const driver: ProbeDriver<z.infer<typeof inputSchema>, { ok: true }> = {
+      kind: "smoke",
+      inputSchema,
+      async run(ctx, input) {
+        startTimes.push(Date.now());
+        return {
+          key: input.key,
+          state: "green",
+          signal: { ok: true },
+          observedAt: ctx.now().toISOString(),
+        };
+      },
+    };
+    const targets = Array.from({ length: 4 }, (_, i) => ({
+      key: `smoke:t${i}`,
+    }));
+    const cfg: ProbeConfig = {
+      kind: "smoke",
+      id: "smoke-no-stagger",
+      schedule: "*/15 * * * *",
+      max_concurrency: 4,
+      targets,
+    };
+    const { writer } = mkWriter();
+    const invoker = buildProbeInvoker(cfg, {
+      driver,
+      schedulerId: cfg.id,
+      discoveryRegistry: createDiscoveryRegistry(),
+      writer,
+      ...BASE_DEPS,
+      env: { SERVICE_STARTUP_STAGGER_MS: "0" } as Readonly<
+        Record<string, string | undefined>
+      >,
+    });
+    await invoker();
+    // With stagger=0 every worker pulls its first input in the same
+    // microtask flush — the spread is bounded by JS scheduling latency
+    // for sibling promises, well under any meaningful stagger value.
+    const totalSpread = startTimes[3]! - startTimes[0]!;
+    expect(totalSpread).toBeLessThan(50);
+  });
 });