From fc4a8d0e66c5e93af8cf39a8318f57580a88884d Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sat, 27 Jun 2026 22:06:14 -0700 Subject: [PATCH 1/2] fix(video): embed testId in Veo/Grok poll ids for multi-tenant isolation (#278) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Veo and Grok async video submit store the job keyed `${testId}:${bareId}` (testId from getTestId(req)) but return an OPAQUE bare id — `operations/{uuid}` for Veo, `{uuid}` for Grok — with no tenant scope. A multi-tenant client that polls the returned id WITHOUT an x-test-id header resolves to DEFAULT_TEST_ID and 404s for any non-default testId. The established OpenRouter provider avoids this by embedding the testId in the returned polling_url via testIdSuffix. Mirror that here: append testIdSuffix(testId, "?") to the id RETURNED to the client at all four submit sites (Veo replay + record, Grok replay + record), while the job stays STORED under the bare id. On poll, route matching uses parsedUrl.pathname (query excluded), so the regex captures the bare id, and getTestId picks `?testId=` off the query — reconstructing the same key. Record mode is untouched: the upstream poll uses the bare job id / upstreamPollingUrl. testIdSuffix returns "" for DEFAULT_TEST_ID, so default-tenant returned ids stay byte-identical, preserving existing fixtures and tests. Adds multi-tenant isolation tests to veo-video.test.ts and grok-video.test.ts mirroring openrouter-video.test.ts: a non-default tenant polls its returned id (testId embedded) without the header and resolves; a bare id polled under another tenant 404s; and a default-tenant submit returns the bare id (no suffix). --- src/__tests__/grok-video.test.ts | 84 +++++++++++++++++++++++++++++ src/__tests__/veo-video.test.ts | 90 ++++++++++++++++++++++++++++++++ src/grok-video.ts | 13 +++-- src/veo-video.ts | 13 +++-- 4 files changed, 194 insertions(+), 6 deletions(-) diff --git a/src/__tests__/grok-video.test.ts b/src/__tests__/grok-video.test.ts index 32cd35e..67ce25d 100644 --- a/src/__tests__/grok-video.test.ts +++ b/src/__tests__/grok-video.test.ts @@ -363,6 +363,90 @@ describe("GET /v1/videos/{id} dispatch journals handler errors as 500 (Grok-firs }); }); +// ─── Multi-tenant poll isolation (#278) ────────────────────────────────────── +// Submit stores the job keyed `${testId}:${requestId}`, but the returned +// request_id is opaque. A multi-tenant client polls the returned id WITHOUT an +// x-test-id header, so the testId must travel in the returned id (via +// testIdSuffix) for getTestId's `?testId=` fallback to resolve the scope — +// mirroring OpenRouter's polling_url treatment. + +describe("Grok video — testId scoping of returned request_id", () => { + let mock: LLMock | undefined; + + afterEach(async () => { + await mock?.stop(); + mock = undefined; + }); + + test("returned request_id carries testId and resolves the poll without the header", async () => { + mock = new LLMock({ port: 0 }); + mock.addFixture({ + match: { userMessage: "scoped poll", endpoint: "video" }, + response: { video: { id: "vid_scoped", status: "completed", url: "https://cdn.x.ai/s.mp4" } }, + }); + await mock.start(); + + const submit = await fetch(`${mock.url}/v1/videos/generations`, { + method: "POST", + headers: { "Content-Type": "application/json", "X-Test-Id": "test-a" }, + body: JSON.stringify({ model: "grok-imagine-video", prompt: "scoped poll" }), + }); + const { request_id } = (await submit.json()) as { request_id: string }; + expect(request_id).toContain("testId=test-a"); + + // Bare poll of the returned id — no X-Test-Id header — must still resolve + // via the embedded query parameter. + const poll = await fetch(`${mock.url}/v1/videos/${request_id}`); + expect(poll.status).toBe(200); + const data = await poll.json(); + expect(data.status).toBe("done"); + }); + + test("job submitted under one testId is invisible to another", async () => { + mock = new LLMock({ port: 0 }); + mock.addFixture({ + match: { userMessage: "isolated", endpoint: "video" }, + response: { video: { id: "vid_iso", status: "completed", url: "https://cdn.x.ai/i.mp4" } }, + }); + await mock.start(); + + const submit = await fetch(`${mock.url}/v1/videos/generations`, { + method: "POST", + headers: { "Content-Type": "application/json", "X-Test-Id": "test-a" }, + body: JSON.stringify({ model: "grok-imagine-video", prompt: "isolated" }), + }); + const { request_id } = (await submit.json()) as { request_id: string }; + + // Strip the testId suffix to recover the bare request_id, then poll it under + // a DIFFERENT tenant. Grok misses fall through to Sora status, which 404s an + // unknown id — so a cross-tenant poll must NOT return a Grok "done" body. + const bareId = request_id.split("?")[0]; + const crossPoll = await fetch(`${mock.url}/v1/videos/${bareId}`, { + headers: { "X-Test-Id": "test-b" }, + }); + expect(crossPoll.status).toBe(404); + }); + + test("default testId returns the bare request_id (no testId param)", async () => { + mock = new LLMock({ port: 0 }); + mock.addFixture({ + match: { userMessage: "clean id", endpoint: "video" }, + response: { video: { id: "vid_clean", status: "completed", url: "https://cdn.x.ai/c.mp4" } }, + }); + await mock.start(); + + const submit = await fetch(`${mock.url}/v1/videos/generations`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model: "grok-imagine-video", prompt: "clean id" }), + }); + const { request_id } = (await submit.json()) as { request_id: string }; + // Byte-identical to pre-fix behaviour: bare uuid, no query. + expect(request_id).not.toContain("testId="); + expect(request_id).not.toContain("?"); + }); +}); + // Type-only smoke: the stored video status union remains the 3-value set // (no "done"); the Grok wire "done" is derived at serialization. describe("VideoResponse stored status", () => { diff --git a/src/__tests__/veo-video.test.ts b/src/__tests__/veo-video.test.ts index dde561b..1445854 100644 --- a/src/__tests__/veo-video.test.ts +++ b/src/__tests__/veo-video.test.ts @@ -278,3 +278,93 @@ describe("GET /v1beta/operations/{name} dispatch journals handler errors as 500 expect(failed[0].method).toBe("GET"); }); }); + +// ─── Multi-tenant poll isolation (#278) ────────────────────────────────────── +// Submit stores the job keyed `${testId}:${operationName}`, but the returned +// operation name is opaque. A multi-tenant client polls the returned name +// WITHOUT an x-test-id header, so the testId must travel in the returned name +// (via testIdSuffix) for getTestId's `?testId=` fallback to resolve the scope — +// mirroring OpenRouter's polling_url treatment. + +describe("Veo video — testId scoping of returned operation name", () => { + let mock: LLMock | undefined; + + afterEach(async () => { + await mock?.stop(); + mock = undefined; + }); + + test("returned name carries testId and resolves the poll without the header", async () => { + mock = new LLMock({ port: 0 }); + mock.addFixture({ + match: { userMessage: "scoped poll", endpoint: "video" }, + response: { + video: { id: "veo_scoped", status: "completed", url: "https://files.example/s.mp4" }, + }, + }); + await mock.start(); + + const submit = await fetch(submitUrl(mock.url), { + method: "POST", + headers: { "Content-Type": "application/json", "X-Test-Id": "test-a" }, + body: JSON.stringify({ instances: [{ prompt: "scoped poll" }] }), + }); + const { name } = (await submit.json()) as { name: string }; + expect(name).toContain("testId=test-a"); + + // Bare poll of the returned name — no X-Test-Id header — must still resolve + // via the embedded query parameter. + const poll = await fetch(`${mock.url}/v1beta/${name}`); + expect(poll.status).toBe(200); + const data = await poll.json(); + expect(data.done).toBe(true); + }); + + test("job submitted under one testId is invisible to another", async () => { + mock = new LLMock({ port: 0 }); + mock.addFixture({ + match: { userMessage: "isolated", endpoint: "video" }, + response: { + video: { id: "veo_iso", status: "completed", url: "https://files.example/i.mp4" }, + }, + }); + await mock.start(); + + const submit = await fetch(submitUrl(mock.url), { + method: "POST", + headers: { "Content-Type": "application/json", "X-Test-Id": "test-a" }, + body: JSON.stringify({ instances: [{ prompt: "isolated" }] }), + }); + const { name } = (await submit.json()) as { name: string }; + + // Strip the testId suffix to recover the bare operation name, then poll it + // under a DIFFERENT tenant — must 404 (cross-tenant leak otherwise). + const bareName = name.split("?")[0]; + const crossPoll = await fetch(`${mock.url}/v1beta/${bareName}`, { + headers: { "X-Test-Id": "test-b" }, + }); + expect(crossPoll.status).toBe(404); + }); + + test("default testId returns the bare operation name (no testId param)", async () => { + mock = new LLMock({ port: 0 }); + mock.addFixture({ + match: { userMessage: "clean name", endpoint: "video" }, + response: { + video: { id: "veo_clean", status: "completed", url: "https://files.example/c.mp4" }, + }, + }); + await mock.start(); + + const submit = await fetch(submitUrl(mock.url), { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ instances: [{ prompt: "clean name" }] }), + }); + const { name } = (await submit.json()) as { name: string }; + // Byte-identical to pre-fix behaviour: bare `operations/{uuid}`, no query. + expect(name).not.toContain("testId="); + expect(name).not.toContain("?"); + expect(name.startsWith("operations/")).toBe(true); + }); +}); diff --git a/src/grok-video.ts b/src/grok-video.ts index 3439df6..1494954 100644 --- a/src/grok-video.ts +++ b/src/grok-video.ts @@ -33,7 +33,7 @@ import { } from "./recorder.js"; import { resolveUpstreamUrl } from "./url.js"; import { handleVideoStatus, type VideoStateMap } from "./video.js"; -import { readEnvelopeText, upstreamTimeoutSignal } from "./video-proxy-shared.js"; +import { readEnvelopeText, testIdSuffix, upstreamTimeoutSignal } from "./video-proxy-shared.js"; /** * xAI Grok Imagine async video lifecycle mock. Submit @@ -555,8 +555,12 @@ export async function handleGrokVideoCreate( ); } + // Embed the testId in the RETURNED request_id (not the stored key) so a + // multi-tenant client can poll the opaque id without an x-test-id header — the + // testId travels via getTestId's `?testId=` fallback. Mirrors OpenRouter's + // polling_url treatment; testIdSuffix is "" for the default testId (bare id). res.writeHead(200, { "Content-Type": "application/json" }); - res.end(JSON.stringify({ request_id: requestId })); + res.end(JSON.stringify({ request_id: requestId + testIdSuffix(testId, "?") })); } // ─── GET /v1/videos/{id} — status poll (Grok-first, Sora fall-through) ─────── @@ -872,8 +876,11 @@ async function proxyGrokVideoSubmit(args: { }, }); + // Embed the testId in the RETURNED request_id (see the replay submit) so + // record-mode multi-tenant polls resolve via `?testId=`; the upstream poll + // still uses the bare job.requestId / job.upstreamPollingUrl, untouched. res.writeHead(200, { "Content-Type": "application/json" }); - res.end(JSON.stringify({ request_id: requestId })); + res.end(JSON.stringify({ request_id: requestId + testIdSuffix(testId, "?") })); return "handled"; } diff --git a/src/veo-video.ts b/src/veo-video.ts index ce42533..2555e55 100644 --- a/src/veo-video.ts +++ b/src/veo-video.ts @@ -32,7 +32,7 @@ import { sanitizeHeaderValue, } from "./recorder.js"; import { resolveUpstreamUrl } from "./url.js"; -import { readEnvelopeText, upstreamTimeoutSignal } from "./video-proxy-shared.js"; +import { readEnvelopeText, testIdSuffix, upstreamTimeoutSignal } from "./video-proxy-shared.js"; /** * Google Veo async video lifecycle mock. Submit @@ -461,8 +461,12 @@ export async function handleVeoVideoCreate( ); } + // Embed the testId in the RETURNED name (not the stored key) so a multi-tenant + // client can poll the opaque name without an x-test-id header — the testId + // travels via getTestId's `?testId=` fallback. Mirrors OpenRouter's polling_url + // treatment; testIdSuffix is "" for the default testId, keeping the name bare. res.writeHead(200, { "Content-Type": "application/json" }); - res.end(JSON.stringify({ name: operationName })); + res.end(JSON.stringify({ name: operationName + testIdSuffix(testId, "?") })); } // ─── GET /v1beta/operations/{name} — status poll ───────────────────────────── @@ -801,8 +805,11 @@ async function proxyVeoVideoSubmit(args: { }, }); + // Embed the testId in the RETURNED name (see the replay submit) so record-mode + // multi-tenant polls resolve via `?testId=`; the upstream poll still uses the + // bare job.operationName / job.upstreamPollingUrl, untouched by the suffix. res.writeHead(200, { "Content-Type": "application/json" }); - res.end(JSON.stringify({ name: operationName })); + res.end(JSON.stringify({ name: operationName + testIdSuffix(testId, "?") })); return "handled"; } From 3f584c6ae269dc47512cc30cf6941068628c3c0c Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Sat, 27 Jun 2026 23:30:33 -0700 Subject: [PATCH 2/2] test(video): prove cross-tenant poll routing via header-less testId suffix (#278) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replace the tautological header-stripping isolation tests (which 404'd on both pre- and post-fix code) with a decisive concurrent-tenant test in both veo-video and grok-video suites. Two tenants submit distinct jobs concurrently so both coexist in the job map, then each polls ONLY its own RETURNED suffix-bearing id sending NO x-test-id header. The header-less poll must resolve via the embedded `?testId=` suffix to that tenant's own job (asserting its own video uri/url, not the peer's, not 404). On pre-fix code the returned id is a bare uuid carrying no testId, so a header-less poll falls to the default testId, the key misses, and the poll 404s — the test fails (RED). Also tighten the existing substring suffix check to an exact `?testId=` query-param assertion. --- src/__tests__/grok-video.test.ts | 71 ++++++++++++++++++++++--------- src/__tests__/veo-video.test.ts | 73 ++++++++++++++++++++++++-------- 2 files changed, 108 insertions(+), 36 deletions(-) diff --git a/src/__tests__/grok-video.test.ts b/src/__tests__/grok-video.test.ts index 67ce25d..e3093d0 100644 --- a/src/__tests__/grok-video.test.ts +++ b/src/__tests__/grok-video.test.ts @@ -392,7 +392,8 @@ describe("Grok video — testId scoping of returned request_id", () => { body: JSON.stringify({ model: "grok-imagine-video", prompt: "scoped poll" }), }); const { request_id } = (await submit.json()) as { request_id: string }; - expect(request_id).toContain("testId=test-a"); + // Exact query param, not just a substring: `{uuid}?testId=test-a`. + expect(request_id).toMatch(/^[^?]+\?testId=test-a$/); // Bare poll of the returned id — no X-Test-Id header — must still resolve // via the embedded query parameter. @@ -402,29 +403,61 @@ describe("Grok video — testId scoping of returned request_id", () => { expect(data.status).toBe("done"); }); - test("job submitted under one testId is invisible to another", async () => { + // DECISIVE multi-tenant routing (#278). Two tenants submit DISTINCT jobs + // concurrently — both live in the map at once — then each polls ONLY its own + // RETURNED suffix-bearing request_id, sending NO x-test-id header. The + // header-less poll must resolve via the embedded `?testId=` suffix to that + // tenant's own job (not the other tenant's, not a Sora 404 fall-through). + // This FAILS on pre-fix code: a bare returned request_id carries no testId, so + // a header-less poll falls to the default testId, the `${testId}:${requestId}` + // key misses, and the Grok-first dispatch falls through to Sora → 404. The + // earlier header-stripping "isolation" test was tautological (it 404s pre- AND + // post-fix); this one is the real cross-tenant routing proof. + test("concurrent tenants each resolve their OWN job via the header-less suffix", async () => { mock = new LLMock({ port: 0 }); mock.addFixture({ - match: { userMessage: "isolated", endpoint: "video" }, - response: { video: { id: "vid_iso", status: "completed", url: "https://cdn.x.ai/i.mp4" } }, + match: { userMessage: "tenant A clip", endpoint: "video" }, + response: { video: { id: "vid_A", status: "completed", url: "https://cdn.x.ai/a.mp4" } }, }); - await mock.start(); - - const submit = await fetch(`${mock.url}/v1/videos/generations`, { - method: "POST", - headers: { "Content-Type": "application/json", "X-Test-Id": "test-a" }, - body: JSON.stringify({ model: "grok-imagine-video", prompt: "isolated" }), + mock.addFixture({ + match: { userMessage: "tenant B clip", endpoint: "video" }, + response: { video: { id: "vid_B", status: "completed", url: "https://cdn.x.ai/b.mp4" } }, }); - const { request_id } = (await submit.json()) as { request_id: string }; + await mock.start(); - // Strip the testId suffix to recover the bare request_id, then poll it under - // a DIFFERENT tenant. Grok misses fall through to Sora status, which 404s an - // unknown id — so a cross-tenant poll must NOT return a Grok "done" body. - const bareId = request_id.split("?")[0]; - const crossPoll = await fetch(`${mock.url}/v1/videos/${bareId}`, { - headers: { "X-Test-Id": "test-b" }, - }); - expect(crossPoll.status).toBe(404); + // Both tenants submit concurrently → both jobs coexist in the map. + const [resA, resB] = await Promise.all([ + fetch(`${mock.url}/v1/videos/generations`, { + method: "POST", + headers: { "Content-Type": "application/json", "X-Test-Id": "tenant-a" }, + body: JSON.stringify({ model: "grok-imagine-video", prompt: "tenant A clip" }), + }), + fetch(`${mock.url}/v1/videos/generations`, { + method: "POST", + headers: { "Content-Type": "application/json", "X-Test-Id": "tenant-b" }, + body: JSON.stringify({ model: "grok-imagine-video", prompt: "tenant B clip" }), + }), + ]); + const { request_id: idA } = (await resA.json()) as { request_id: string }; + const { request_id: idB } = (await resB.json()) as { request_id: string }; + expect(idA).toMatch(/^[^?]+\?testId=tenant-a$/); + expect(idB).toMatch(/^[^?]+\?testId=tenant-b$/); + // Distinct underlying request ids (no accidental collision). + expect(idA.split("?")[0]).not.toBe(idB.split("?")[0]); + + // Header-less poll of A's returned id → resolves A's job (a.mp4), not B's. + const pollA = await fetch(`${mock.url}/v1/videos/${idA}`); + expect(pollA.status).toBe(200); + const dataA = await pollA.json(); + expect(dataA.status).toBe("done"); + expect(dataA.video.url).toBe("https://cdn.x.ai/a.mp4"); + + // Header-less poll of B's returned id → resolves B's job (b.mp4), not A's. + const pollB = await fetch(`${mock.url}/v1/videos/${idB}`); + expect(pollB.status).toBe(200); + const dataB = await pollB.json(); + expect(dataB.status).toBe("done"); + expect(dataB.video.url).toBe("https://cdn.x.ai/b.mp4"); }); test("default testId returns the bare request_id (no testId param)", async () => { diff --git a/src/__tests__/veo-video.test.ts b/src/__tests__/veo-video.test.ts index 1445854..3277758 100644 --- a/src/__tests__/veo-video.test.ts +++ b/src/__tests__/veo-video.test.ts @@ -310,7 +310,8 @@ describe("Veo video — testId scoping of returned operation name", () => { body: JSON.stringify({ instances: [{ prompt: "scoped poll" }] }), }); const { name } = (await submit.json()) as { name: string }; - expect(name).toContain("testId=test-a"); + // Exact query param, not just a substring: `operations/{uuid}?testId=test-a`. + expect(name).toMatch(/^operations\/[^?]+\?testId=test-a$/); // Bare poll of the returned name — no X-Test-Id header — must still resolve // via the embedded query parameter. @@ -320,30 +321,68 @@ describe("Veo video — testId scoping of returned operation name", () => { expect(data.done).toBe(true); }); - test("job submitted under one testId is invisible to another", async () => { + // DECISIVE multi-tenant routing (#278). Two tenants submit DISTINCT jobs + // concurrently — both live in the map at once — then each polls ONLY its own + // RETURNED suffix-bearing id, sending NO x-test-id header. The header-less + // poll must resolve via the embedded `?testId=` suffix to that tenant's own + // job (not the other tenant's, not 404). This FAILS on pre-fix code: a bare + // returned name carries no testId, so a header-less poll falls to the default + // testId and the `${testId}:${operationName}` key misses → 404. The earlier + // header-stripping "isolation" test was tautological (it 404s pre- AND + // post-fix); this one is the real cross-tenant routing proof. + test("concurrent tenants each resolve their OWN job via the header-less suffix", async () => { mock = new LLMock({ port: 0 }); mock.addFixture({ - match: { userMessage: "isolated", endpoint: "video" }, + match: { userMessage: "tenant A clip", endpoint: "video" }, response: { - video: { id: "veo_iso", status: "completed", url: "https://files.example/i.mp4" }, + video: { id: "veo_A", status: "completed", url: "https://files.example/a.mp4" }, + }, + }); + mock.addFixture({ + match: { userMessage: "tenant B clip", endpoint: "video" }, + response: { + video: { id: "veo_B", status: "completed", url: "https://files.example/b.mp4" }, }, }); await mock.start(); - const submit = await fetch(submitUrl(mock.url), { - method: "POST", - headers: { "Content-Type": "application/json", "X-Test-Id": "test-a" }, - body: JSON.stringify({ instances: [{ prompt: "isolated" }] }), - }); - const { name } = (await submit.json()) as { name: string }; + // Both tenants submit concurrently → both jobs coexist in the map. + const [resA, resB] = await Promise.all([ + fetch(submitUrl(mock.url), { + method: "POST", + headers: { "Content-Type": "application/json", "X-Test-Id": "tenant-a" }, + body: JSON.stringify({ instances: [{ prompt: "tenant A clip" }] }), + }), + fetch(submitUrl(mock.url), { + method: "POST", + headers: { "Content-Type": "application/json", "X-Test-Id": "tenant-b" }, + body: JSON.stringify({ instances: [{ prompt: "tenant B clip" }] }), + }), + ]); + const { name: nameA } = (await resA.json()) as { name: string }; + const { name: nameB } = (await resB.json()) as { name: string }; + expect(nameA).toMatch(/^operations\/[^?]+\?testId=tenant-a$/); + expect(nameB).toMatch(/^operations\/[^?]+\?testId=tenant-b$/); + // Distinct underlying operation names (no accidental collision). + expect(nameA.split("?")[0]).not.toBe(nameB.split("?")[0]); + + // Header-less poll of A's returned name → resolves A's job (a.mp4), not B's. + const pollA = await fetch(`${mock.url}/v1beta/${nameA}`); + expect(pollA.status).toBe(200); + const dataA = await pollA.json(); + expect(dataA.done).toBe(true); + expect(dataA.response.generateVideoResponse.generatedSamples[0].video.uri).toBe( + "https://files.example/a.mp4", + ); - // Strip the testId suffix to recover the bare operation name, then poll it - // under a DIFFERENT tenant — must 404 (cross-tenant leak otherwise). - const bareName = name.split("?")[0]; - const crossPoll = await fetch(`${mock.url}/v1beta/${bareName}`, { - headers: { "X-Test-Id": "test-b" }, - }); - expect(crossPoll.status).toBe(404); + // Header-less poll of B's returned name → resolves B's job (b.mp4), not A's. + const pollB = await fetch(`${mock.url}/v1beta/${nameB}`); + expect(pollB.status).toBe(200); + const dataB = await pollB.json(); + expect(dataB.done).toBe(true); + expect(dataB.response.generateVideoResponse.generatedSamples[0].video.uri).toBe( + "https://files.example/b.mp4", + ); }); test("default testId returns the bare operation name (no testId param)", async () => {