diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 7ed79d85..a3571ced 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "source": { "source": "npm", "package": "@copilotkit/aimock", - "version": "^1.35.0" + "version": "^1.35.1" }, "description": "Fixture authoring skill for @copilotkit/aimock — LLM, multimedia (image/TTS/transcription/video), MCP, A2A, AG-UI, vector, embeddings, structured output, sequential responses, streaming physics, record/replay, agent loop patterns, and debugging" } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 96a75c9d..0c719560 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "aimock", - "version": "1.35.0", + "version": "1.35.1", "description": "Fixture authoring guidance for @copilotkit/aimock — LLM, multimedia, MCP, A2A, AG-UI, vector, and service mocking", "author": { "name": "CopilotKit" diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index e392e17e..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: 2 - -updates: - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "daily" - groups: - minor-and-patch: - patterns: - - "*" - update-types: - - "minor" - - "patch" - # Workaround for dependabot/dependabot-core#14202: without an explicit - # major group, major updates matching the minor-and-patch pattern are - # silently suppressed. Remove this group when #14202 is fixed to get - # individual (ungrouped) PRs per major bump instead. - major: - patterns: - - "*" - update-types: - - "major" - labels: - - "dependencies" - - "github-actions" - commit-message: - prefix: "ci" - include: "scope" - open-pull-requests-limit: 10 - cooldown: - default-days: 1 diff --git a/.github/workflows/dependabot-auto-merge.yml b/.github/workflows/dependabot-auto-merge.yml deleted file mode 100644 index b7a6e555..00000000 --- a/.github/workflows/dependabot-auto-merge.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Dependabot Auto-Merge (Minor/Patch) - -on: - pull_request_target: - types: [opened, synchronize] - -permissions: - contents: write - pull-requests: write - -jobs: - auto-merge: - runs-on: ubuntu-latest - if: github.event.pull_request.user.login == 'dependabot[bot]' - steps: - - name: Fetch Dependabot metadata - id: metadata - uses: dependabot/fetch-metadata@25dd0e34f4fe68f24cc83900b1fe3fe149efef98 # v3.1.0 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Auto-approve and merge minor/patch github-actions updates - if: >- - steps.metadata.outputs.package-ecosystem == 'github_actions' && - (steps.metadata.outputs.update-type == 'version-update:semver-minor' || - steps.metadata.outputs.update-type == 'version-update:semver-patch') - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_URL: ${{ github.event.pull_request.html_url }} - run: | - gh pr review "$PR_URL" --approve - gh pr merge "$PR_URL" --auto --merge diff --git a/.github/workflows/dependabot-major-analysis.yml b/.github/workflows/dependabot-major-analysis.yml deleted file mode 100644 index c51ce55c..00000000 --- a/.github/workflows/dependabot-major-analysis.yml +++ /dev/null @@ -1,144 +0,0 @@ -name: Dependabot Major Version Analysis - -on: - pull_request_target: - types: [opened] - -permissions: - contents: read - pull-requests: write - -jobs: - analyze-major: - runs-on: ubuntu-latest - if: github.event.pull_request.user.login == 'dependabot[bot]' - steps: - - name: Fetch Dependabot metadata - id: metadata - uses: dependabot/fetch-metadata@25dd0e34f4fe68f24cc83900b1fe3fe149efef98 # v3.1.0 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Analyze major version bump - if: >- - steps.metadata.outputs.package-ecosystem == 'github_actions' && - steps.metadata.outputs.update-type == 'version-update:semver-major' - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - DEP_NAME: ${{ steps.metadata.outputs.dependency-names }} - PREV_VERSION: ${{ steps.metadata.outputs.previous-version }} - NEW_VERSION: ${{ steps.metadata.outputs.new-version }} - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const depName = process.env.DEP_NAME; - const prevVersion = process.env.PREV_VERSION; - const newVersion = process.env.NEW_VERSION; - const parts = depName.split('/'); - const owner = parts[0]; - const repo = parts[1]; - const repoSlug = `${owner}/${repo}`; - - let releases = []; - try { - const { data } = await github.rest.repos.listReleases({ owner, repo, per_page: 50 }); - releases = data; - } catch (err) { - core.warning(`Could not fetch releases for ${repoSlug}: ${err.message}`); - } - - const prevMajor = parseInt(prevVersion.replace(/^v/, ''), 10); - const newMajor = parseInt(newVersion.replace(/^v/, ''), 10); - - const relevantReleases = releases.filter(r => { - const major = parseInt(r.tag_name.replace(/^v/, ''), 10); - return major > prevMajor && major <= newMajor; - }); - - let releaseNotesSummary = ''; - let breakingChanges = ''; - - if (relevantReleases.length === 0) { - releaseNotesSummary = '_No releases found between these versions._'; - breakingChanges = `_Unable to determine breaking changes automatically. Please review the [full changelog](https://github.com/${repoSlug}/releases)._`; - } else { - for (const release of relevantReleases.slice(0, 10)) { - const body = (release.body || '_No release notes._').replace(/(?<=^|\s)@(?=[a-zA-Z0-9])(?![a-zA-Z0-9-]*\/)/gm, ''); - releaseNotesSummary += `### ${release.tag_name}${release.name && release.name !== release.tag_name ? ' — ' + release.name : ''}\n\n`; - releaseNotesSummary += body.substring(0, 2000); - if (body.length > 2000) releaseNotesSummary += '\n\n_...truncated_'; - releaseNotesSummary += '\n\n---\n\n'; - const lines = body.split('\n'); - for (const line of lines) { - if (/breaking|BREAKING|removed|deprecated|incompatible|migration/i.test(line)) { - breakingChanges += `- ${line.trim()}\n`; - } - } - } - } - - if (!breakingChanges) { - breakingChanges = '_No explicit breaking changes detected in release notes. Manual review recommended._'; - } - - let commentBody = `## :warning: Major Version Update — Manual Review Required - - | Field | Value | - |-------|-------| - | **Action** | [\`${depName}\`](https://github.com/${repoSlug}) | - | **Previous** | \`v${prevVersion}\` | - | **New** | \`v${newVersion}\` | - | **Type** | Major (\`v${prevMajor}\` → \`v${newMajor}\`) | - - ### Breaking Changes - - ${breakingChanges} - - ### Release Notes (v${prevMajor + 1} → v${newMajor}) - - ${releaseNotesSummary} - - ### Next Steps - - 1. Review breaking changes above - 2. Check if workflow inputs/outputs changed - 3. Verify compatibility with your CI/CD configuration - - > Full changelog: https://github.com/${repoSlug}/releases - - --- - _Generated automatically for Dependabot major version PRs._`.replace(/^ /gm, ''); - - if (commentBody.length > 64000) { - commentBody = commentBody.substring(0, 63900) + '\n\n_...comment truncated due to size limit._'; - } - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.pull_request.number, - body: commentBody, - }); - - try { - const labelsToAdd = ['major-update', 'needs-review']; - for (const label of labelsToAdd) { - try { - await github.rest.issues.getLabel({ owner: context.repo.owner, repo: context.repo.repo, name: label }); - } catch { - const colors = { 'major-update': 'B60205', 'needs-review': 'FBCA04' }; - await github.rest.issues.createLabel({ - owner: context.repo.owner, repo: context.repo.repo, - name: label, color: colors[label] || 'EDEDED', - }); - } - } - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.pull_request.number, - labels: labelsToAdd, - }); - } catch (err) { - core.warning(`Could not add labels: ${err.message}`); - } diff --git a/.github/workflows/fix-drift.yml b/.github/workflows/fix-drift.yml index e5610bfd..a402bc16 100644 --- a/.github/workflows/fix-drift.yml +++ b/.github/workflows/fix-drift.yml @@ -63,6 +63,8 @@ jobs: echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT if [ "$EXIT_CODE" -eq 2 ]; then : # critical drift found, continue + elif [ "$EXIT_CODE" -eq 5 ]; then + : # quarantine: collector encountered unparseable output, continue without aborting elif [ "$EXIT_CODE" -ne 0 ]; then echo "::error::Collector script crashed with exit code $EXIT_CODE" exit $EXIT_CODE diff --git a/.github/workflows/sentinel.yml b/.github/workflows/sentinel.yml new file mode 100644 index 00000000..eb9070d9 --- /dev/null +++ b/.github/workflows/sentinel.yml @@ -0,0 +1,19 @@ +name: Sentinel +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + scan: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: jpr5/sentinel@3001b71ad3fc6998649be5a18e39585479a3ef5b # v1.3.4 + with: + severity: high + fail-on-findings: true diff --git a/.github/workflows/test-drift.yml b/.github/workflows/test-drift.yml index 18b099be..ed190275 100644 --- a/.github/workflows/test-drift.yml +++ b/.github/workflows/test-drift.yml @@ -6,6 +6,11 @@ on: paths: - "src/agui-types.ts" - "src/__tests__/drift/agui-schema.drift.ts" + # PR-scoped live drift leg (drift-live-pr job): trigger when the drift + # scripts, any drift test, or a drift workflow itself changes. + - "scripts/drift-*.ts" + - "src/__tests__/drift/**" + - ".github/workflows/*drift*" workflow_dispatch: # Manual trigger permissions: @@ -44,6 +49,11 @@ jobs: # persisted across every retry). Lets the alert say "confirmed across N # runs". Empty on the common green/transient path. runs: ${{ steps.drift.outputs.drift_runs }} + # Final collector exit code (0 clean/transient, 2 critical drift, 5 + # quarantine, other = crash). Promoted to a job output so the `notify` + # job can classify an exit-5 quarantine distinctly instead of folding it + # into the generic "job failed → HTTP drift" fallback (M5 backstop). + exit_code: ${{ steps.drift.outputs.exit_code }} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: { persist-credentials: false } @@ -54,6 +64,89 @@ jobs: cache: pnpm - run: pnpm install --frozen-lockfile + # Per-provider key-freshness PREFLIGHT. Before spending 15 minutes on the + # full suite, do a cheap GET /models per provider. A dead/expired/revoked + # key returns 401/403 — classify that as `stale-key` and fail the job LOUD + # with a "rotate _API_KEY" message, so a dead key surfaces up + # front instead of as a late, generic streaming failure buried in the + # suite (or worse, a misleading green skip). Classification keys off the + # numeric InfraError.status (401/403), NEVER the prose message. Transient + # statuses (429/5xx/network) are NOT stale keys: the preflight warns and + # continues, since the retry-wrapped collector below handles those. + # (C5.3b, a separate task, adds the exit_code job-output + notify + # plumbing — this step is only the preflight.) + - name: Preflight — provider key freshness + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + run: | + # Written at the checkout root (cwd) so the relative provider import + # below resolves against the repo. ESM relative specifiers key off the + # module's own location, not the process cwd — a file under + # $RUNNER_TEMP would look for src/ beside itself and fail. Cleaned up + # on exit either way. + PREFLIGHT_FILE="$GITHUB_WORKSPACE/.drift-preflight.mts" + trap 'rm -f "$PREFLIGHT_FILE"' EXIT + cat > "$PREFLIGHT_FILE" <<'PREFLIGHT' + import { + InfraError, + listOpenAIModels, + listAnthropicModels, + listGeminiModels, + } from "./src/__tests__/drift/providers.js"; + + const PROVIDERS: { + name: string; + keyEnv: string; + list: (key: string) => Promise; + }[] = [ + { name: "OpenAI", keyEnv: "OPENAI_API_KEY", list: listOpenAIModels }, + { name: "Anthropic", keyEnv: "ANTHROPIC_API_KEY", list: listAnthropicModels }, + { name: "Gemini", keyEnv: "GOOGLE_API_KEY", list: listGeminiModels }, + ]; + + const stale: string[] = []; + + for (const p of PROVIDERS) { + const key = process.env[p.keyEnv]; + if (!key) { + console.error(`::error::${p.keyEnv} is not set — cannot run drift for ${p.name}`); + stale.push(p.keyEnv); + continue; + } + try { + await p.list(key); + console.log(`preflight: ${p.name} key OK`); + } catch (err) { + const status = err instanceof InfraError ? err.status : 0; + if (status === 401 || status === 403) { + // stale-key: the key is dead/expired/revoked. LOUD, classified. + console.error( + `::error title=stale-key::${p.name} preflight got HTTP ${status} — rotate ${p.keyEnv}`, + ); + stale.push(p.keyEnv); + } else { + // Transient (429/5xx/network) at preflight time is NOT a stale + // key — don't block; the retry-wrapped collector handles it. + const msg = err instanceof Error ? err.message : String(err); + console.warn( + `preflight: ${p.name} non-auth error (status ${status}), continuing: ${msg}`, + ); + } + } + } + + if (stale.length > 0) { + console.error( + `::error::Preflight failed (stale-key): rotate ${stale.join(", ")} before drift can run`, + ); + process.exit(1); + } + console.log("preflight: all provider keys fresh"); + PREFLIGHT + npx tsx "$PREFLIGHT_FILE" + # Run the collector behind a "retry before alert" wrapper. A single # critical run can be a transient real-API hiccup (a streaming call # failing mid-flight), NOT a format change — so the wrapper re-runs the @@ -76,6 +169,9 @@ jobs: echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT" if [ "$EXIT_CODE" -eq 2 ]; then : # critical drift persisted across retries, continue + elif [ "$EXIT_CODE" -eq 5 ]; then + echo "::error::Drift collector quarantined unparseable output (exit 5) — manual triage required" + exit "$EXIT_CODE" elif [ "$EXIT_CODE" -ne 0 ]; then echo "::error::Collector script crashed with exit code $EXIT_CODE" exit "$EXIT_CODE" @@ -129,6 +225,7 @@ jobs: AGUI_RESULT: ${{ needs.agui-schema-drift.result }} DRIFT_SUMMARY: ${{ needs.drift.outputs.summary }} DRIFT_RUNS: ${{ needs.drift.outputs.runs }} + DRIFT_EXIT_CODE: ${{ needs.drift.outputs.exit_code }} REPO: ${{ github.repository }} RUN_ID: ${{ github.run_id }} run: | @@ -137,9 +234,17 @@ jobs: HTTP_DRIFT=false AGUI_DRIFT=false INFRA_ERROR=false + QUARANTINE=false - # Determine what happened in each job - if [ "$DRIFT_RESULT" = "failure" ]; then + # Determine what happened in each job. An exit_code of 5 means the + # collector QUARANTINED unparseable output — the drift job concludes + # `failure`, but this is NOT real HTTP API drift: it needs human + # triage, not a "providers changed formats" alert. Classify it as + # quarantine FIRST, before the generic failure→HTTP_DRIFT fallback, + # so an exit-5 quarantine is never misreported as real drift. + if [ "$DRIFT_EXIT_CODE" = "5" ]; then + QUARANTINE=true + elif [ "$DRIFT_RESULT" = "failure" ]; then HTTP_DRIFT=true elif [ "$DRIFT_RESULT" != "success" ] && [ "$DRIFT_RESULT" != "skipped" ]; then INFRA_ERROR=true @@ -175,8 +280,15 @@ jobs: CONFIRMED="${NL}_(confirmed across ${DRIFT_RUNS} runs)_" fi + # Quarantine (collector exit 5): unparseable output was quarantined + # and needs manual triage. Classified distinctly — NOT real HTTP API + # drift — so nobody chases a phantom provider format change. The + # DRIFT_SUMMARY already carries the quarantine detail block (C5.2). + if [ "$QUARANTINE" = "true" ]; then + EMOJI="🔬" + MSG="*Drift collector quarantined unparseable output* in aimock — manual triage required (not confirmed API drift).${DETAIL}${NL}${RUN_URL}" # Both types of drift - if [ "$HTTP_DRIFT" = "true" ] && [ "$AGUI_DRIFT" = "true" ]; then + elif [ "$HTTP_DRIFT" = "true" ] && [ "$AGUI_DRIFT" = "true" ]; then EMOJI="🚨" MSG="*Drift detected* in aimock — HTTP API drift + AG-UI schema drift.${DETAIL}${CONFIRMED}${NL}${RUN_URL}" # HTTP API drift only @@ -204,3 +316,451 @@ jobs: curl -sf -X POST "$SLACK_WEBHOOK" \ -H "Content-Type: application/json" \ -d "$PAYLOAD" + + # PR-scoped live drift leg (D6.3a skeleton). Triggered ONLY on `pull_request` + # — deliberately NOT `pull_request_target` (O3): a `pull_request` run on a + # fork gets a read-only GITHUB_TOKEN and NO repo secrets, so untrusted fork + # code can never exfiltrate the provider keys. The trade-off (a fork PR has + # no live provider keys and so cannot run the live leg) is handled by the + # fork→maintainer-dry-run fallback that D6.3b documents; this skeleton only + # produces the two reports on the trusted (non-fork) path. + # + # The job runs the live drift collector on BOTH the base (`main`) and the + # head (the PR ref), emitting two artifacts — `drift-report-base` and + # `drift-report-head`. For base it first tries to REUSE the most recent + # same-UTC-day, well-formed `main` scheduled report (via + # `isBaseReportReusable`); only when no reusable report exists does it run a + # fresh live base. The delta comparison / block-vs-advisory decision that + # consumes these two reports is intentionally NOT here — that is D6.3b, which + # plugs in at the "D6.3b SEAM" marker below. + drift-live-pr: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + steps: + # HEAD checkout: the PR ref (default checkout behavior on pull_request). + # persist-credentials:false — this leg never pushes; keeps the token off + # disk on the fork path. + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + # Need main's history reachable so the base leg can check it out + # into a sibling worktree without a second full clone. + fetch-depth: 0 + - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24 + cache: pnpm + - run: pnpm install --frozen-lockfile + + # ---- PR preflight — per-provider key triage (graceful degradation) --- + # The PR-scoped live leg must NOT hard-block a drift-code PR merely because + # a provider key is absent or a provider is transiently down — that would + # re-introduce the live-key coupling the whole delta design set out to + # reduce. So classify each provider independently, keying off the numeric + # InfraError.status (401/403), NEVER the prose message: + # + # - NO key configured → SKIP that provider's live leg with a + # NEUTRAL ::notice:: advisory. Not a failure. + # - key present but 401/403 → LOUD `stale-key` ::error:: → block. A + # configured key that is dead is a real + # failure (rotate it) — this is the one case + # that stays red (the InfraError.status + # contract, mirroring the daily preflight). + # - transient (429/5xx/net) → warn + continue; the collector's own retry + # path handles a transient blip. + # + # `live_legs_ran` records whether ANY provider had a usable key. When it is + # `false` (all skipped for no-key — e.g. a fork PR with no secrets), the + # delta gate below is a neutral pass with a "live drift leg skipped — no + # keys" advisory, and the maintainer-dry-run fallback (documented at the + # bottom of this job) applies. + - name: PR preflight — per-provider key triage + id: preflight + run: | + set -uo pipefail + # Written at the checkout root (cwd) so the relative provider import + # resolves against the repo — ESM relative specifiers key off the + # module's own location, not the process cwd. Cleaned up on exit. + PREFLIGHT_FILE="$GITHUB_WORKSPACE/.drift-pr-preflight.mts" + trap 'rm -f "$PREFLIGHT_FILE"' EXIT + cat > "$PREFLIGHT_FILE" <<'PREFLIGHT' + import { appendFileSync } from "node:fs"; + import { + InfraError, + listOpenAIModels, + listAnthropicModels, + listGeminiModels, + } from "./src/__tests__/drift/providers.js"; + + const PROVIDERS: { + name: string; + keyEnv: string; + list: (key: string) => Promise; + }[] = [ + { name: "OpenAI", keyEnv: "OPENAI_API_KEY", list: listOpenAIModels }, + { name: "Anthropic", keyEnv: "ANTHROPIC_API_KEY", list: listAnthropicModels }, + { name: "Gemini", keyEnv: "GOOGLE_API_KEY", list: listGeminiModels }, + ]; + + const stale: string[] = []; + let liveLegs = 0; + + for (const p of PROVIDERS) { + const key = process.env[p.keyEnv]; + if (!key) { + // NO key configured → SKIP this provider's live leg. Neutral + // advisory, NOT a failure (the drift suite's own describe.skipIf + // already drops the provider's tests when the key is absent). + console.log( + `::notice title=drift-live skipped (no key)::${p.name} live drift leg skipped — ${p.keyEnv} not configured`, + ); + continue; + } + try { + await p.list(key); + liveLegs++; + console.log(`preflight: ${p.name} key OK`); + } catch (err) { + const status = err instanceof InfraError ? err.status : 0; + if (status === 401 || status === 403) { + // stale-key: a CONFIGURED key is dead/expired/revoked. LOUD, + // classified, blocking — this stays a real failure. + console.error( + `::error title=stale-key::${p.name} preflight got HTTP ${status} — rotate ${p.keyEnv}`, + ); + stale.push(p.keyEnv); + } else { + // Transient (429/5xx/network) is NOT a stale key — don't block; + // count it as a live leg and let the collector's retry handle it. + liveLegs++; + const msg = err instanceof Error ? err.message : String(err); + console.warn( + `preflight: ${p.name} non-auth error (status ${status}), continuing: ${msg}`, + ); + } + } + } + + const out = process.env.GITHUB_OUTPUT; + if (out) { + appendFileSync(out, `live_legs_ran=${liveLegs > 0 ? "true" : "false"}\n`); + } + + if (stale.length > 0) { + console.error( + `::error::PR preflight failed (stale-key): rotate ${stale.join(", ")} — a configured key is dead`, + ); + process.exit(1); + } + if (liveLegs === 0) { + console.log( + "::notice title=drift-live skipped::no provider keys configured — live drift leg skipped (neutral); maintainer-dry-run fallback applies", + ); + } + console.log(`preflight: ${liveLegs} live provider leg(s) will run`); + PREFLIGHT + npx tsx "$PREFLIGHT_FILE" + + # ---- BASE leg ------------------------------------------------------- + # Prefer reusing the most recent same-UTC-day, well-formed `main` + # scheduled drift report as the delta base — a same-day main baseline + # already captures the day's environmental drift, so reusing it avoids a + # redundant live run and keeps the PR fast. Only when no reusable report + # is available do we fall back to a fresh live base run. + # + # Reusability is decided by `isBaseReportReusable(report, conclusion, + # sameUtcDay)` from scripts/drift-delta.ts (D6.1): non-empty entries + + # known-good collector conclusion + same UTC day. The producing step + # writes `drift-report-base.json` either way, so downstream steps have a + # single stable base path regardless of which path produced it. + - name: Base drift report — reuse same-UTC-day main run, else run live + id: base + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + LIVE_LEGS_RAN: ${{ steps.preflight.outputs.live_legs_ran }} + run: | + set -euo pipefail + # No provider key had a usable credential (fork PR / all keys absent). + # The live legs are skipped — write an empty, well-formed base report so + # the delta gate has a stable file to read, and stop here (neutral). + if [ "${LIVE_LEGS_RAN:-true}" != "true" ]; then + echo "base: live legs skipped (no keys) — writing empty base report" + printf '%s\n' '{"timestamp":"'"$(date -u +%Y-%m-%dT%H:%M:%SZ)"'","entries":[]}' \ + > drift-report-base.json + echo "reused=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + # Attempt reuse: download the most recent successful scheduled + # `main` run's drift-report artifact, then let isBaseReportReusable + # (D6.1) decide if it is a trustworthy same-UTC-day baseline. On any + # failure to obtain a reusable report, fall through to a live run. + REUSED=false + if BASE_RUN_ID=$(gh run list \ + --workflow="Drift Tests" --repo="$REPO" --branch=main \ + --event=schedule --status=success --limit=1 \ + --json databaseId --jq '.[0].databaseId // empty'); then + if [ -n "$BASE_RUN_ID" ]; then + rm -rf ./.base-artifact + if gh run download "$BASE_RUN_ID" --repo="$REPO" \ + --name drift-report --dir ./.base-artifact; then + BASE_CONCLUSION=$(gh run view "$BASE_RUN_ID" --repo="$REPO" \ + --json conclusion --jq '.conclusion // empty' || echo "") + if node --input-type=module -e ' + import { readFileSync } from "node:fs"; + import { isBaseReportReusable } from "./scripts/drift-delta.ts"; + const path = "./.base-artifact/drift-report.json"; + let report = null; + try { report = JSON.parse(readFileSync(path, "utf-8")); } catch {} + const conclusion = report?.conclusion ?? process.env.BASE_CONCLUSION ?? ""; + const genAt = report?.generatedAt ? new Date(report.generatedAt) : null; + const now = new Date(); + const sameUtcDay = + !!genAt && + genAt.getUTCFullYear() === now.getUTCFullYear() && + genAt.getUTCMonth() === now.getUTCMonth() && + genAt.getUTCDate() === now.getUTCDate(); + process.exit(isBaseReportReusable(report, conclusion, sameUtcDay) ? 0 : 1); + ' BASE_CONCLUSION="$BASE_CONCLUSION"; then + cp ./.base-artifact/drift-report.json drift-report-base.json + REUSED=true + echo "base: reusing same-UTC-day main scheduled report (run $BASE_RUN_ID)" + fi + fi + fi + fi + + if [ "$REUSED" != "true" ]; then + echo "base: no reusable same-UTC-day main report — running fresh live base" + # Check main out into a sibling worktree and run the collector + # there, writing the base report back into the workspace root. + git worktree add ../base-main origin/main + cd ../base-main + pnpm install --frozen-lockfile + # The collector exits 2 when it finds critical drift — that is the + # NORMAL "drift present" signal that feeds the delta gate, NOT a step + # failure. The delta gate (not this leg) decides block-vs-advisory, + # so exit 2 here must be tolerated. `set +e`/`set -e` around the run + # captures the exit code as DATA so an exit 2 does not abort under + # the step's `set -euo pipefail`: + # exit 0 → clean, continue. + # exit 2 → drift present, NON-FATAL (report feeds the delta gate). + # exit 5 → quarantine (unparseable output) — genuine fault, FAIL. + # any other non-{0,2} → genuine collector fault, FAIL. + set +e + npx tsx scripts/drift-report-collector.ts \ + --out "$GITHUB_WORKSPACE/drift-report-base.json" + BASE_EXIT=$? + set -e + cd "$GITHUB_WORKSPACE" + if [ "$BASE_EXIT" -eq 5 ]; then + echo "::error::Base collector quarantined unparseable output (exit 5) — manual triage required" + exit "$BASE_EXIT" + fi + if [ "$BASE_EXIT" -ne 0 ] && [ "$BASE_EXIT" -ne 2 ]; then + echo "::error::Base collector faulted (exit $BASE_EXIT) — not a drift signal" + exit "$BASE_EXIT" + fi + fi + echo "reused=$REUSED" >> "$GITHUB_OUTPUT" + + # ---- HEAD leg ------------------------------------------------------- + # Run the collector live against the PR head (the checked-out ref) to + # produce the head report the delta compares against the base. + # + # CRITICAL (this was the wiring bug that hard-failed the required check): + # the collector exits 2 whenever it finds ANY critical drift — that is the + # EXPECTED "drift present" signal on the head leg, NOT a step failure. The + # whole point of the two-report + delta design is that `computeDelta` + # (the delta gate below) decides block-vs-advisory by KEY PRESENCE — a + # both-present critical is advisory, only a new-in-head key blocks. Letting + # the collector's exit 2 fail THIS step meant the delta gate was skipped + # and the check died for the wrong reason (every drift-code PR would + # red-fail regardless of whether its diff introduced the drift). So exit 2 + # here is tolerated; the report file is the signal. Exit 5 (quarantine) and + # any other non-zero are genuine collector faults and DO fail the leg. + # + # When no provider key had a usable credential (fork PR / all keys absent), + # skip the live head run and write an empty, well-formed head report so the + # delta gate has a stable file — it will then neutral-pass with an advisory. + - name: Head drift report — run live on PR ref + env: + LIVE_LEGS_RAN: ${{ steps.preflight.outputs.live_legs_ran }} + run: | + set -uo pipefail + if [ "${LIVE_LEGS_RAN:-true}" != "true" ]; then + echo "head: live legs skipped (no keys) — writing empty head report" + printf '%s\n' '{"timestamp":"'"$(date -u +%Y-%m-%dT%H:%M:%SZ)"'","entries":[]}' \ + > drift-report-head.json + exit 0 + fi + # The collector exits 2 when it finds critical drift — that is the + # NORMAL "drift present" signal that feeds the delta gate below, NOT a + # step failure. GitHub runs `run:` under `bash -e`, so without an + # explicit `set +e` guard the `-e` would abort THIS step the instant + # the collector returns 2 (never reaching the capture below) — that is + # the #297 wiring bug this leg must avoid. So capture the exit code + # with `set +e`/`set -e` and treat it as DATA: + # exit 0 → clean, continue. + # exit 2 → drift present, NON-FATAL (report feeds the delta gate). + # exit 5 → quarantine (unparseable output) — genuine fault, FAIL. + # any other non-{0,2} → genuine collector fault, FAIL. + set +e + npx tsx scripts/drift-report-collector.ts --out drift-report-head.json + HEAD_EXIT=$? + set -e + if [ "$HEAD_EXIT" -eq 5 ]; then + echo "::error::Head collector quarantined unparseable output (exit 5) — manual triage required" + exit "$HEAD_EXIT" + fi + if [ "$HEAD_EXIT" -ne 0 ] && [ "$HEAD_EXIT" -ne 2 ]; then + echo "::error::Head collector faulted (exit $HEAD_EXIT) — not a drift signal" + exit "$HEAD_EXIT" + fi + echo "head: collector exit $HEAD_EXIT (0 clean / 2 drift-present — both non-fatal here)" + + - name: Upload base drift report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: drift-report-base + path: drift-report-base.json + if-no-files-found: warn + retention-days: 30 + + - name: Upload head drift report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: drift-report-head + path: drift-report-head.json + if-no-files-found: warn + retention-days: 30 + + # =================================================================== + # D6.3b — DELTA STEP. Load the two reports produced above and run + # `computeDelta(base, head)` (scripts/drift-delta.ts, D6.1). Routing is + # by KEY PRESENCE only (#292 invariant), NEVER by DriftClass: + # - block[] → keys NEW in head (absent from base): diff-attributable. + # ANY block key fails this required check (exit 1) — a + # new-in-head *critical* blocks, and so does a new-in-head + # advisory-class finding. Class is annotation only. + # - advisory[] → keys in BOTH base and head: pre-existing / world drift. + # Surfaced as a NEUTRAL annotation + job-summary line but + # NEVER fails the check (that is the daily job's concern, + # not the PR's). + # - fixed[] → keys in base but gone in head: informational only. + # The block-only-on-new-in-head decision is the incident-3 / M-1 backstop: + # a PR is blamed strictly for drift its diff introduced. + - name: Delta gate - block new-in-head, advisory-annotate base-present + env: + LIVE_LEGS_RAN: ${{ steps.preflight.outputs.live_legs_ran }} + run: | + set -euo pipefail + # When no provider key had a usable credential (fork PR / all keys + # absent), no live leg ran — there is no diff-attributable drift to + # evaluate. Neutral-pass with a "live drift leg skipped — no keys" + # advisory (the maintainer-dry-run fallback documented below applies), + # rather than blocking a drift-code PR on absent keys. + if [ "${LIVE_LEGS_RAN:-true}" != "true" ]; then + echo "::notice title=drift-live skipped (no keys)::live drift leg skipped — no provider keys configured; delta gate is a neutral pass (maintainer-dry-run fallback applies)" + { + echo "## Drift delta" + echo "" + echo "_Live drift leg skipped — no provider keys configured. Neutral pass; the maintainer-dry-run fallback applies._" + } >> "$GITHUB_STEP_SUMMARY" + echo "Drift gate PASSED (neutral): live legs skipped — no keys." + exit 0 + fi + # Written at the checkout root (cwd) so the relative drift-delta + # import resolves against the repo — same rationale as the preflight + # step (ESM relative specifiers key off the module's own location). + # Quoted heredoc delimiter: no shell expansion inside the script. + DELTA_FILE="$GITHUB_WORKSPACE/.drift-delta-gate.mts" + trap 'rm -f "$DELTA_FILE"' EXIT + cat > "$DELTA_FILE" <<'DELTAGATE' + import { readFileSync, appendFileSync } from "node:fs"; + import { computeDelta } from "./scripts/drift-delta.js"; + import type { DeltaKey } from "./scripts/drift-delta.js"; + + const read = (p: string) => JSON.parse(readFileSync(p, "utf-8")); + const base = read("drift-report-base.json"); + const head = read("drift-report-head.json"); + + const { block, advisory, fixed } = computeDelta(base, head); + const fmt = (k: DeltaKey) => + `${k.provider} ${k.id}${k.class ? ` [${k.class}]` : ""}`; + + const summary: string[] = []; + summary.push("## Drift delta"); + summary.push(""); + summary.push(`- block (new-in-head): ${block.length}`); + summary.push(`- advisory (base-present): ${advisory.length}`); + summary.push(`- fixed (gone in head): ${fixed.length}`); + + // Advisory: pre-existing / environmental drift. Surface it as a + // NEUTRAL annotation (::notice::) and in the summary, but do NOT fail + // the check on it — that drift is the daily main job's concern. + for (const k of advisory) { + console.log(`::notice title=drift-advisory (pre-existing)::${fmt(k)}`); + } + if (advisory.length > 0) { + summary.push(""); + summary.push("### Advisory (pre-existing on main - not blamed on this PR)"); + for (const k of advisory) summary.push(`- ${fmt(k)}`); + } + + // Block: drift the diff INTRODUCED. Each is a loud error annotation; + // a non-empty block list fails the required check. + for (const k of block) { + console.log(`::error title=drift-block (new-in-head)::${fmt(k)}`); + } + if (block.length > 0) { + summary.push(""); + summary.push("### Block (introduced by this diff - required check FAILS)"); + for (const k of block) summary.push(`- ${fmt(k)}`); + } + + const summaryFile = process.env.GITHUB_STEP_SUMMARY; + if (summaryFile) { + appendFileSync(summaryFile, summary.join("\n") + "\n"); + } + + if (block.length > 0) { + console.error( + `::error::Drift gate BLOCKED: ${block.length} new-in-head failure(s) introduced by this diff: ${block.map(fmt).join(", ")}`, + ); + process.exit(1); + } + console.log( + advisory.length > 0 + ? `Drift gate PASSED: ${advisory.length} advisory (pre-existing) finding(s) surfaced, none new-in-head.` + : "Drift gate PASSED: no new-in-head drift.", + ); + DELTAGATE + npx tsx "$DELTA_FILE" + + # ---- fork → maintainer-dry-run fallback -------------------------------- + # This whole `drift-live-pr` job is gated on `pull_request` (NOT + # `pull_request_target`, O3), so a run originating from a FORK receives a + # read-only GITHUB_TOKEN and NO repo secrets — the provider API keys above + # are simply empty. Consequently a fork PR cannot run the live base/head + # collectors and this delta gate never executes meaningfully for it. + # + # Fallback protocol (maintainer dry-run): for a fork PR, a maintainer runs + # the delta gate manually from a trusted checkout — + # gh pr checkout # fetch the fork head + # npx tsx scripts/drift-report-collector.ts --out drift-report-head.json + # # (reuse or run a same-UTC-day main base as drift-report-base.json) + # node --input-type=module -e '' + # — and records the block/advisory conclusion as a PR comment. The + # branch-protection required-check registration for this job is a manual + # orchestrator step outside this diff (the check must be marked required in + # branch protection for the block=exit-1 signal to actually gate merges). + # =================================================================== diff --git a/.github/zizmor.yml b/.github/zizmor.yml index 3d691212..d6379923 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -4,16 +4,6 @@ # Adding a rule here REQUIRES a comment explaining why it is safe. rules: - # ─── artipacked ──────────────────────────────────────────────────────── - artipacked: - ignore: - # Dependabot auto-merge: uses pull_request_target for write token. - # Does NOT checkout PR code. Actor-gated to dependabot[bot]. - - dependabot-auto-merge.yml - # Dependabot major analysis: uses pull_request_target for PR comments. - # Does NOT checkout PR code. Actor-gated to dependabot[bot]. - - dependabot-major-analysis.yml - # ─── dangerous-triggers ──────────────────────────────────────────────── # `notify-pr.yml` uses `pull_request_target` intentionally: it needs # `secrets.SLACK_WEBHOOK` to fire on PRs from forks. It does NOT check out @@ -29,9 +19,3 @@ rules: ignore: - notify-pr.yml - fix-drift.yml - # Dependabot auto-merge: uses pull_request_target for write token. - # Does NOT checkout PR code. Actor-gated to dependabot[bot]. - - dependabot-auto-merge.yml - # Dependabot major analysis: uses pull_request_target for PR comments. - # Does NOT checkout PR code. Actor-gated to dependabot[bot]. - - dependabot-major-analysis.yml diff --git a/CHANGELOG.md b/CHANGELOG.md index 034359bb..9f433dea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,22 @@ # @copilotkit/aimock -## [Unreleased] +## [1.36.1] - 2026-07-14 + +### Fixed + +- `aimock.json` now honours `llm.latency`, `llm.chunkSize`, `llm.replaySpeed` and `llm.logLevel`, which the config loader previously accepted and ignored. Each behaves as its `llmock` CLI flag and `createMockSuite({ llm })` equivalent. `llm.replaySpeed` divides every delay source (recorded timings, streaming profiles, `latency`), so a fixture suite can replay recorded inter-chunk timings faster than they were recorded; a fixture's own `replaySpeed` still takes precedence. A non-positive `llm.replaySpeed` is ignored with a warning, matching the existing fixture-level guard (#294) + +## [1.36.0] - 2026-07-13 + +### Added + +- On a fixture-miss passthrough (record/proxy mode), aimock can inject its own configured upstream provider key instead of forwarding a caller's dummy placeholder key. Every static-key provider aimock proxies is wired end-to-end, each with an independent `AIMOCK_PROVIDER__KEY` env var applied with the provider-correct wire scheme: `Authorization: Bearer` for OpenAI (`_OPENAI_KEY`), OpenRouter (`_OPENROUTER_KEY`), Cohere (`_COHERE_KEY`), Grok/xAI (`_GROK_KEY`), and Ollama (`_OLLAMA_KEY`); `x-api-key` for Anthropic (`_ANTHROPIC_KEY`); `x-goog-api-key` for Gemini (`_GEMINI_KEY`, also used for Gemini Interactions) and Veo (`_VEO_KEY`); `api-key` for Azure OpenAI (`_AZURE_KEY`); `xi-api-key` for ElevenLabs (`_ELEVENLABS_KEY`); and `Authorization: Key` for fal.ai (`_FAL_KEY`). Injection fires only when the caller credential is absent or dummy-prefixed (`sk-aimock-`, overridable via `AIMOCK_DUMMY_KEY_MARKER`); a real caller key is always forwarded unchanged, an empty-string env var is treated as unset, and with no built-in key configured the feature is inert. Signed/exchanged credentials — AWS Bedrock (SigV4) and Vertex AI (OAuth) — are never rewritten (#293) + +## [1.35.1] - 2026-07-06 + +### Fixed + +- Record mode no longer silently drops fixtures when an SDK closes its socket immediately after `data: [DONE]` (e.g. the OpenAI Python SDK); the completed upstream response is now persisted. Genuine mid-stream aborts (client disconnect before `[DONE]`) still abort upstream and write no fixture (#288) ## [1.35.0] - 2026-06-27 diff --git a/README.md b/README.md index 0e3a29b7..687ca280 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,28 @@ Private and link-local addresses (loopback, RFC1918, CGNAT, cloud metadata, ULA, On replay, `turnIndex` is a non-fatal disambiguator, not a hard reject gate: a content-matching fixture is served even when its scripted `turnIndex` differs from the request's assistant-message count. This kills false "no fixture matched" misses for multi-bubble agent runs (multi-step agents emit several assistant bubbles per logical turn). When a served fixture diverges from its scripted `turnIndex`, the match diagnostic carries `turnIndexRelaxed: true` and aimock logs a one-shot warning (at the `warn` log level — silent by default). To restore the legacy strict behavior where a defined `turnIndex` must equal the assistant count exactly, set `AIMOCK_STRICT_TURN_INDEX=1`. The record path is always strict regardless of this flag. +### aimock-owned upstream keys — `AIMOCK_PROVIDER_*_KEY` + +In record or `--proxy-only` mode, aimock forwards the caller's auth header to the real provider unchanged. If your tests can only send a dummy placeholder key (e.g. an SDK that refuses to start without a non-empty API key), aimock can inject its own configured upstream key on a fixture-miss passthrough so the proxied call actually authenticates. Each provider has an independent env var, and the key is applied with the provider-correct wire scheme: + +| Env var | Provider | Injected header | +| -------------------------------- | -------------------------------- | ----------------------------- | +| `AIMOCK_PROVIDER_OPENAI_KEY` | OpenAI | `Authorization: Bearer ` | +| `AIMOCK_PROVIDER_OPENROUTER_KEY` | OpenRouter | `Authorization: Bearer ` | +| `AIMOCK_PROVIDER_COHERE_KEY` | Cohere | `Authorization: Bearer ` | +| `AIMOCK_PROVIDER_GROK_KEY` | Grok (xAI) | `Authorization: Bearer ` | +| `AIMOCK_PROVIDER_OLLAMA_KEY` | Ollama (Cloud / bearer-gated) | `Authorization: Bearer ` | +| `AIMOCK_PROVIDER_ANTHROPIC_KEY` | Anthropic | `x-api-key: ` | +| `AIMOCK_PROVIDER_GEMINI_KEY` | Gemini (and Gemini Interactions) | `x-goog-api-key: ` | +| `AIMOCK_PROVIDER_VEO_KEY` | Veo | `x-goog-api-key: ` | +| `AIMOCK_PROVIDER_AZURE_KEY` | Azure OpenAI | `api-key: ` | +| `AIMOCK_PROVIDER_ELEVENLABS_KEY` | ElevenLabs | `xi-api-key: ` | +| `AIMOCK_PROVIDER_FAL_KEY` | fal.ai | `Authorization: Key ` | + +The Gemini interactions provider mode reuses `AIMOCK_PROVIDER_GEMINI_KEY` (same upstream API as Gemini). An empty-string value is treated as unset. + +This is opt-in and backward-compatible: with no key configured the feature is inert and the caller's header passes through as-is. Injection fires only when the caller sent no credential **or** a dummy credential prefixed with `sk-aimock-` (overridable via `AIMOCK_DUMMY_KEY_MARKER`); a real caller key never starting with that marker is always forwarded verbatim, so the caller overrides aimock. Signed and exchanged credentials — AWS Bedrock (SigV4) and Vertex AI (OAuth) — are never rewritten and always forwarded unchanged. (Azure's static `api-key` is injected; a real Microsoft Entra ID `Authorization: Bearer` token from the caller is never dummy-prefixed, so it too passes through verbatim.) + ## Framework Guides Test your AI agents with aimock — no API keys, no network calls: [LangChain](https://aimock.copilotkit.dev/integrate-langchain) · [CrewAI](https://aimock.copilotkit.dev/integrate-crewai) · [PydanticAI](https://aimock.copilotkit.dev/integrate-pydanticai) · [LlamaIndex](https://aimock.copilotkit.dev/integrate-llamaindex) · [Mastra](https://aimock.copilotkit.dev/integrate-mastra) · [Google ADK](https://aimock.copilotkit.dev/integrate-adk) · [Microsoft Agent Framework](https://aimock.copilotkit.dev/integrate-maf) diff --git a/charts/aimock/Chart.yaml b/charts/aimock/Chart.yaml index 6c63c6eb..d972ea60 100644 --- a/charts/aimock/Chart.yaml +++ b/charts/aimock/Chart.yaml @@ -3,4 +3,4 @@ name: aimock description: Mock infrastructure for AI application testing (OpenAI, Anthropic, Gemini, MCP, A2A, vector) type: application version: 0.1.0 -appVersion: "1.35.0" +appVersion: "1.35.1" diff --git a/docs/aimock-cli/index.html b/docs/aimock-cli/index.html index 2ad382de..51f37421 100644 --- a/docs/aimock-cli/index.html +++ b/docs/aimock-cli/index.html @@ -102,6 +102,7 @@

Config File Format

"fixtures": "./fixtures", "latency": 0, "chunkSize": 20, + "replaySpeed": 1, "logLevel": "info", "validateOnLoad": true, "metrics": true, @@ -135,9 +136,9 @@

Config Fields

object LLMock configuration. Accepts fixtures, latency, - chunkSize, logLevel, validateOnLoad, - metrics, strict, chaos, - streamingProfile, record. + chunkSize, replaySpeed, logLevel, + validateOnLoad, metrics, strict, + chaos, streamingProfile, record. diff --git a/docs/record-replay/index.html b/docs/record-replay/index.html index 062ab46c..6d49dfd4 100644 --- a/docs/record-replay/index.html +++ b/docs/record-replay/index.html @@ -489,6 +489,114 @@

Header Forwarding

recordedTimings block (streaming frame timings) — never the request headers.

+

aimock-Owned Upstream Keys

+

+ By default, aimock forwards the caller's auth header to the upstream provider unchanged. + If your tests can only send a dummy placeholder key — for example, an SDK that refuses to + start without a non-empty API key — aimock can inject its own configured upstream + key on a fixture-miss passthrough so the recorded/proxied call actually authenticates. + This is opt-in and backward-compatible: with no key configured the + feature is fully inert and the caller's header is forwarded as-is. +

+

+ Set one or more of these env vars to aimock's real upstream keys. Each is independent — + configure only the providers you record against: +

+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Env varProviderInjected header
AIMOCK_PROVIDER_OPENAI_KEYOpenAIAuthorization: Bearer <key>
AIMOCK_PROVIDER_OPENROUTER_KEYOpenRouterAuthorization: Bearer <key>
AIMOCK_PROVIDER_COHERE_KEYCohereAuthorization: Bearer <key>
AIMOCK_PROVIDER_GROK_KEYGrok (xAI)Authorization: Bearer <key>
AIMOCK_PROVIDER_OLLAMA_KEYOllama (Cloud / bearer-gated)Authorization: Bearer <key>
AIMOCK_PROVIDER_ANTHROPIC_KEYAnthropicx-api-key: <key>
AIMOCK_PROVIDER_GEMINI_KEYGemini (and Gemini Interactions)x-goog-api-key: <key>
AIMOCK_PROVIDER_VEO_KEYVeox-goog-api-key: <key>
AIMOCK_PROVIDER_AZURE_KEYAzure OpenAIapi-key: <key>
AIMOCK_PROVIDER_ELEVENLABS_KEYElevenLabsxi-api-key: <key>
AIMOCK_PROVIDER_FAL_KEYfal.aiAuthorization: Key <key>
+

+ gemini-interactions reuses AIMOCK_PROVIDER_GEMINI_KEY (same + upstream API as Gemini). An empty-string value is treated as unset. +

+

Injection fires only when both of these hold:

+
    +
  • + The request is a fixture-miss passthrough (record mode or + --proxy-only). +
    • +
  • + The caller sent no credential for that provider, + or sent a dummy credential prefixed with + sk-aimock-. A caller credential that does not start with the marker is + treated as a real key and forwarded verbatim — the caller always overrides aimock. +
    • +
+

+ The dummy marker prefix is overridable via AIMOCK_DUMMY_KEY_MARKER for setups + that mint placeholder keys under a different prefix. Gemini is injected as an + x-goog-api-key header only (no query-param ?key= rewrite). +

+

+ Signed and exchanged credentials are never rewritten. AWS Bedrock + (SigV4), Vertex AI, and Azure AD carry OAuth/signed auth rather than a simple bearer or + api-key header, so aimock always forwards their credentials unchanged regardless of these + env vars. +

+

Strict Mode

When --strict is enabled, an unmatched request returns diff --git a/package.json b/package.json index a32e00eb..8fd650c2 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@copilotkit/aimock", - "version": "1.35.0", + "version": "1.36.1", "description": "Mock infrastructure for AI application testing — LLM APIs, image generation, image editing, text-to-speech, transcription, audio translation, audio generation, video generation, embeddings, MCP tools, A2A agents, AG-UI event streams, vector databases, search, rerank, and moderation. One package, one port, zero dependencies.", "license": "MIT", "keywords": [ diff --git a/packages/aimock-pytest/README.md b/packages/aimock-pytest/README.md index 7f8d4031..ae099024 100644 --- a/packages/aimock-pytest/README.md +++ b/packages/aimock-pytest/README.md @@ -79,7 +79,7 @@ aimock.reset() # alias for reset_fixtures() ``` --aimock-node PATH Path to node binary ---aimock-version VER aimock npm version (default: 1.35.0) +--aimock-version VER aimock npm version (default: 1.35.1) ``` ## Environment Variables diff --git a/packages/aimock-pytest/src/aimock_pytest/_version.py b/packages/aimock-pytest/src/aimock_pytest/_version.py index d3751b8e..e1f62ee5 100644 --- a/packages/aimock-pytest/src/aimock_pytest/_version.py +++ b/packages/aimock-pytest/src/aimock_pytest/_version.py @@ -8,4 +8,4 @@ control routes ship in the next release). Keep it tracking npm releases. """ -AIMOCK_VERSION = "1.35.0" +AIMOCK_VERSION = "1.35.1" diff --git a/renovate.json b/renovate.json new file mode 100644 index 00000000..62dc1e5a --- /dev/null +++ b/renovate.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": ["local>CopilotKit/renovate"] +} diff --git a/scripts/drift-delta.ts b/scripts/drift-delta.ts new file mode 100644 index 00000000..4c435904 --- /dev/null +++ b/scripts/drift-delta.ts @@ -0,0 +1,146 @@ +/** + * Delta-gating core for the drift pipeline. + * + * A PR-scoped drift run produces a HEAD report; a same-UTC-day cached MAIN run + * produces a BASE report. The gate must only BLOCK on drift that is attributable + * to the diff under review — a drift already present on main (environmental / + * world drift) is the daily job's concern, NOT the PR's. This module reduces the + * two reports to a per-key delta: + * + * - block → keys NEW in head (absent from base): diff-attributable, hard fail + * - advisory → keys present in BOTH base and head: pre-existing / environmental + * - fixed → keys present in base but GONE in head: informational + * + * The block/advisory/fixed decision is driven ENTIRELY by key presence. The + * coarse `DriftClass` (critical/advisory/quarantine) is annotation only and NEVER + * enters the routing decision — a new-in-head *critical* still BLOCKS, and a + * both-present *critical* is still only ADVISORY. This is the #292 invariant: a + * real-drift/critical finding that is new in head must block regardless of class. + * + * Pure and side-effect-free — no filesystem, no network, no process state. + */ + +import type { DriftClass, DriftReport } from "./drift-types.js"; + +/** + * A single failure identified by the delta layer. + * + * A failure is keyed by `provider` + per-item `id` (NOT `path`, which is a + * generic bucket like `"knownModels"` that would collapse N distinct model + * drifts into one key). `class` is carried through purely for annotation and + * never participates in routing. + */ +export interface DeltaKey { + /** Provider the failing entry belongs to (e.g. "anthropic"). */ + provider: string; + /** Stable per-item key (e.g. a model id). Falls back to path when absent. */ + id: string; + /** Coarse classification — annotation only, never routes block vs advisory. */ + class?: DriftClass; +} + +/** The delta between a base (main) report and a head (PR) report. */ +export interface DeltaResult { + /** New-in-head failures — diff-attributable, hard fail. */ + block: DeltaKey[]; + /** Failures present in both base and head — pre-existing / environmental. */ + advisory: DeltaKey[]; + /** Failures present in base but gone in head — informational. */ + fixed: DeltaKey[]; +} + +/** + * Build the canonical string key for a failure. Keys by provider + per-item id. + * When a diff has no `id` (legacy diffs), fall back to its `path` so it still + * participates in the delta rather than being dropped — this is a coarse bucket + * but preserves the block/advisory distinction across reports. + */ +function keyOf(provider: string, id: string): string { + return `${provider}::${id}`; +} + +/** + * Flatten a report into a map of `key → DeltaKey`. When multiple diffs collapse + * to the same key, the last one wins for annotation purposes (the routing + * decision does not depend on which representative is retained). + */ +function indexReport(report: DriftReport): Map { + const index = new Map(); + for (const entry of report.entries) { + for (const diff of entry.diffs) { + // `id` is the stable per-item key; `path` is the legacy fallback bucket. + const id = diff.id ?? diff.path; + const key = keyOf(entry.provider, id); + index.set(key, { provider: entry.provider, id, class: diff.class }); + } + } + return index; +} + +/** + * Compute the delta between a base report and a head report. + * + * Routing is by KEY PRESENCE only: + * - key in head but not base → block (new-in-head, diff-attributable) + * - key in both → advisory (pre-existing / environmental) + * - key in base but not head → fixed (informational) + * + * `DriftClass` NEVER enters this decision — a new-in-head critical blocks, a + * both-present critical is advisory. See the module docstring (#292 invariant). + */ +export function computeDelta(baseReport: DriftReport, headReport: DriftReport): DeltaResult { + const base = indexReport(baseReport); + const head = indexReport(headReport); + + const block: DeltaKey[] = []; + const advisory: DeltaKey[] = []; + const fixed: DeltaKey[] = []; + + for (const [key, headKey] of head) { + if (base.has(key)) { + advisory.push(headKey); + } else { + block.push(headKey); + } + } + + for (const [key, baseKey] of base) { + if (!head.has(key)) { + fixed.push(baseKey); + } + } + + return { block, advisory, fixed }; +} + +/** + * Known-good collector conclusions that make a cached base report trustworthy to + * reuse. Anything else (crash, quarantine, unknown) means the base is not a + * reliable baseline and a fresh live base run should be preferred. + */ +const REUSABLE_CONCLUSIONS: ReadonlySet = new Set(["clean", "success"]); + +/** + * Guard (O-2) for reusing a cached same-UTC-day main report as the delta base. + * + * A base report is reusable only when ALL of: + * 1. it has a non-empty `entries[]` (rejects empty / malformed cached JSON), + * 2. its collector `conclusion` is known-good (rejects crash / quarantine), + * 3. it was produced on the same UTC day (`sameUtcDay`) — a stale baseline + * would misattribute a day's worth of environmental drift to the PR. + * + * Anything else → not reusable → caller should run a fresh live base. + */ +export function isBaseReportReusable( + report: DriftReport | null | undefined, + conclusion: string | null | undefined, + sameUtcDay: boolean, +): boolean { + if (!report || !Array.isArray(report.entries) || report.entries.length === 0) { + return false; + } + if (!conclusion || !REUSABLE_CONCLUSIONS.has(conclusion)) { + return false; + } + return sameUtcDay === true; +} diff --git a/scripts/drift-report-collector.ts b/scripts/drift-report-collector.ts index f46626c4..1108a8c4 100644 --- a/scripts/drift-report-collector.ts +++ b/scripts/drift-report-collector.ts @@ -10,7 +10,8 @@ * Exit codes: * 0 — no critical diffs found (or no drift at all) * 2 — at least one critical diff exists - * 1 — script error (unhandled exception) + * 5 — at least one failure was quarantined (unparseable/untrusted — needs review) + * 1 — AG-UI drift detection was skipped (infra), or an unhandled script error * * Usage: * npx tsx scripts/drift-report-collector.ts [--out drift-report.json] @@ -19,8 +20,15 @@ import { execSync } from "node:child_process"; import { existsSync, statSync, writeFileSync } from "node:fs"; import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; -import type { DriftEntry, DriftReport, DriftSeverity, ParsedDiff } from "./drift-types.js"; +import type { + DriftEntry, + DriftReport, + DriftSeverity, + ParsedDiff, + QuarantineEntry, +} from "./drift-types.js"; // --------------------------------------------------------------------------- // Vitest JSON reporter types (subset we care about) @@ -178,7 +186,7 @@ const AGUI_DRIFT_TEST = "src/__tests__/drift/agui-schema.drift.ts"; */ const VALID_SEVERITIES = new Set(["critical", "warning", "info"]); -function parseDriftBlock(text: string): { context: string; diffs: ParsedDiff[] } | null { +export function parseDriftBlock(text: string): { context: string; diffs: ParsedDiff[] } | null { const headerMatch = text.match(/API DRIFT DETECTED:\s*(.+)/); if (!headerMatch) return null; @@ -199,13 +207,17 @@ function parseDriftBlock(text: string): { context: string; diffs: ParsedDiff[] } ); continue; } + const path = match[3].trim(); diffs.push({ severity: severity as DriftSeverity, issue: match[2].trim(), - path: match[3].trim(), + path, expected: match[4].trim(), real: match[5].trim(), mock: match[6].trim(), + // Stable per-item key derived from the offending path so different paths + // yield different delta keys (provider+id) in the D6.1 delta layer. + id: path, }); } @@ -225,7 +237,7 @@ function parseDriftBlock(text: string): { context: string; diffs: ParsedDiff[] } * "OpenAI Chat (non-streaming text)" → "OpenAI Chat" * "Anthropic Claude drift" → "Anthropic Claude" */ -function extractProviderName(text: string): string | null { +export function extractProviderName(text: string): string | null { // Try matching against known provider keys (longest first to avoid partial matches) const sorted = Object.keys(PROVIDER_MAP).sort((a, b) => b.length - a.length); for (const key of sorted) { @@ -240,11 +252,148 @@ function extractProviderName(text: string): string | null { * "OpenAI Chat (non-streaming text)" → "non-streaming text" * "Anthropic Claude (streaming tool call)" → "streaming tool call" */ -function extractScenario(context: string): string { +export function extractScenario(context: string): string { const parenMatch = context.match(/\(([^)]+)\)/); return parenMatch ? parenMatch[1] : context; } +// --------------------------------------------------------------------------- +// Known-models canary recognizer +// --------------------------------------------------------------------------- + +/** + * Discriminated result of parsing the ws-realtime canary failure. + * + * CLASS 3: `ids` holds ONLY genuine model ids (never a prose sentinel). The + * "some ids were truncated in CI output" fact is a boolean flag, not a fake id, + * so a non-model annotation can never flow into `DriftEntry.real` downstream. + * + * CLASS 2: `noGA` marks the hasGA-false mode — the GA realtime family was + * renamed/removed or the credential could not see any realtime models. `ids` + * then carries the OBSERVED realtime models (possibly empty), not "unknown" + * ones. Either mode is a critical, exit-2 drift signal. + */ +export interface CanaryParseResult { + ids: string[]; + truncated?: boolean; + noGA?: boolean; + /** + * NO_GA mode only: the unknown-model list observed in the SAME run. Because + * the hasGA assertion short-circuits the later unknown-models assertion, the + * NO_GA marker carries both lists (`… | UNKNOWN_REALTIME_MODELS=…`); this + * field holds the unknown segment so the combined case loses no information. + */ + unknownIds?: string[]; +} + +/** + * Parse the ws-realtime canary assertion failure. Two canary modes exist: + * + * 1. UNKNOWN models — `expect(unknown, \`UNKNOWN_REALTIME_MODELS=…\`).toEqual([])` + * shape: `AssertionError: UNKNOWN_REALTIME_MODELS=a,b: expected [ 'a', …(1) ] + * to deeply equal []`. + * 2. NO GA models (CLASS 2) — `expect(hasGA, \`NO_GA_REALTIME_MODELS=…\`).toBe(true)` + * shape: `AssertionError: NO_GA_REALTIME_MODELS=x,y: expected false to be true`. + * + * PRIMARY SOURCE: the `*_REALTIME_MODELS=` marker carries the FULL, non-truncated + * comma-joined list verbatim (vitest truncates the printed array with `…(N)` / + * `... (N)`, so ids beyond the first are unrecoverable from the array alone). + * + * FALLBACK (unknown-mode only, marker missing/mangled): parse the printed array + * and set `truncated` when a CI ellipsis is present. + * + * Returns a CanaryParseResult, or null if the message is not a canary shape. + */ +export function parseKnownModelsCanary(text: string): CanaryParseResult | null { + // CLASS 2 PRIMARY: hasGA-false marker. The GA family is gone/unreachable — a + // critical signal even when the observed list is empty. Recognize it FIRST so + // the "expected false to be true" shape is a structured entry, not a crash. + const noGaMatch = text.match(/NO_GA_REALTIME_MODELS=(.*?)(?::\s*expected\b|\n|$)/); + if (noGaMatch) { + // The NO_GA marker carries BOTH the observed realtime models AND the + // unknown-model list observed in the same run, joined as + // ` | UNKNOWN_REALTIME_MODELS=`. Split the two apart so + // neither list pollutes the other (the combined no-GA + unknown case must + // not lose the unknown list, since the hasGA assertion short-circuits the + // later unknown-models assertion). A legacy message without the unknown + // segment still parses — `split` yields a single element. + const [observedRaw, unknownRaw] = noGaMatch[1].split(/\s*\|\s*UNKNOWN_REALTIME_MODELS=/); + const toList = (s: string | undefined): string[] => + (s ?? "") + .split(",") + .map((v) => v.trim()) + .filter((v) => v.length > 0); + const ids = toList(observedRaw); + const unknownIds = toList(unknownRaw); + return unknownIds.length > 0 ? { ids, noGA: true, unknownIds } : { ids, noGA: true }; + } + + // UNKNOWN-mode PRIMARY: stable marker carrying the full, non-truncated list. + // Scan the whole message (not just line 0) so a leading blank line cannot hide + // it. Capture the ENTIRE value up to the vitest boilerplate (`: expected …`) + // or end-of-line. On a malformed/empty marker we fall THROUGH to the + // printed-array fallback so a recoverable id in the array is still surfaced. + const markerMatch = text.match(/UNKNOWN_REALTIME_MODELS=(.*?)(?::\s*expected\b|\n|$)/); + if (markerMatch) { + const ids = markerMatch[1] + .split(",") + .map((s) => s.trim()) + .filter((s) => s.length > 0); + if (ids.length > 0) return { ids }; + } + + // FALLBACK GATE: the printed-array fallback matches the GENERIC vitest shape + // `expected [ ... ] to deeply equal []`, which ANY `toEqual([])` assertion in + // ANY provider's test emits. On its own that shape is NOT a canary signal — a + // non-canary failure (e.g. a different provider asserting an empty array whose + // observed value happens to be a secret or an object shape) would otherwise be + // misclassified as OpenAI-Realtime known-models drift and have its arbitrary + // array contents relabeled as "unknown model ids". So the fallback fires ONLY + // when the message is clearly the ws-realtime known-models canary. Two + // recognizers (either suffices): + // 1. the realtime-canary marker family (`UNKNOWN_REALTIME_MODELS=` / + // `NO_GA_REALTIME_MODELS=`), even mangled/partial — the marker paths + // above only fall through here when the marker VALUE was empty/unusable, + // but the TOKEN itself still identifies the canary; and + // 2. the canary's origin file in the stack trace — a real canary failure + // ALWAYS carries an `at …/ws-realtime.drift.ts` frame. + // A generic non-canary `toEqual([])` failure has neither, so it returns null + // and falls through to the collector's normal unparseable/fail-loud handling. + const isRealtimeCanaryContext = + /_REALTIME_MODELS=/.test(text) || /ws-realtime\.drift\.ts/.test(text); + if (!isRealtimeCanaryContext) return null; + + // FALLBACK: no (usable) marker but a confirmed canary context. Best-effort + // parse of the printed array on the first line. The canary assertion is always + // on line 1. + const firstLine = text.split("\n")[0]; + + // Shape: ...: expected [ ... ] to deeply equal [] + const canaryMatch = firstLine.match(/expected\s*\[([^\]]*)\]\s*to deeply equal \[\]/); + if (!canaryMatch) return null; + + const inner = canaryMatch[1].trim(); + + // Extract quoted model ids from the bracket list + const ids: string[] = []; + const idPattern = /'([^']+)'/g; + let m: RegExpExecArray | null; + while ((m = idPattern.exec(inner)) !== null) { + ids.push(m[1]); + } + + // CI truncation marker in BOTH forms: single-glyph `…(N)` and three-dot ASCII + // `... (N)`. CLASS 3: this is a BOOLEAN flag — never a synthetic id. + const truncated = /(?:…|\.{3})\s*\(\d+\)/.test(inner); + + // A genuinely-empty inner list with no truncation (`expected [] to deeply + // equal []`) means the canary's `unknown` array was empty — no drift to + // surface. Return null so it is not a spurious entry. + if (ids.length === 0 && !truncated) return null; + + return truncated ? { ids, truncated: true } : { ids }; +} + // --------------------------------------------------------------------------- // Run drift tests and collect results // --------------------------------------------------------------------------- @@ -324,9 +473,205 @@ function runDriftTests(): VitestJsonResult { } } -function collectDriftEntries(results: VitestJsonResult): DriftEntry[] { +/** + * Distinguish genuine infrastructure errors from genuine-but-unparseable drift + * reports. Returns true when the whole batch of unparseable messages should be + * SWALLOWED as benign infra (collector exits 0); false means genuine drift is + * present and the collector must throw. + * + * A false "infra" classification here is dangerous: it makes the collector exit + * 0 and silently drop real drift. + * + * F2: `/AssertionError/i` is DELIBERATELY NOT an infra indicator. Every vitest + * drift failure is an AssertionError, so treating it as benign infra masks real + * drift. Infra failures announce themselves with network/HTTP/parse markers + * below; a bare AssertionError is drift until proven infra. + * + * A3: BOTH the infra-indicator scan and the drift-like scan run against the + * SAME normalized text (stack-trace ` at …` frames stripped). An earlier + * version scanned the RAW message for infra indicators but the stack-stripped + * message for drift indicators; that asymmetry could tip the benign-infra gate + * true and silently drop genuine drift. Normalizing both identically keeps the + * two scans from ever disagreeing on the same input. + */ +// Infra indicators. IMPORTANT (CLASS 1 / uniform anchoring): every infra +// indicator is defined here as a BARE phrase (the source string only, no `^`, +// no flags, no anchoring-defeating `(?:.*:\s*)?` prefix). The line-anchor is +// applied UNIFORMLY by `anchorInfraIndicator` below, so no single indicator can +// be individually mis-anchored and a phrase added to this list later is +// automatically anchored the same way as its siblings. +// +// The anchor requires the phrase to BE the failure reason at the START of a +// line (optional leading whitespace, an optional `HTTP ` prefix for the numeric +// ones), followed by a word boundary. This means a genuine infra REASON like +// `empty response`, `fetch failed`, or `API returned 503` at line start still +// classifies as infra, while a labelled drift-body VALUE like +// ` Real: API returned 503` / ` Real: empty response` can NEVER trip +// the gate (the phrase is not at the line start, it follows a `Field:` label). +// +// Each entry is a bare `source` phrase plus a `boundary` flag: `true` appends a +// trailing `\b` (word-like phrases such as `empty response` / `ECONNREFUSED`), +// `false` omits it for phrases whose next char is punctuation (`INFRA_ERROR:`) +// or a tag (`[\b] + * + * so the phrase must be the failure reason at the start of a line. A labelled + * drift-body value (`Real: `) is preceded by a `Field:` label and thus + * never matches. This is the single choke point that makes anchoring uniform — + * there is no per-indicator anchoring to get wrong. + */ +function anchorInfraIndicator(spec: InfraIndicatorSpec): RegExp { + const boundary = spec.boundary === false ? "" : "\\b"; + return new RegExp(`^\\s*(?:HTTP\\s+)?${spec.source}${boundary}`, "im"); +} + +/** + * The bare infra phrase sources, exported so tests can iterate the REAL list + * (property-based test in drift-collector.test.ts) rather than a hand-copied + * subset. A future indicator added to INFRA_INDICATOR_SPECS is automatically + * covered — if it were somehow mis-anchored the property test would fail. + */ +export const INFRA_INDICATOR_SOURCES: readonly string[] = INFRA_INDICATOR_SPECS.map( + (s) => s.source, +); + +/** + * A concrete failure-reason sample for a given bare infra phrase source, used + * by the property-based test to synthesize both a bare (line-start, infra) and + * a labelled (`Real: …`, drift) line. Regex metacharacters in the source + * (`\d{3}`, `\s*:?`, tag `<`) are replaced with literal, matching sample text. + */ +export function infraIndicatorSample(source: string): string { + return source + .replace(/\\s\*:\?\\s\*/g, ": ") // status\s*:?\s* → "status: " + .replace(/\\s\*/g, " ") + .replace(/\\d\{3\}/g, "503") + .replace(/\\d\+/g, "503") + .replace(/\\b/g, ""); +} + +const INFRA_INDICATORS: RegExp[] = INFRA_INDICATOR_SPECS.map(anchorInfraIndicator); + +// Drift-like indicators. A genuine drift failure is drift until proven infra. +// The `expected … to be/equal/deeply equal …` shape is the canonical vitest +// assertion body for our drift/canary assertions — the OLD `/expected.*but/i` +// guard was DEAD (vitest never emits "expected … but …"), so it never fired and +// left bare assertion failures indistinguishable from infra. This repaired +// guard fires on the shapes vitest actually emits, so an unrecognized assertion +// failure counts as drift-like and forces a fail-loud (throw → exit 1). +const DRIFT_LIKE_INDICATORS = [ + /drift/i, + /mismatch/i, + /\bexpected\b.*\bto\s+(?:be|equal|deeply equal|contain|match|have)\b/i, + /LLMOCK DRIFT/i, + /API DRIFT/i, +]; + +/** Strip vitest stack-trace frames (` at …`) so filenames like + * "ws-realtime.drift.ts" cannot influence content-based classification. */ +function stripStackFrames(msg: string): string { + return msg + .split("\n") + .filter((line) => !/^\s*at\s/.test(line)) + .join("\n"); +} + +/** + * O-1: capture the raw `file:line` (or `file:line:col`) from the FIRST usable + * stack frame BEFORE `stripStackFrames` removes it, so a quarantined failure + * carries a pointer the human reviewer can jump to. Prefers a project-source + * frame (`src/…`) over node_modules/internal frames; falls back to the first + * frame with any `path:line` shape. Returns "" when no frame is present. + */ +export function extractRawLocation(msg: string): string { + const frames = msg.split("\n").filter((line) => /^\s*at\s/.test(line)); + // Match `path:line` or `path:line:col`, with an optional trailing `)`. + const locRe = /((?:\/|\.\/|[A-Za-z]:\\|file:\/\/)?[^\s()]+?:\d+(?::\d+)?)\)?\s*$/; + const pick = (predicate: (f: string) => boolean): string | null => { + for (const frame of frames) { + if (!predicate(frame)) continue; + const m = frame.match(locRe); + if (m) return m[1]; + } + return null; + }; + // Prefer a project-source frame, skipping node internals / node_modules. + return ( + pick((f) => /src\//.test(f) && !/node_modules/.test(f) && !/node:internal/.test(f)) ?? + pick((f) => !/node_modules/.test(f) && !/node:internal/.test(f)) ?? + pick(() => true) ?? + "" + ); +} + +export function classifyUnparseableAsInfra(unparseableMessages: string[]): boolean { + // CLASS 1 — fail-loud on absent evidence. No messages means NO positive infra + // evidence, so this is NOT a benign "all clear". `[].every(...)` is vacuously + // true, which would have wrongly classified an empty batch as infra and made + // the collector exit 0. Root invariant: unrecognized ⇒ fail loud, never a + // false all-clear. + if (unparseableMessages.length === 0) return false; + + // Normalize ONCE per message; both scans consume the identical normalized + // text (A3 — the two scans must never disagree due to differing inputs). + const normalized = unparseableMessages.map(stripStackFrames); + + // Infra requires POSITIVE evidence on EVERY message AND zero drift signal on + // ALL of them. If any message lacks an infra indicator, or any message looks + // drift-like, we do NOT swallow — the caller throws (exit 1, investigate). + const allInfraErrors = normalized.every((msg) => INFRA_INDICATORS.some((re) => re.test(msg))); + const anyDriftLike = normalized.some((msg) => DRIFT_LIKE_INDICATORS.some((re) => re.test(msg))); + + return allInfraErrors && !anyDriftLike; +} + +/** + * The result of collecting drift entries. In addition to the trustworthy drift + * `entries`, `quarantine` holds failures that could not be parsed/mapped into a + * trustworthy finding AND were not positively classified as benign infra (A1.3). + * These no longer crash the collector (exit 1) nor get silently dropped (exit + * 0): the caller routes a non-empty `quarantine` to exit 5. Exit 1 is now + * reserved for genuine collector bugs (unhandled exceptions). + */ +export interface CollectResult { + entries: DriftEntry[]; + quarantine: QuarantineEntry[]; +} + +export function collectDriftEntries(results: VitestJsonResult): CollectResult { const entries: DriftEntry[] = []; - const unmapped: string[] = []; + const quarantine: QuarantineEntry[] = []; let unparseable = 0; for (const file of results.testResults) { @@ -337,21 +682,138 @@ function collectDriftEntries(results: VitestJsonResult): DriftEntry[] { const fullMessage = assertion.failureMessages.join("\n"); const parsed = parseDriftBlock(fullMessage); if (!parsed || parsed.diffs.length === 0) { + // Check for the ws-realtime canary assertion shapes BEFORE classifying + // as unparseable — both the unknown-models mode and the hasGA-false mode + // (CLASS 2) are genuine, critical drift and must be surfaced (exit 2), + // never crashed as unparseable (exit 1). + const canary = parseKnownModelsCanary(fullMessage); + if (canary !== null) { + const mapping = PROVIDER_MAP["OpenAI Realtime"]; + + // Build the critical diffs. F4: severity MUST be "critical" — the Fix + // Drift workflow (.github/workflows/fix-drift.yml) gates remediation + // entirely on exit code 2, which main() emits only when + // criticalCount > 0. F5: this canary is real-API-only — there is no + // mock leg, so we never mislabel a model id as a mock value. + const NO_MOCK_LEG = ""; + const diffs: ParsedDiff[] = canary.noGA + ? [ + // CLASS 2: the GA realtime family is renamed/removed or the + // credential cannot see it. Surface the observed models (may be + // empty) so the auto-fix prompt knows what IS present. + { + severity: "critical" as const, + issue: + "GA realtime family unavailable — no known GA model in the OpenAI realtime list. " + + "OpenAI may have renamed/removed the GA family, or the realtime credential cannot see it. " + + "Update the gaModels list in ws-realtime.drift.ts.", + path: "gaModels", + expected: "(at least one GA realtime model present)", + real: + canary.ids.length > 0 + ? `observed realtime models: ${canary.ids.join(", ")}` + : "no realtime models observed", + mock: NO_MOCK_LEG, + }, + // The hasGA assertion short-circuits the later unknown-models + // assertion, so surface any unknown models observed in the SAME + // run here (carried by the NO_GA marker). Without this, a run + // that is BOTH no-GA AND has new unknown models would lose the + // unknown list from the auto-fix prompt. One diff per id — each + // is a genuine model id (never a prose sentinel). + ...(canary.unknownIds ?? []).map((id) => ({ + severity: "critical" as const, + issue: + "Unknown realtime model detected (observed in the same run as the missing GA " + + "family) — add to knownModels in ws-realtime.drift.ts", + path: "knownModels", + expected: "(not in knownModels set)", + real: id, + mock: NO_MOCK_LEG, + // D6.2: set `id` to the model id so the delta layer (D6.1) + // can key by provider+id, yielding one distinct key per model + // rather than collapsing all entries under path:"knownModels". + id, + })), + ] + : // CLASS 3: only genuine model ids become `real` diffs. The + // truncation fact is carried as a SEPARATE diff whose `real` is a + // count note, never a fake model id in a per-model slot. + [ + ...canary.ids.map((id) => ({ + severity: "critical" as const, + issue: + "Unknown realtime model detected — add to knownModels in ws-realtime.drift.ts", + path: "knownModels", + expected: "(not in knownModels set)", + real: id, + mock: NO_MOCK_LEG, + // D6.2: set `id` to the model id so the delta layer (D6.1) + // can key by provider+id, yielding one distinct key per model + // rather than collapsing all entries under path:"knownModels". + id, + })), + ...(canary.truncated + ? [ + { + severity: "critical" as const, + issue: + "Additional unknown realtime models were truncated in CI output — " + + "the full list is unrecoverable without the UNKNOWN_REALTIME_MODELS= marker. " + + "Re-run with the marker to enumerate them.", + path: "knownModels[truncated]", + // CLASS 3: `real`/`expected` NEVER carry a prose sentinel. + // The truncation fact lives entirely in `issue`/`path`; + // there is no observed model value to report here. + expected: "", + real: "", + mock: NO_MOCK_LEG, + }, + ] + : []), + ]; + + entries.push({ + provider: "OpenAI Realtime", + scenario: "known-models canary", + builderFile: mapping.builderFile, + builderFunctions: mapping.builderFunctions, + typesFile: mapping.typesFile ?? null, + sdkShapesFile: SDK_SHAPES_FILE, + diffs, + }); + continue; + } unparseable++; continue; } // Determine provider from ancestor titles (describe block) or context const ancestorText = assertion.ancestorTitles.join(" "); + const testName = `${ancestorText} > ${assertion.title}`; const provider = extractProviderName(ancestorText) ?? extractProviderName(parsed.context); if (!provider) { - unmapped.push(`${ancestorText} > ${assertion.title}`); + // Unmapped provider: a parseable drift block whose provider we cannot + // route to a source file. Held for review (exit 5) rather than crashing + // the whole run (was exit 1). O-1: capture the raw frame location BEFORE + // any stack stripping. + quarantine.push({ + provider: parsed.context || ancestorText || "unknown", + testName, + rawLocation: extractRawLocation(fullMessage), + message: fullMessage, + }); continue; } const mapping = PROVIDER_MAP[provider]; if (!mapping) { - unmapped.push(`${ancestorText} > ${assertion.title} (provider: ${provider})`); + quarantine.push({ + provider, + testName, + rawLocation: extractRawLocation(fullMessage), + message: fullMessage, + }); continue; } @@ -367,75 +829,64 @@ function collectDriftEntries(results: VitestJsonResult): DriftEntry[] { } } - if (unmapped.length > 0) { - console.error(`ERROR: ${unmapped.length} drift failure(s) could not be mapped to a provider:`); - for (const u of unmapped) console.error(` - ${u}`); - throw new Error(`${unmapped.length} unmapped drift entries — update PROVIDER_MAP`); + if (quarantine.length > 0) { + console.warn( + `WARNING: ${quarantine.length} drift failure(s) could not be mapped to a provider — ` + + `quarantined for review (exit 5), not crashed:`, + ); + for (const q of quarantine) + console.warn(` - ${q.testName} @ ${q.rawLocation || ""}`); } if (unparseable > 0 && entries.length === 0) { - // Collect the unparseable failure messages to classify them - const unparseableMessages: string[] = []; + // Collect the unparseable failure messages (with their raw pre-strip + // locations) to classify them. O-1: capture file:line BEFORE stripping. + const unparseableFailures: { message: string; testName: string; rawLocation: string }[] = []; for (const file of results.testResults) { for (const assertion of file.assertionResults) { if (assertion.status !== "failed" || assertion.failureMessages.length === 0) continue; const fullMessage = assertion.failureMessages.join("\n"); const parsed = parseDriftBlock(fullMessage); if (!parsed || parsed.diffs.length === 0) { - unparseableMessages.push(fullMessage); + // Canary shapes are handled above (they became entries) — only truly + // unparseable messages reach here. + if (parseKnownModelsCanary(fullMessage) !== null) continue; + unparseableFailures.push({ + message: fullMessage, + testName: `${assertion.ancestorTitles.join(" ")} > ${assertion.title}`, + rawLocation: extractRawLocation(fullMessage), + }); } } } + const unparseableMessages = unparseableFailures.map((f) => f.message); for (const msg of unparseableMessages) { console.warn(` Unparseable failure message (first 300 chars): ${msg.slice(0, 300)}`); } - // Distinguish infrastructure errors from broken drift report formats - const infraIndicators = [ - /^INFRA_ERROR:/m, - /API returned \d{3}/i, - /status \d{3}/i, - / - infraIndicators.some((re) => re.test(msg)), - ); - const anyDriftLike = unparseableMessages.some((msg) => - driftLikeIndicators.some((re) => re.test(msg)), - ); - - if (allInfraErrors && !anyDriftLike) { + if (classifyUnparseableAsInfra(unparseableMessages)) { console.warn( `WARNING: ${unparseable} test failure(s) appear to be API/infrastructure errors ` + `(not drift reports). Continuing with 0 drift entries.`, ); } else { - console.error( - `ERROR: ${unparseable} test failure(s) could not be parsed as drift reports.`, - "This may indicate broken test infrastructure or a changed report format.", - ); - throw new Error( - `${unparseable} unparseable test failures with 0 drift entries — investigate`, + // A1.3: genuine-but-unparseable drift is no longer a fail-loud crash (exit + // 1). Each such failure is quarantined (exit 5) so it surfaces for human + // review without being silently swallowed as a green. Exit 1 is now + // reserved for genuine collector bugs (unhandled exceptions). + console.warn( + `WARNING: ${unparseable} test failure(s) could not be parsed as drift reports — ` + + `quarantined for review (exit 5).`, ); + for (const f of unparseableFailures) { + quarantine.push({ + provider: "unknown", + testName: f.testName, + rawLocation: f.rawLocation, + message: f.message, + }); + } } } else if (unparseable > 0) { console.warn( @@ -443,7 +894,7 @@ function collectDriftEntries(results: VitestJsonResult): DriftEntry[] { ); } - return entries; + return { entries, quarantine }; } // --------------------------------------------------------------------------- @@ -629,6 +1080,39 @@ function collectAgUiDriftEntries(results: VitestJsonResult): DriftEntry[] { return entries; } +// --------------------------------------------------------------------------- +// Exit-code policy +// --------------------------------------------------------------------------- + +/** + * Map the three terminal signals a drift run can produce onto the collector's + * process exit code. Pure and side-effect-free so the mapping is unit-testable + * without spawning the drift suite. + * + * Precedence (highest first): + * - `criticalCount > 0` → 2 — at least one trustworthy critical drift. + * - `quarantineCount > 0`→ 5 — a failure could not be parsed/mapped into a + * trustworthy finding and was held for review; + * distinct from a critical (2) and from a clean + * pass (0) so it is never silently swallowed. + * - `agUiSkipped` → 1 — AG-UI drift detection could not run (infra). + * O2: AG-UI-skipped stays exit 1 (unchanged). + * - otherwise → 0 — no drift. + * + * Critical wins over quarantine: if the run found a genuine critical drift, that + * is the actionable signal even if some other failure was also quarantined. + */ +export function computeExitCode( + criticalCount: number, + quarantineCount: number, + agUiSkipped: boolean, +): 0 | 1 | 2 | 5 { + if (criticalCount > 0) return 2; + if (quarantineCount > 0) return 5; + if (agUiSkipped) return 1; + return 0; +} + // --------------------------------------------------------------------------- // Main // --------------------------------------------------------------------------- @@ -644,7 +1128,8 @@ function main(): void { console.log("Running HTTP API drift tests..."); const httpResults = runDriftTests(); console.log("Collecting HTTP API drift entries..."); - const httpEntries = collectDriftEntries(httpResults); + const httpResult = collectDriftEntries(httpResults); + const httpEntries = httpResult.entries; // Collect AG-UI schema drift entries console.log("Running AG-UI schema drift tests..."); @@ -659,10 +1144,12 @@ function main(): void { } const entries = [...httpEntries, ...agUiEntries]; + const quarantine = httpResult.quarantine; const report: DriftReport = { timestamp: new Date().toISOString(), entries, + ...(quarantine.length > 0 ? { quarantine } : {}), }; try { @@ -687,22 +1174,50 @@ function main(): void { ); console.log(` Critical diffs: ${criticalCount}`); - if (criticalCount > 0) { - console.log("Exiting with code 2 (critical diffs found)."); - process.exit(2); + const quarantineCount = quarantine.length; + console.log(` Quarantined failures: ${quarantineCount}`); + + const exitCode = computeExitCode(criticalCount, quarantineCount, agUiSkipped); + switch (exitCode) { + case 2: + console.log("Exiting with code 2 (critical diffs found)."); + process.exit(2); + // eslint-disable-next-line no-fallthrough + case 5: + console.warn(`Exiting with code 5 (${quarantineCount} failure(s) quarantined for review).`); + process.exit(5); + // eslint-disable-next-line no-fallthrough + case 1: + console.warn("Exiting with code 1 (AG-UI drift detection was skipped — infra failure)."); + process.exit(1); + // eslint-disable-next-line no-fallthrough + default: + console.log("No critical diffs found."); } +} - if (agUiSkipped) { - console.warn("Exiting with code 1 (AG-UI drift detection was skipped — infra failure)."); - process.exit(1); +/** + * Entry-point guard: only run main() when this module is executed directly + * (e.g. `npx tsx scripts/drift-report-collector.ts` from the Fix Drift + * workflow), NOT when it is imported (e.g. by the vitest suite that exercises + * the exported pure functions). Without this guard, importing the module for + * tests would spawn the whole drift suite via execSync and call process.exit. + */ +function isDirectRun(): boolean { + const entry = process.argv[1]; + if (!entry) return false; + try { + return fileURLToPath(import.meta.url) === resolve(entry); + } catch { + return false; } - - console.log("No critical diffs found."); } -try { - main(); -} catch (err: unknown) { - console.error("Fatal error:", err); - process.exit(1); +if (isDirectRun()) { + try { + main(); + } catch (err: unknown) { + console.error("Fatal error:", err); + process.exit(1); + } } diff --git a/scripts/drift-retry.ts b/scripts/drift-retry.ts index 6ba8e3a8..945bad5d 100644 --- a/scripts/drift-retry.ts +++ b/scripts/drift-retry.ts @@ -50,6 +50,7 @@ import { fileURLToPath } from "node:url"; // Collector exit-code contract (see drift-report-collector.ts header). export const EXIT_CLEAN = 0; export const EXIT_CRITICAL_DRIFT = 2; +export const EXIT_QUARANTINE = 5; // Defaults: keep the fleet of real-API calls small. 3 total attempts with a // ~45s backoff mirrors the observed transient window (the Fix Drift workflow @@ -78,7 +79,7 @@ export interface RetryOptions { } export interface RetryResult { - /** Final exit code to propagate (0 = clean/transient, 2 = persistent, other = crash). */ + /** Final exit code to propagate (0 = clean/transient, 2 = persistent, 5 = quarantine, other = crash). */ exitCode: number; /** True when at least one critical run was seen but a later run cleared it. */ transient: boolean; @@ -86,6 +87,8 @@ export interface RetryResult { criticalRuns: number; /** Per-attempt record, in order. */ attempts: RetryAttempt[]; + /** True when the collector exited with EXIT_QUARANTINE (5): unparseable output quarantined. */ + quarantine?: boolean; } /** @@ -125,6 +128,19 @@ export function retryUntilStable(opts: RetryOptions): RetryResult { continue; } + if (exitCode === EXIT_QUARANTINE) { + // Quarantine is a distinct terminal outcome: the collector encountered + // output it could not parse/classify. No retry — propagate immediately. + opts.log(`Collector exited ${exitCode} (quarantine) — propagating without retry.`); + return { + exitCode: EXIT_QUARANTINE, + transient: false, + criticalRuns, + attempts, + quarantine: true, + }; + } + // Any other code = collector crash / infra error. Do not retry — surface it. opts.log(`Collector exited ${exitCode} (not drift) — propagating without retry.`); return { exitCode, transient: false, criticalRuns, attempts }; diff --git a/scripts/drift-slack-summary.ts b/scripts/drift-slack-summary.ts index 6fb5412a..6b705477 100644 --- a/scripts/drift-slack-summary.ts +++ b/scripts/drift-slack-summary.ts @@ -29,6 +29,54 @@ import { fileURLToPath } from "node:url"; import { readDriftReport } from "./fix-drift.js"; import type { DriftEntry, DriftReport, DriftSeverity } from "./drift-types.js"; +// --------------------------------------------------------------------------- +// Headline classification +// --------------------------------------------------------------------------- + +/** + * Closed headline-class enum for a drift run. + * + * Derivation priority (highest wins): + * stale-key — InfraError status 401 or 403 + * infra-transient — InfraError status 429 or ≥500 + * quarantine — report.quarantine[] is non-empty (exit 5) + * real-drift — at least one entry has a critical diff (exit 2) + * test-infra-false-positive — everything else (exit 0 / advisory only) + */ +export type HeadlineClass = + | "real-drift" + | "test-infra-false-positive" + | "infra-transient" + | "stale-key" + | "quarantine"; + +/** Optional context the caller may supply to sharpen classification. */ +export interface SummarizeOptions { + /** HTTP status from an InfraError that aborted the run, if any. */ + infraErrorStatus?: number; + /** Process exit code produced by the collector/retry wrapper, if known. */ + exitCode?: number; +} + +function computeHeadlineClass(report: DriftReport, opts: SummarizeOptions): HeadlineClass { + const { infraErrorStatus } = opts; + + if (infraErrorStatus === 401 || infraErrorStatus === 403) return "stale-key"; + if (infraErrorStatus === 429 || (infraErrorStatus !== undefined && infraErrorStatus >= 500)) + return "infra-transient"; + + // Critical drift wins over quarantine — a confirmed critical finding is the + // actionable signal even when some failures were also quarantined (mirrors the + // computeExitCode contract: crit→2 wins over quarantine→5). + const hasCritical = report.entries.some((e) => e.diffs.some((d) => d.severity === "critical")); + if (hasCritical) return "real-drift"; + + const hasQuarantine = (report.quarantine?.length ?? 0) > 0; + if (hasQuarantine) return "quarantine"; + + return "test-infra-false-positive"; +} + // Keep the message scannable: cap how many example paths we list per provider // and how many total providers we enumerate before collapsing to a count. const MAX_PATHS_PER_PROVIDER = 3; @@ -39,12 +87,19 @@ const SEVERITY_ORDER: DriftSeverity[] = ["critical", "warning", "info"]; interface ProviderSummary { provider: string; counts: Record; + /** Ordered list of changed path strings (de-duplicated). */ paths: string[]; + /** Ordered list of per-diff ids (de-duplicated), when present. */ + ids: string[]; + /** The builderFile from the first matching entry, for the file reference. */ + builderFile: string; + /** The issue text from the first diff, for one-line context. */ + firstIssue: string; } /** * Group drift entries by provider, tallying severities and collecting a small, - * de-duplicated set of representative changed paths. + * de-duplicated set of representative changed paths and ids. */ function summarizeByProvider(entries: DriftEntry[]): ProviderSummary[] { const byProvider = new Map(); @@ -56,14 +111,23 @@ function summarizeByProvider(entries: DriftEntry[]): ProviderSummary[] { provider: entry.provider, counts: { critical: 0, warning: 0, info: 0 }, paths: [], + ids: [], + builderFile: entry.builderFile, + firstIssue: "", }; byProvider.set(entry.provider, summary); } for (const diff of entry.diffs) { summary.counts[diff.severity]++; + if (diff.id && !summary.ids.includes(diff.id)) { + summary.ids.push(diff.id); + } if (diff.path && !summary.paths.includes(diff.path)) { summary.paths.push(diff.path); } + if (!summary.firstIssue && diff.issue) { + summary.firstIssue = diff.issue; + } } } @@ -75,6 +139,18 @@ function summarizeByProvider(entries: DriftEntry[]): ProviderSummary[] { }); } +/** + * For a provider summary, return the ordered list of identifiers to use as + * per-item references in the Slack bullet. + * + * When any diff carries a stable `id` (e.g. a model id), prefer those over + * raw paths — they're more human-readable in a Slack alert. Fall back to paths + * when no ids are present. + */ +function buildIdOrPaths(s: ProviderSummary): string[] { + return s.ids.length > 0 ? s.ids : s.paths; +} + /** Format a severity tally like "2 critical, 1 warning" (omitting zeroes). */ function formatCounts(counts: Record): string { const parts: string[] = []; @@ -87,32 +163,90 @@ function formatCounts(counts: Record): string { /** * Build the Slack mrkdwn summary lines for the drifted providers. * - * Returns a single string with real `\n` line breaks. Each provider gets a - * bullet line: `• *Provider* — 2 critical: \`path.a\`, \`path.b\``. - * Returns an empty string when there are no entries (caller decides fallback). + * Returns a single string with real `\n` line breaks. The first line is a + * headline classification (`*Classification:* `), followed by per-item + * bullets in one of two forms depending on the class: + * + * Real-drift/advisory: + * `• *Provider* — 2 critical: \`path.a\`, \`path.b\` (src/helpers.ts)` + * + * Quarantine: + * `• [quarantine] Provider — testName (file:line): truncated message` + * + * Returns an empty string only when no entries, no quarantine, no InfraError + * context, and no exitCode hint are present (i.e. caller has nothing to say). + * + * The `drift_summary` GITHUB_OUTPUT key is the full return value of this + * function — callers write it verbatim to `GITHUB_OUTPUT` via writeGithubOutput. + * Existing consumers that read `steps.summary.outputs.drift_summary` receive + * the augmented text; they pattern-match on its content (not on internal + * structure), so the addition of the classification line is backward-compatible. */ -export function summarizeDriftReport(report: DriftReport): string { +export function summarizeDriftReport(report: DriftReport, opts: SummarizeOptions = {}): string { + const headlineClass = computeHeadlineClass(report, opts); + const summaries = summarizeByProvider(report.entries); - if (summaries.length === 0) return ""; + const quarantine = report.quarantine ?? []; + + // Nothing to say at all — preserve historical empty-string behaviour for a + // clean run with no context supplied. + if ( + summaries.length === 0 && + quarantine.length === 0 && + headlineClass === "test-infra-false-positive" && + opts.infraErrorStatus === undefined && + opts.exitCode === undefined + ) { + return ""; + } - const shown = summaries.slice(0, MAX_PROVIDERS_LISTED); const lines: string[] = []; - for (const s of shown) { - const examplePaths = s.paths.slice(0, MAX_PATHS_PER_PROVIDER).map((p) => `\`${p}\``); - const extraPaths = s.paths.length - examplePaths.length; - let pathStr = examplePaths.join(", "); - if (extraPaths > 0) pathStr += `, +${extraPaths} more`; + // ── Headline classification ──────────────────────────────────────────────── + lines.push(`*Classification:* ${headlineClass}`); - const counts = formatCounts(s.counts); - lines.push( - pathStr ? `• *${s.provider}* — ${counts}: ${pathStr}` : `• *${s.provider}* — ${counts}`, - ); + // ── Per-entry drift bullets (real-drift / advisory) ─────────────────────── + if (summaries.length > 0) { + const shown = summaries.slice(0, MAX_PROVIDERS_LISTED); + + for (const s of shown) { + // Per-item: prefer id over path when present on the first diff that has one. + const idOrPaths = buildIdOrPaths(s); + const exampleRefs = idOrPaths.slice(0, MAX_PATHS_PER_PROVIDER).map((r) => `\`${r}\``); + const extraRefs = idOrPaths.length - exampleRefs.length; + let refStr = exampleRefs.join(", "); + if (extraRefs > 0) refStr += `, +${extraRefs} more`; + + // Per-item: file reference from the entry's builderFile. + const fileRef = s.builderFile ? ` (${s.builderFile})` : ""; + + // Per-item: first one-line issue text. + const issueStr = s.firstIssue ? ` — _${s.firstIssue}_` : ""; + + const counts = formatCounts(s.counts); + lines.push( + refStr + ? `• *${s.provider}* — ${counts}: ${refStr}${issueStr}${fileRef}` + : `• *${s.provider}* — ${counts}${issueStr}${fileRef}`, + ); + } + + const hiddenProviders = summaries.length - shown.length; + if (hiddenProviders > 0) { + lines.push(`• …and ${hiddenProviders} more provider${hiddenProviders === 1 ? "" : "s"}`); + } } - const hiddenProviders = summaries.length - shown.length; - if (hiddenProviders > 0) { - lines.push(`• …and ${hiddenProviders} more provider${hiddenProviders === 1 ? "" : "s"}`); + // ── Quarantine entries ───────────────────────────────────────────────────── + if (quarantine.length > 0) { + lines.push( + `*Quarantined* (${quarantine.length} failure${quarantine.length === 1 ? "" : "s"} need human review):`, + ); + for (const q of quarantine) { + const loc = q.rawLocation ? ` (${q.rawLocation})` : ""; + const msg = q.message.slice(0, 120); + lines.push(`• [quarantine] *${q.provider}* — ${q.testName}${loc}: ${msg}`); + } } return lines.join("\n"); diff --git a/scripts/drift-types.ts b/scripts/drift-types.ts index 5eaec247..71f0d93d 100644 --- a/scripts/drift-types.ts +++ b/scripts/drift-types.ts @@ -15,6 +15,29 @@ */ export type DriftSeverity = "critical" | "warning" | "info"; +/** + * Coarse drift-classification enum used by the delta/summary layers to route a + * report to the right terminal outcome (block vs advisory vs quarantine). It is + * orthogonal to per-diff `DriftSeverity`: a report is `Quarantine` when its + * findings could not be trusted (unparseable / unmapped surface), independent of + * whether individual diffs were `critical`. Purely additive — existing consumers + * that never read `class` are unaffected. + */ +export enum DriftClass { + /** At least one critical, trustworthy drift finding — hard failure. */ + Critical = "critical", + /** Non-critical, informational drift — advisory only. */ + Advisory = "advisory", + /** + * A failure that could not be parsed/mapped into a trustworthy drift finding. + * Neither a clean pass nor a confirmed critical — held aside for human review + * so it is never silently swallowed as a green. + */ + Quarantine = "quarantine", + /** No drift detected. */ + None = "none", +} + export interface ParsedDiff { path: string; severity: DriftSeverity; @@ -22,6 +45,35 @@ export interface ParsedDiff { expected: string; real: string; mock: string; + /** + * Optional stable per-item key (e.g. a model id) used by the delta layer to + * key findings by provider+id. Absent on legacy diffs. + */ + id?: string; + /** Optional coarse classification for this diff. Absent on legacy diffs. */ + class?: DriftClass; +} + +/** + * A test failure that could not be parsed into a trustworthy drift finding and + * was NOT positively classified as benign infrastructure. Rather than crash the + * collector (exit 1) or silently drop the failure (exit 0), the failure is + * captured here so it can surface as a distinct quarantine outcome (exit 5) for + * human review. + */ +export interface QuarantineEntry { + /** Provider inferred from the failing assertion, or a best-effort label. */ + provider: string; + /** The failing test's name (ancestor titles + title). */ + testName: string; + /** + * Raw `file:line` captured from the original stack frame BEFORE stack-frame + * stripping, so the human reviewer can locate the failing assertion. Empty + * string when no frame was available. + */ + rawLocation: string; + /** The (possibly truncated) failure message that could not be parsed. */ + message: string; } export interface DriftEntry { @@ -37,4 +89,10 @@ export interface DriftEntry { export interface DriftReport { timestamp: string; entries: DriftEntry[]; + /** + * Optional list of failures held aside for human review (see QuarantineEntry). + * Absent/empty when there was nothing to quarantine — legacy consumers that + * ignore this field are unaffected. + */ + quarantine?: QuarantineEntry[]; } diff --git a/src/__tests__/config-loader.test.ts b/src/__tests__/config-loader.test.ts index 78a814a4..7f03af2e 100644 --- a/src/__tests__/config-loader.test.ts +++ b/src/__tests__/config-loader.test.ts @@ -1,9 +1,11 @@ -import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { loadConfig, startFromConfig } from "../config-loader.js"; import type { AimockConfig } from "../config-loader.js"; +import type { RecordedTimings } from "../types.js"; +import { Logger } from "../logger.js"; function makeTmpDir(): string { return mkdtempSync(join(tmpdir(), "config-loader-test-")); @@ -32,6 +34,47 @@ function writeFixtureFile(dir: string, name = "fixtures.json"): string { return filePath; } +// Content is long enough to split into several chunks so per-chunk delays are observable +function writeStreamFixtureFile( + dir: string, + recordedTimings?: RecordedTimings, + name = "stream-fixtures.json", +): string { + const filePath = join(dir, name); + writeFileSync( + filePath, + JSON.stringify({ + fixtures: [ + { + match: { userMessage: "hello" }, + response: { content: "Hello world from the streaming config test!" }, + ...(recordedTimings ? { recordedTimings } : {}), + }, + ], + }), + "utf-8", + ); + return filePath; +} + +// Streams a completion and counts SSE frames. Without stream: true there is no SSE and no +// chunk delays, so the timing assertions below would pass whether or not the options are wired. +async function streamChunkCount(url: string): Promise { + const resp = await fetch(`${url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4", + stream: true, + messages: [{ role: "user", content: "hello" }], + }), + }); + const body = await resp.text(); + return body + .split("\n\n") + .filter((frame) => frame.startsWith("data: ") && !frame.includes("[DONE]")).length; +} + describe("loadConfig", () => { let tmpDir: string; @@ -156,6 +199,108 @@ describe("startFromConfig", () => { expect(resp.status).toBe(500); }); + it("with llm.chunkSize, the response is split into more chunks", async () => { + const fixturePath = writeStreamFixtureFile(tmpDir); + + const small = await startFromConfig({ llm: { fixtures: fixturePath, chunkSize: 5 } }); + cleanups.push(() => small.llmock.stop()); + const smallChunks = await streamChunkCount(small.url); + + const large = await startFromConfig({ llm: { fixtures: fixturePath, chunkSize: 1000 } }); + cleanups.push(() => large.llmock.stop()); + const largeChunks = await streamChunkCount(large.url); + + expect(smallChunks).toBeGreaterThan(largeChunks); + }); + + it("with llm.latency, streamed chunks are delayed", async () => { + const fixturePath = writeStreamFixtureFile(tmpDir); + const config: AimockConfig = { + llm: { fixtures: fixturePath, chunkSize: 5, latency: 100 }, + }; + const { llmock, url } = await startFromConfig(config); + cleanups.push(() => llmock.stop()); + + const start = Date.now(); + const chunks = await streamChunkCount(url); + const elapsed = Date.now() - start; + + // 100ms per chunk across >= 5 chunks. Asserted as a lower bound so a slow + // machine can never fail it; without the passthrough latency is 0. + expect(chunks).toBeGreaterThanOrEqual(5); + expect(elapsed).toBeGreaterThanOrEqual(250); + }); + + it("with llm.replaySpeed, recorded timings replay faster", async () => { + // 500ms TTFT + 4 x 500ms inter-chunk = ~2500ms at 1x speed + const fixturePath = writeStreamFixtureFile(tmpDir, { + ttftMs: 500, + interChunkDelaysMs: [500, 500, 500, 500], + totalDurationMs: 2500, + }); + const config: AimockConfig = { + llm: { fixtures: fixturePath, chunkSize: 5, replaySpeed: 100 }, + }; + const { llmock, url } = await startFromConfig(config); + cleanups.push(() => llmock.stop()); + + const start = Date.now(); + await streamChunkCount(url); + const elapsed = Date.now() - start; + + // At 100x this lands in ~25ms; the bound is deliberately generous so only a + // dropped passthrough (which costs the full ~2500ms) can fail it. + expect(elapsed).toBeLessThan(1000); + }); + + it("with non-positive llm.replaySpeed, warns and ignores it", async () => { + const fixturePath = writeStreamFixtureFile(tmpDir); + // Spy on the logger the code path actually uses rather than console, so the + // assertion verifies the intended warn call without coupling to the + // "[aimock]" prefix or the console transport. + const warn = vi.spyOn(Logger.prototype, "warn").mockImplementation(() => {}); + + // 0 looks like "no delay" but hits the `speed > 0` guard in calculateDelay, + // which would replay at full recorded speed. + const { llmock } = await startFromConfig({ + llm: { fixtures: fixturePath, replaySpeed: 0 }, + }); + cleanups.push(() => llmock.stop()); + + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("llm.replaySpeed must be a positive number"), + ); + warn.mockRestore(); + }); + + it("with llm.logLevel debug, the server logs debug output that silent suppresses", async () => { + const fixturePath = writeFixtureFile(tmpDir); + // The server logger writes debug/info lines via console.log; capture them so + // we can assert the configured level actually reaches the running server. + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + + // A matched request emits a "Fixture matched" debug line, but only when the + // configured logLevel is high enough — so it is a direct probe of the passthrough. + async function debugLineCount(logLevel: "debug" | "silent"): Promise { + log.mockClear(); + const { llmock, url } = await startFromConfig({ + llm: { fixtures: fixturePath, logLevel }, + }); + cleanups.push(() => llmock.stop()); + await fetch(`${url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model: "gpt-4", messages: [{ role: "user", content: "hello" }] }), + }); + return log.mock.calls.filter((args) => String(args[1]).startsWith("Fixture matched")).length; + } + + expect(await debugLineCount("debug")).toBeGreaterThan(0); + expect(await debugLineCount("silent")).toBe(0); + + log.mockRestore(); + }); + it("with mcp tools config, MCPMock created and tools/list works", async () => { const config: AimockConfig = { mcp: { @@ -649,11 +794,8 @@ describe("startFromConfig", () => { { pattern: "stream", events: [ - { - kind: "status-update", - taskId: "t1", - status: { state: "working", message: { parts: [{ text: "streaming..." }] } }, - }, + { type: "status", state: "TASK_STATE_WORKING" }, + { type: "artifact", parts: [{ text: "streaming..." }], name: "out" }, ], delayMs: 0, }, @@ -670,6 +812,28 @@ describe("startFromConfig", () => { expect(cardRes.status).toBe(200); const card = await cardRes.json(); expect(card.name).toBe("stream-agent"); + + // Verify the streaming task actually streams the configured events over SSE + const streamRes = await fetch(`${url}/a2a`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + jsonrpc: "2.0", + method: "SendStreamingMessage", + params: { message: { parts: [{ text: "please stream" }] } }, + id: 1, + }), + }); + expect(streamRes.status).toBe(200); + expect(streamRes.headers.get("content-type")).toBe("text/event-stream"); + const streamBody = await streamRes.text(); + const events = streamBody + .split("\n\n") + .filter((frame) => frame.startsWith("data: ")) + .map((frame) => JSON.parse(frame.slice("data: ".length))); + expect(events.length).toBe(2); + expect(events[0].result.task.status.state).toBe("TASK_STATE_WORKING"); + expect(events[1].result.artifact.parts[0].text).toBe("streaming..."); }); it("with a2a custom path, mounts at specified path for tasks", async () => { diff --git a/src/__tests__/drift-collector.test.ts b/src/__tests__/drift-collector.test.ts index d749934b..59c5e1fb 100644 --- a/src/__tests__/drift-collector.test.ts +++ b/src/__tests__/drift-collector.test.ts @@ -1,40 +1,64 @@ /** - * Tests for key functions in scripts/drift-report-collector.ts + * Tests for the drift-report collector's pure functions. * - * Since scripts/ is outside the rootDir for the main tsconfig (and vitest - * only covers src/__tests__), these functions are duplicated here as local - * test helpers to keep the test runner config intact. Any changes to the - * originals must be reflected here. + * These tests import and exercise the REAL exported functions from + * scripts/drift-report-collector.ts — NOT local reimplementations. Importing + * the module does NOT run main(): the collector guards its entry point with an + * `isDirectRun()` check (main() only fires under `tsx scripts/…` invocation), + * so importing here is side-effect-free. + * + * The canary fixtures below are REAL vitest failure-message shapes captured by + * running the canary / drift / infra assertions under + * `vitest run … --reporter=json` (see PR #291 RED/GREEN logs under tmp/). They + * are NOT hand-authored: the single-glyph Unicode ellipsis `…(N)`, the + * `AssertionError:` prefix, the leading blank line before a formatted drift + * report, and the stack-frame layout are exactly what vitest emits. */ import { describe, it, expect } from "vitest"; import { formatDriftReport } from "./drift/schema.js"; import type { ShapeDiff } from "./drift/schema.js"; +import { + parseDriftBlock, + extractProviderName, + extractScenario, + parseKnownModelsCanary, + collectDriftEntries, + computeExitCode, + classifyUnparseableAsInfra, + INFRA_INDICATOR_SOURCES, + infraIndicatorSample, +} from "../../scripts/drift-report-collector.js"; +import type { DriftEntry, QuarantineEntry } from "../../scripts/drift-types.js"; // --------------------------------------------------------------------------- -// Local copies of the types and functions under test -// (mirrors scripts/drift-report-collector.ts — keep in sync) +// Helpers for the A1.3 CollectResult shape ({ entries, quarantine }). +// collectDriftEntries no longer returns a bare array nor throws on unmapped / +// unparseable-not-infra failures — those are quarantined (→ exit 5). // --------------------------------------------------------------------------- -type DriftSeverity = "critical" | "warning" | "info"; - -interface ParsedDiff { - path: string; - severity: DriftSeverity; - issue: string; - expected: string; - real: string; - mock: string; +function entriesOf(result: VitestJsonResult): DriftEntry[] { + return collectDriftEntries(result).entries; } -interface VitestJsonResult { - testResults: VitestTestFile[]; +function quarantineOf(result: VitestJsonResult): QuarantineEntry[] { + return collectDriftEntries(result).quarantine; } -interface VitestTestFile { - assertionResults: VitestAssertion[]; +/** The exit code main() would emit for a given collect result (agUiSkipped=false). */ +function exitCodeOf(result: VitestJsonResult): 0 | 1 | 2 | 5 { + const { entries, quarantine } = collectDriftEntries(result); + const criticalCount = entries.reduce( + (sum, e) => sum + e.diffs.filter((d) => d.severity === "critical").length, + 0, + ); + return computeExitCode(criticalCount, quarantine.length, false); } +// --------------------------------------------------------------------------- +// Vitest JSON reporter fixture builders +// --------------------------------------------------------------------------- + interface VitestAssertion { status: string; ancestorTitles: string[]; @@ -42,226 +66,10 @@ interface VitestAssertion { failureMessages: string[]; } -interface ProviderMapping { - builderFile: string; - builderFunctions: string[]; - typesFile: string | null; - sdkShapesFile?: string; -} - -const PROVIDER_MAP: Record = { - "OpenAI Chat": { - builderFile: "src/helpers.ts", - builderFunctions: [ - "buildTextCompletion", - "buildToolCallCompletion", - "buildTextChunks", - "buildToolCallChunks", - ], - typesFile: "src/types.ts", - }, - "OpenAI Responses": { - builderFile: "src/responses.ts", - builderFunctions: [ - "buildTextResponse", - "buildToolCallResponse", - "buildTextStreamEvents", - "buildToolCallStreamEvents", - ], - typesFile: null, - }, - Anthropic: { - builderFile: "src/messages.ts", - builderFunctions: [ - "buildClaudeTextResponse", - "buildClaudeToolCallResponse", - "buildClaudeTextStreamEvents", - "buildClaudeToolCallStreamEvents", - ], - typesFile: null, - }, - "Anthropic Claude": { - builderFile: "src/messages.ts", - builderFunctions: [ - "buildClaudeTextResponse", - "buildClaudeToolCallResponse", - "buildClaudeTextStreamEvents", - "buildClaudeToolCallStreamEvents", - ], - typesFile: null, - }, - "Google Gemini": { - builderFile: "src/gemini.ts", - builderFunctions: [ - "buildGeminiTextResponse", - "buildGeminiToolCallResponse", - "buildGeminiTextStreamChunks", - "buildGeminiToolCallStreamChunks", - ], - typesFile: null, - }, - Gemini: { - builderFile: "src/gemini.ts", - builderFunctions: [ - "buildGeminiTextResponse", - "buildGeminiToolCallResponse", - "buildGeminiTextStreamChunks", - "buildGeminiToolCallStreamChunks", - ], - typesFile: null, - }, - "OpenAI Realtime": { - builderFile: "src/ws-realtime.ts", - builderFunctions: ["handleWebSocketRealtime", "realtimeItemsToMessages"], - typesFile: null, - }, - "OpenAI Responses WS": { - builderFile: "src/ws-responses.ts", - builderFunctions: ["handleWebSocketResponses"], - typesFile: null, - }, - "Gemini Live": { - builderFile: "src/ws-gemini-live.ts", - builderFunctions: ["handleWebSocketGeminiLive"], - typesFile: null, - }, - "OpenAI Embeddings": { - builderFile: "src/helpers.ts", - builderFunctions: ["buildEmbeddingResponse", "generateDeterministicEmbedding"], - typesFile: null, - sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", - }, - "Gemini Interactions": { - builderFile: "src/gemini-interactions.ts", - builderFunctions: [ - "buildInteractionsTextResponse", - "buildInteractionsToolCallResponse", - "buildInteractionsContentWithToolCallsResponse", - "buildInteractionsTextSSEEvents", - "buildInteractionsToolCallSSEEvents", - "buildInteractionsContentWithToolCallsSSEEvents", - ], - typesFile: null, - }, -}; - -const SDK_SHAPES_FILE = "src/__tests__/drift/sdk-shapes.ts"; - -const VALID_SEVERITIES = new Set(["critical", "warning", "info"]); - -function parseDriftBlock(text: string): { context: string; diffs: ParsedDiff[] } | null { - const headerMatch = text.match(/API DRIFT DETECTED:\s*(.+)/); - if (!headerMatch) return null; - - const context = headerMatch[1].trim(); - const diffs: ParsedDiff[] = []; - - const entryPattern = - /\d+\.\s*\[(\w+)\]\s*(.+)\n\s*Path:\s*(.+)\n\s*SDK:\s*(.+)\n\s*Real:\s*(.+)\n\s*Mock:\s*(.+)/g; - - let match: RegExpExecArray | null; - while ((match = entryPattern.exec(text)) !== null) { - const severity = match[1].trim(); - if (!VALID_SEVERITIES.has(severity as DriftSeverity)) continue; - diffs.push({ - severity: severity as DriftSeverity, - issue: match[2].trim(), - path: match[3].trim(), - expected: match[4].trim(), - real: match[5].trim(), - mock: match[6].trim(), - }); - } - - return { context, diffs }; -} - -function extractProviderName(text: string): string | null { - const sorted = Object.keys(PROVIDER_MAP).sort((a, b) => b.length - a.length); - for (const key of sorted) { - if (text.includes(key)) return key; - } - return null; -} - -function extractScenario(context: string): string { - const parenMatch = context.match(/\(([^)]+)\)/); - return parenMatch ? parenMatch[1] : context; -} - -function collectDriftEntries(results: VitestJsonResult): Array<{ - provider: string; - scenario: string; - builderFile: string; - builderFunctions: string[]; - typesFile: string | null; - sdkShapesFile: string; - diffs: ParsedDiff[]; -}> { - const entries: Array<{ - provider: string; - scenario: string; - builderFile: string; - builderFunctions: string[]; - typesFile: string | null; - sdkShapesFile: string; - diffs: ParsedDiff[]; - }> = []; - const unmapped: string[] = []; - let unparseable = 0; - - for (const file of results.testResults) { - for (const assertion of file.assertionResults) { - if (assertion.status !== "failed") continue; - if (assertion.failureMessages.length === 0) continue; - - const fullMessage = assertion.failureMessages.join("\n"); - const parsed = parseDriftBlock(fullMessage); - if (!parsed || parsed.diffs.length === 0) { - unparseable++; - continue; - } - - const ancestorText = assertion.ancestorTitles.join(" "); - const provider = extractProviderName(ancestorText) ?? extractProviderName(parsed.context); - if (!provider) { - unmapped.push(`${ancestorText} > ${assertion.title}`); - continue; - } - - const mapping = PROVIDER_MAP[provider]; - if (!mapping) { - unmapped.push(`${ancestorText} > ${assertion.title} (provider: ${provider})`); - continue; - } - - entries.push({ - provider, - scenario: extractScenario(parsed.context), - builderFile: mapping.builderFile, - builderFunctions: mapping.builderFunctions, - typesFile: mapping.typesFile, - sdkShapesFile: SDK_SHAPES_FILE, - diffs: parsed.diffs, - }); - } - } - - if (unmapped.length > 0) { - throw new Error(`${unmapped.length} unmapped drift entries — update PROVIDER_MAP`); - } - - if (unparseable > 0 && entries.length === 0) { - throw new Error(`${unparseable} unparseable test failures with 0 drift entries — investigate`); - } - - return entries; +interface VitestJsonResult { + testResults: { assertionResults: VitestAssertion[] }[]; } -// --------------------------------------------------------------------------- -// Helpers for building test fixtures -// --------------------------------------------------------------------------- - function makeResult(assertions: VitestAssertion[]): VitestJsonResult { return { testResults: [{ assertionResults: assertions }] }; } @@ -295,7 +103,87 @@ const SAMPLE_DIFF_WARNING: ShapeDiff = { }; // --------------------------------------------------------------------------- -// parseDriftBlock tests +// REAL captured vitest --reporter=json failure-message fixtures. +// Captured via throwaway `*.drift.ts` capture tests run under the drift config; +// see tmp/canary-fixtures.json + the PR #291 RED/GREEN logs. +// --------------------------------------------------------------------------- + +// Canary tripped with FOUR unknown models. The printed array is truncated by +// vitest to `…(3)` (single-glyph Unicode ellipsis), but the custom assertion +// message `UNKNOWN_REALTIME_MODELS=…` carries the full list verbatim. +// NOTE (A4): the ids below are HYPOTHETICAL future models that are NOT in the +// knownModels set in ws-realtime.drift.ts — so the real canary really could +// emit them as unknown. (gpt-realtime-2.1 / -2.1-mini ARE in knownModels and +// therefore can never appear here — the earlier fixture that used them was +// impossible.) +const CANARY_MARKER_MULTI = + "AssertionError: UNKNOWN_REALTIME_MODELS=gpt-realtime-3,gpt-realtime-3-mini,gpt-realtime-3-preview,gpt-realtime-ultra: expected [ 'gpt-realtime-3', …(3) ] to deeply equal []\n" + + " at /repo/src/__tests__/drift/ws-realtime.drift.ts:108:69\n" + + " at file:///repo/node_modules/@vitest/runner/dist/chunk-hooks.js:155:11\n" + + " at processTicksAndRejections (node:internal/process/task_queues:104:5)"; + +// A GENUINE drift report carried inside an AssertionError. formatDriftReport +// prepends "\n", so line 0 is just "AssertionError: " and the "API DRIFT +// DETECTED" / "mismatch" markers live on LATER lines. This is a fully-formatted +// report body (parseDriftBlock parses it into one critical diff). +const GENUINE_DRIFT_WITH_STACK = + "AssertionError: \nAPI DRIFT DETECTED: OpenAI Chat (non-streaming text)\n\n" + + " 1. [critical] LLMOCK DRIFT — mismatch detected\n" + + " Path: choices[0].message.refusal\n" + + " SDK: null\n" + + " Real: null\n" + + " Mock: \n" + + ": expected [ Array(1) ] to deeply equal []\n" + + " at /repo/src/__tests__/drift/openai-chat.drift.ts:42:30\n" + + " at file:///repo/node_modules/@vitest/runner/dist/chunk-hooks.js:155:11"; + +// A genuinely-unparseable failure whose ONLY infra token (ECONNREFUSED) sits in +// a STACK FRAME; the assertion body is neutral (no infra token, no drift +// marker). This is the A3 asymmetry surface — a raw scan would see the frame +// token and wrongly swallow the failure; a stack-stripped scan does not. +const INFRA_TOKEN_IN_STACKFRAME_ONLY = + "AssertionError: expected 1 to be 2 // Object.is equality\n" + + " at ECONNREFUSED (/repo/src/__tests__/drift/some.drift.ts:8:13)\n" + + " at file:///repo/node_modules/@vitest/runner/dist/chunk-hooks.js:155:11"; + +// A genuine infra failure whose token is in the BODY (the well-behaved case). +const REAL_INFRA_BODY = "fetch failed\n at handler (file:///repo/src/x.drift.ts:5:1)"; + +// CLASS 2 — the canary `hasGA`-false mode. Captured REAL from a throwaway +// `*.drift.ts` capture run under the drift config (see tmp/captured-vitest-shapes.json). +// When OpenAI renames/removes the GA realtime family, the canary emits the +// NO_GA_REALTIME_MODELS= marker (symmetric to UNKNOWN_REALTIME_MODELS=) and the +// assertion fails with "expected false to be true". The collector must map this +// to a CRITICAL OpenAI-Realtime DriftEntry (exit-2), NOT crash to exit-1. +const CANARY_NO_GA_MARKER = + "AssertionError: NO_GA_REALTIME_MODELS=gpt-foo,gpt-bar | UNKNOWN_REALTIME_MODELS=: expected false to be true // Object.is equality\n" + + " at /repo/src/__tests__/drift/ws-realtime.drift.ts:96:44\n" + + " at file:///repo/node_modules/@vitest/runner/dist/chunk-hooks.js:155:11\n" + + " at processTicksAndRejections (node:internal/process/task_queues:104:5)"; + +// CLASS 2 combined case — the run is BOTH no-GA AND carries new unknown models. +// The hasGA assertion throws first (short-circuiting the unknown-models +// assertion), so the NO_GA marker carries BOTH lists. The collector must +// preserve the unknown list (no information loss into the auto-fix prompt). +const CANARY_NO_GA_WITH_UNKNOWN = + "AssertionError: NO_GA_REALTIME_MODELS=gpt-foo,gpt-bar | UNKNOWN_REALTIME_MODELS=gpt-realtime-99,gpt-realtime-99-mini: expected false to be true // Object.is equality\n" + + " at /repo/src/__tests__/drift/ws-realtime.drift.ts:96:44\n" + + " at file:///repo/node_modules/@vitest/runner/dist/chunk-hooks.js:155:11"; + +// A NO_GA marker with an empty observed list (key could not see ANY realtime +// models — still a critical signal that the GA family is unreachable/gone). +const CANARY_NO_GA_EMPTY = + "AssertionError: NO_GA_REALTIME_MODELS= | UNKNOWN_REALTIME_MODELS=: expected false to be true // Object.is equality\n" + + " at /repo/src/__tests__/drift/ws-realtime.drift.ts:96:44"; + +// CLASS 3 — a marker-less truncated canary array. The truncation fact must +// become a boolean flag, never a prose sentinel occupying a model-id slot. +const CANARY_FALLBACK_TRUNCATED = + "AssertionError: expected [ 'gpt-realtime-9', …(2) ] to deeply equal []\n" + + " at /repo/src/__tests__/drift/ws-realtime.drift.ts:108:69"; + +// --------------------------------------------------------------------------- +// parseDriftBlock // --------------------------------------------------------------------------- describe("parseDriftBlock", () => { @@ -337,7 +225,6 @@ describe("parseDriftBlock", () => { }); it("skips entries with unknown severity", () => { - // Manually construct a report with a bad severity const text = ` API DRIFT DETECTED: OpenAI Chat (test) @@ -355,7 +242,6 @@ API DRIFT DETECTED: OpenAI Chat (test) `; const result = parseDriftBlock(text); expect(result).not.toBeNull(); - // Only the critical entry should be in diffs expect(result!.diffs).toHaveLength(1); expect(result!.diffs[0].severity).toBe("critical"); expect(result!.diffs[0].path).toBe("baz.qux"); @@ -401,7 +287,7 @@ API DRIFT DETECTED: OpenAI Chat (test) }); // --------------------------------------------------------------------------- -// extractProviderName tests +// extractProviderName // --------------------------------------------------------------------------- describe("extractProviderName", () => { @@ -412,7 +298,6 @@ describe("extractProviderName", () => { }); it("uses longest match — Anthropic Claude over Anthropic", () => { - // "Anthropic Claude" is longer and should win over "Anthropic" expect(extractProviderName("Anthropic Claude drift")).toBe("Anthropic Claude"); expect(extractProviderName("Anthropic Claude (streaming tool call)")).toBe("Anthropic Claude"); }); @@ -441,23 +326,44 @@ describe("extractProviderName", () => { }); // --------------------------------------------------------------------------- -// collectDriftEntries tests +// extractScenario +// --------------------------------------------------------------------------- + +describe("extractScenario", () => { + it("extracts the parenthetical scenario", () => { + expect(extractScenario("OpenAI Chat (non-streaming text)")).toBe("non-streaming text"); + expect(extractScenario("Anthropic Claude (streaming tool call)")).toBe("streaming tool call"); + }); + + it("returns the whole context when there is no parenthetical", () => { + expect(extractScenario("OpenAI Chat")).toBe("OpenAI Chat"); + }); +}); + +// --------------------------------------------------------------------------- +// collectDriftEntries (HTTP drift path) // --------------------------------------------------------------------------- describe("collectDriftEntries", () => { - it("returns empty array when no failed tests", () => { + it("returns empty entries+quarantine when no failed tests", () => { const result = makeResult([ makeAssertion({ status: "passed" }), makeAssertion({ status: "pending" }), ]); - expect(collectDriftEntries(result)).toEqual([]); + expect(entriesOf(result)).toEqual([]); + expect(quarantineOf(result)).toEqual([]); + expect(exitCodeOf(result)).toBe(0); }); - it("returns empty array when there are no test files at all", () => { - expect(collectDriftEntries({ testResults: [] })).toEqual([]); + it("returns empty entries+quarantine when there are no test files at all", () => { + const result: VitestJsonResult = { testResults: [] }; + expect(entriesOf(result)).toEqual([]); + expect(quarantineOf(result)).toEqual([]); }); - it("throws when an unmapped provider is found in drift report", () => { + it("QUARANTINES (does NOT throw) an unmapped provider found in a drift report → exit 5", () => { + // A1.3: an unmapped provider is untrusted, not a collector crash. It is held + // for review (exit 5) instead of throwing (was exit 1). const driftText = formatDriftReport("UnknownProvider (non-streaming text)", [SAMPLE_DIFF]); const result = makeResult([ makeAssertion({ @@ -466,21 +372,43 @@ describe("collectDriftEntries", () => { failureMessages: [driftText], }), ]); - expect(() => collectDriftEntries(result)).toThrow(/unmapped drift entries/); + expect(() => collectDriftEntries(result)).not.toThrow(); + const q = quarantineOf(result); + expect(q).toHaveLength(1); + expect(entriesOf(result)).toEqual([]); + expect(exitCodeOf(result)).toBe(5); }); - it("throws when all failures are unparseable and no drift entries collected", () => { + it("QUARANTINES (does NOT throw) all-unparseable-not-infra failures → exit 5 (incident-5)", () => { + // A1.3: the incident-5 surface. Genuine-but-unparseable failures are no + // longer a fail-loud crash (exit 1) — they are quarantined (exit 5) so they + // surface for review without being swallowed. const result = makeResult([ makeAssertion({ status: "failed", - failureMessages: ["Error: expected true to equal false\n at Object."], + ancestorTitles: ["Some Suite"], + title: "a", + failureMessages: [ + "AssertionError: expected 1 to be 2 // Object.is equality\n at foo (/repo/src/__tests__/drift/some.drift.ts:8:13)", + ], }), makeAssertion({ status: "failed", - failureMessages: ["TypeError: Cannot read property 'foo' of undefined"], + ancestorTitles: ["Other Suite"], + title: "b", + failureMessages: [ + "TypeError: Cannot read property 'foo' of undefined\n at bar (/repo/src/__tests__/drift/other.drift.ts:3:1)", + ], }), ]); - expect(() => collectDriftEntries(result)).toThrow(/unparseable test failures/); + expect(() => collectDriftEntries(result)).not.toThrow(); + const q = quarantineOf(result); + expect(q).toHaveLength(2); + // O-1: raw file:line captured from the stack frame BEFORE stripping. + expect(q[0].rawLocation).toBe("/repo/src/__tests__/drift/some.drift.ts:8:13"); + expect(q[1].rawLocation).toBe("/repo/src/__tests__/drift/other.drift.ts:3:1"); + expect(entriesOf(result)).toEqual([]); + expect(exitCodeOf(result)).toBe(5); }); it("returns valid entries and tolerates unparseable failures mixed in", () => { @@ -500,13 +428,19 @@ describe("collectDriftEntries", () => { }), ]); - const entries = collectDriftEntries(result); + const entries = entriesOf(result); expect(entries).toHaveLength(1); expect(entries[0].provider).toBe("OpenAI Chat"); expect(entries[0].scenario).toBe("non-streaming text"); expect(entries[0].builderFile).toBe("src/helpers.ts"); expect(entries[0].diffs).toHaveLength(1); expect(entries[0].diffs[0].severity).toBe("critical"); + // When real drift entries ARE present, a mixed-in unparseable sibling stays + // TOLERATED (warn-only) rather than quarantined — the quarantine path only + // fires for the all-unparseable, zero-entries case (former throw site). + expect(quarantineOf(result)).toEqual([]); + // A critical entry present → exit 2 (dominates any tolerated sibling). + expect(exitCodeOf(result)).toBe(2); }); it("ignores passed assertions in a mixed result set", () => { @@ -521,7 +455,7 @@ describe("collectDriftEntries", () => { }), ]); - const entries = collectDriftEntries(result); + const entries = entriesOf(result); expect(entries).toHaveLength(1); expect(entries[0].provider).toBe("OpenAI Chat"); }); @@ -555,9 +489,863 @@ describe("collectDriftEntries", () => { ], }; - const entries = collectDriftEntries(results); + const entries = entriesOf(results); expect(entries).toHaveLength(2); expect(entries[0].provider).toBe("OpenAI Chat"); expect(entries[1].provider).toBe("Google Gemini"); }); + + // ------------------------------------------------------------------------- + // INTEGRATION: the canary → critical → exit-2 contract, end to end through + // the REAL collectDriftEntries. This is the whole reason PR #291 exists. + // ------------------------------------------------------------------------- + + it("emits ONE critical DriftEntry carrying the FULL unknown-model list from a real canary failure (RED without the marker fix)", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "canary: GA realtime models available", + failureMessages: [CANARY_MARKER_MULTI], + }), + ]); + + const entries = entriesOf(result); + expect(entries).toHaveLength(1); + + const entry = entries[0]; + expect(entry.provider).toBe("OpenAI Realtime"); + expect(entry.scenario).toBe("known-models canary"); + expect(entry.builderFile).toBe("src/ws-realtime.ts"); + + // FULL list recovered from the marker (NOT truncated to just the first id). + const reals = entry.diffs.map((d) => d.real); + expect(reals).toEqual([ + "gpt-realtime-3", + "gpt-realtime-3-mini", + "gpt-realtime-3-preview", + "gpt-realtime-ultra", + ]); + + // Every diff is critical so the collector exits 2 and the Fix Drift + // workflow reaches the auto-fix step. + expect(entry.diffs.every((d) => d.severity === "critical")).toBe(true); + // Real-API-only canary: the model id must NOT be mislabeled as a mock value. + for (const d of entry.diffs) { + expect(d.mock).not.toBe(d.real); + expect(d.mock).toContain("no mock leg"); + } + + // The exit-2 gate condition (criticalCount > 0) that main() checks. + const criticalCount = entries.reduce( + (sum, e) => sum + e.diffs.filter((d) => d.severity === "critical").length, + 0, + ); + expect(criticalCount).toBe(4); + }); + + it("does NOT misattribute a non-canary toEqual([]) failure from another provider as OpenAI-Realtime drift → quarantine (exit 5)", () => { + // A different provider's test failed with the generic vitest shape + // `expected [ 'sk-leaked' ] to deeply equal []`. Pre-fix, the unguarded + // canary fallback matched this and emitted a CRITICAL "OpenAI Realtime + // known-models canary" entry with `real: 'sk-leaked'`, pointing the auto-fix + // at src/ws-realtime.ts and relabeling arbitrary array contents as a model + // id. It must NOT be claimed as a canary. A1.3: because the message is + // neither a parseable drift block, a canary, nor infra, it is QUARANTINED + // (exit 5) — never fabricated into a false entry, never silently dropped, + // and (A1.3) no longer a fail-loud crash. + const NON_CANARY_TOEQUAL_EMPTY = + "AssertionError: expected [ 'sk-leaked' ] to deeply equal []\n" + + " at /repo/src/__tests__/drift/openai-chat.drift.ts:42:30\n" + + " at file:///repo/node_modules/@vitest/runner/dist/chunk-hooks.js:155:11"; + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + title: "non-streaming text matches real API", + failureMessages: [NON_CANARY_TOEQUAL_EMPTY], + }), + ]); + + expect(() => collectDriftEntries(result)).not.toThrow(); + // No OpenAI-Realtime (nor any) entry may be produced. + expect(entriesOf(result)).toEqual([]); + expect(quarantineOf(result)).toHaveLength(1); + // The arbitrary array content is NOT relabeled as a model id anywhere. + expect(quarantineOf(result)[0].message).toContain("sk-leaked"); + expect(exitCodeOf(result)).toBe(5); + }); + + it("surfaces a genuine drift report carried in an AssertionError with a leading blank line (does not swallow)", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + title: "non-streaming text matches real API", + failureMessages: [GENUINE_DRIFT_WITH_STACK], + }), + ]); + + const entries = entriesOf(result); + expect(entries).toHaveLength(1); + expect(entries[0].provider).toBe("OpenAI Chat"); + expect(entries[0].diffs).toHaveLength(1); + expect(entries[0].diffs[0].severity).toBe("critical"); + }); + + it("does NOT exit 0 (quarantines → exit 5) when the only failure has an infra token confined to a stack frame (A3)", () => { + // The raw-vs-stripped asymmetry classified this as benign infra and swallowed + // it (returned []). The fix normalizes both scans, so an infra token that + // survives ONLY in a stripped-away stack frame no longer flips the gate — the + // failure is surfaced. A1.3: surfaced now means QUARANTINE (exit 5), not a + // crash; the invariant that matters is it is NEVER a green (exit 0). + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "canary: GA realtime models available", + failureMessages: [INFRA_TOKEN_IN_STACKFRAME_ONLY], + }), + ]); + expect(() => collectDriftEntries(result)).not.toThrow(); + expect(quarantineOf(result)).toHaveLength(1); + expect(exitCodeOf(result)).toBe(5); + expect(exitCodeOf(result)).not.toBe(0); + }); + + // ------------------------------------------------------------------------- + // CLASS 2 — hasGA-false canary maps to a CRITICAL OpenAI-Realtime entry + // (exit-2 path) instead of crashing to exit-1. + // ------------------------------------------------------------------------- + it("maps a NO_GA_REALTIME_MODELS canary failure to a CRITICAL OpenAI-Realtime entry (exit-2, not a throw)", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "canary: GA realtime models available", + failureMessages: [CANARY_NO_GA_MARKER], + }), + ]); + + const entries = entriesOf(result); + expect(entries).toHaveLength(1); + const entry = entries[0]; + expect(entry.provider).toBe("OpenAI Realtime"); + expect(entry.builderFile).toBe("src/ws-realtime.ts"); + expect(entry.diffs.length).toBeGreaterThan(0); + expect(entry.diffs.every((d) => d.severity === "critical")).toBe(true); + + const criticalCount = entries.reduce( + (sum, e) => sum + e.diffs.filter((d) => d.severity === "critical").length, + 0, + ); + expect(criticalCount).toBeGreaterThan(0); + }); + + it("preserves the unknown-model list in the NO_GA entry (no info loss when both fire)", () => { + // A run that is BOTH no-GA AND has new unknown models must surface the + // unknown ids as critical diffs, not lose them to the short-circuited + // unknown-models assertion. + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "canary: GA realtime models available", + failureMessages: [CANARY_NO_GA_WITH_UNKNOWN], + }), + ]); + const entries = entriesOf(result); + expect(entries).toHaveLength(1); + const entry = entries[0]; + expect(entry.provider).toBe("OpenAI Realtime"); + expect(entry.diffs.every((d) => d.severity === "critical")).toBe(true); + // The unknown model ids survive as `real` values on knownModels diffs. + const knownModelReals = entry.diffs.filter((d) => d.path === "knownModels").map((d) => d.real); + expect(knownModelReals).toEqual(["gpt-realtime-99", "gpt-realtime-99-mini"]); + // The GA-family diff is still present. + expect(entry.diffs.some((d) => d.path === "gaModels")).toBe(true); + }); + + it("maps an EMPTY NO_GA marker (no realtime models observed) to a CRITICAL entry too", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "canary: GA realtime models available", + failureMessages: [CANARY_NO_GA_EMPTY], + }), + ]); + const entries = entriesOf(result); + expect(entries).toHaveLength(1); + expect(entries[0].provider).toBe("OpenAI Realtime"); + expect(entries[0].diffs.every((d) => d.severity === "critical")).toBe(true); + }); + + // ------------------------------------------------------------------------- + // CLASS 3 — no DriftEntry.real is a non-model prose sentinel, even when the + // canary array was truncated in CI output. + // ------------------------------------------------------------------------- + it("never lands a prose sentinel in DriftEntry.real when the canary array is truncated (CLASS 3)", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "canary: GA realtime models available", + failureMessages: [CANARY_FALLBACK_TRUNCATED], + }), + ]); + const entries = entriesOf(result); + expect(entries).toHaveLength(1); + for (const d of entries[0].diffs) { + // No `real` value may be a prose annotation (e.g. "(additional models…)"). + expect(d.real.startsWith("(")).toBe(false); + } + }); + + // ------------------------------------------------------------------------- + // CLASS 1 — corpus/table test asserting the SAFE outcome for the recurring + // classifier failure shapes. A1.3 replaces the old binary throws/no-throw + // with a three-way `outcome`: + // "entry" → surfaced as a structured drift entry (exit 2), NEVER a + // silent exit-0; + // "quarantine" → held for human review (exit 5), NEVER a crash and NEVER a + // silent exit-0 (was: fail-loud throw / exit 1); + // "infra" → benign infra, collector returns [] (exit 0). + // The invariant the corpus protects: an untrusted failure is never a green. + // ------------------------------------------------------------------------- + describe("CLASS 1 safe-outcome corpus", () => { + const DRIFT_VALUE_WITH_STATUS_200 = formatDriftReport("OpenAI Chat (non-streaming text)", [ + { + path: "choices[0].message.content", + severity: "critical", + issue: "LLMOCK DRIFT — value mismatch", + expected: "status 200", + real: "status 200", + mock: "", + }, + ]); + + const rows: { + name: string; + messages: string[]; + outcome: "entry" | "quarantine" | "infra"; + }[] = [ + { + name: "a drift body containing the substring 'status 200' is surfaced as an entry, NOT swallowed as infra", + messages: [DRIFT_VALUE_WITH_STATUS_200], + outcome: "entry", + }, + { + name: "a genuine drift report with a leading blank line is surfaced as an entry", + messages: [GENUINE_DRIFT_WITH_STACK], + outcome: "entry", + }, + { + name: "an 'expected false to be true' hasGA shape is surfaced as a (canary) entry, not swallowed", + messages: [CANARY_NO_GA_MARKER], + outcome: "entry", + }, + { + name: "an 'expected […] to deeply equal []' canary shape is surfaced as an entry, not swallowed", + messages: [CANARY_MARKER_MULTI], + outcome: "entry", + }, + { + name: "a bare AssertionError with no infra token and no drift marker is quarantined (exit 5), not swallowed", + messages: ["AssertionError: expected 1 to be 2 // Object.is equality\n at foo (x:1:1)"], + outcome: "quarantine", + }, + { + name: "an infra token confined to a stack frame is quarantined (exit 5) (A3)", + messages: [INFRA_TOKEN_IN_STACKFRAME_ONLY], + outcome: "quarantine", + }, + ]; + + for (const row of rows) { + it(row.name, () => { + const result = makeResult( + row.messages.map((m) => + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "canary: GA realtime models available", + failureMessages: [m], + }), + ), + ); + expect(() => collectDriftEntries(result)).not.toThrow(); + const { entries, quarantine } = collectDriftEntries(result); + if (row.outcome === "entry") { + expect(entries.length).toBeGreaterThan(0); + expect(exitCodeOf(result)).toBe(2); + } else if (row.outcome === "quarantine") { + expect(entries).toEqual([]); + expect(quarantine.length).toBeGreaterThan(0); + expect(exitCodeOf(result)).toBe(5); + expect(exitCodeOf(result)).not.toBe(0); + } else { + expect(entries).toEqual([]); + expect(quarantine).toEqual([]); + expect(exitCodeOf(result)).toBe(0); + } + }); + } + + it("still classifies genuine infra (body token) as benign — collector returns [] entries+quarantine (exit 0)", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + title: "non-streaming text matches real API", + failureMessages: [REAL_INFRA_BODY], + }), + ]); + expect(entriesOf(result)).toEqual([]); + expect(quarantineOf(result)).toEqual([]); + expect(exitCodeOf(result)).toBe(0); + }); + }); + + // ------------------------------------------------------------------------- + // A1.4 TAXONOMY — the exit-code taxonomy end-to-end through the REAL collector + // + computeExitCode. Each row asserts the full path from a vitest failure + // shape to the process exit code main() would emit. + // ------------------------------------------------------------------------- + describe("exit-code taxonomy (collector → computeExitCode)", () => { + it("critical drift → exit 2", () => { + const driftText = formatDriftReport("OpenAI Chat (non-streaming text)", [SAMPLE_DIFF]); + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + failureMessages: [driftText], + }), + ]); + expect(entriesOf(result)).toHaveLength(1); + expect(quarantineOf(result)).toEqual([]); + expect(exitCodeOf(result)).toBe(2); + }); + + it("incident-5 unparseable failure → quarantine + exit 5 (NOT a throw)", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["Broken Suite"], + title: "a", + failureMessages: [ + "AssertionError: expected 1 to be 2 // Object.is equality\n at foo (/repo/src/__tests__/drift/x.drift.ts:8:13)", + ], + }), + ]); + expect(() => collectDriftEntries(result)).not.toThrow(); + expect(entriesOf(result)).toEqual([]); + expect(quarantineOf(result)).toHaveLength(1); + expect(exitCodeOf(result)).toBe(5); + }); + + it("all-infra failures → exit 0 (benign, collector returns nothing)", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + failureMessages: [REAL_INFRA_BODY], + }), + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Responses drift"], + failureMessages: [ + "INFRA_ERROR: upstream down\n at h (file:///repo/src/y.drift.ts:1:1)", + ], + }), + ]); + expect(entriesOf(result)).toEqual([]); + expect(quarantineOf(result)).toEqual([]); + expect(exitCodeOf(result)).toBe(0); + }); + + it("canary (unknown-model) failure → critical entry + exit 2", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "canary: GA realtime models available", + failureMessages: [CANARY_MARKER_MULTI], + }), + ]); + const entries = entriesOf(result); + expect(entries).toHaveLength(1); + expect(entries[0].diffs.every((d) => d.severity === "critical")).toBe(true); + expect(exitCodeOf(result)).toBe(2); + }); + + it("empty → fail-loud invariant: an all-unparseable batch is NEVER classified as a benign all-clear", () => { + // CLASS 1 root invariant surfaced at the collector: an empty evidence set + // (no positive infra evidence) must NOT be treated as infra. The batch is + // quarantined (exit 5), never a silent exit 0. + expect(classifyUnparseableAsInfra([])).toBe(false); + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["Broken Suite"], + title: "unrecognized", + failureMessages: [ + "AssertionError: expected 1 to be 2\n at foo (/repo/src/z.drift.ts:1:1)", + ], + }), + ]); + expect(exitCodeOf(result)).not.toBe(0); + expect(exitCodeOf(result)).toBe(5); + }); + + it("critical + quarantine together → exit 2 (critical dominates quarantine)", () => { + const driftText = formatDriftReport("OpenAI Chat (non-streaming text)", [SAMPLE_DIFF]); + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + title: "non-streaming text matches real API", + failureMessages: [driftText], + }), + // An unmapped-provider drift block → quarantined (never dropped) even + // though a real critical entry is also present. + makeAssertion({ + status: "failed", + ancestorTitles: ["UnknownProvider drift"], + title: "some scenario", + failureMessages: [formatDriftReport("UnknownProvider (streaming text)", [SAMPLE_DIFF])], + }), + ]); + const { entries, quarantine } = collectDriftEntries(result); + expect(entries).toHaveLength(1); + expect(entries[0].provider).toBe("OpenAI Chat"); + expect(quarantine).toHaveLength(1); + // Critical dominates: exit 2, not 5. + expect(exitCodeOf(result)).toBe(2); + }); + }); +}); + +// --------------------------------------------------------------------------- +// parseKnownModelsCanary +// --------------------------------------------------------------------------- + +describe("parseKnownModelsCanary", () => { + it("recovers the FULL unknown-model list from the UNKNOWN_REALTIME_MODELS marker (not truncated)", () => { + // The printed array is truncated to `…(3)` but the marker carries all four. + const result = parseKnownModelsCanary(CANARY_MARKER_MULTI); + expect(result).not.toBeNull(); + expect(result!.ids).toEqual([ + "gpt-realtime-3", + "gpt-realtime-3-mini", + "gpt-realtime-3-preview", + "gpt-realtime-ultra", + ]); + // CLASS 3: the marker carries the full list, so nothing is truncated and no + // prose sentinel may ever occupy an id slot. + expect(result!.truncated).toBeFalsy(); + expect(result!.ids.every((id) => !id.startsWith("("))).toBe(true); + }); + + it("returns null when the marker is present but the unknown list was empty", () => { + // Empty unknown list = no drift to surface. + const msg = "AssertionError: UNKNOWN_REALTIME_MODELS=: expected [] to deeply equal []"; + expect(parseKnownModelsCanary(msg)).toBeNull(); + }); + + it("falls through to the printed-array fallback when the marker value is mangled/empty (A2)", () => { + // A2: an empty/mangled marker must NOT short-circuit to null; it must fall + // through so a recoverable id in the printed array is still surfaced. + const msg = + "AssertionError: UNKNOWN_REALTIME_MODELS=: expected [ 'gpt-realtime-3', …(1) ] to deeply equal []"; + const result = parseKnownModelsCanary(msg); + expect(result).not.toBeNull(); + expect(result!.ids[0]).toBe("gpt-realtime-3"); + // CLASS 3: truncation is a boolean flag, NOT a prose id in the list. + expect(result!.truncated).toBe(true); + expect(result!.ids.every((id) => !id.startsWith("("))).toBe(true); + }); + + it("returns null for a non-canary message", () => { + expect(parseKnownModelsCanary("TypeError: something unrelated")).toBeNull(); + expect(parseKnownModelsCanary("")).toBeNull(); + }); + + describe("fallback (no marker — legacy message shape)", () => { + // NOTE: the fallback fires ONLY in a confirmed ws-realtime canary context. + // A REAL marker-less canary failure ALWAYS carries the canary's origin frame + // (`at …/ws-realtime.drift.ts`), which these fixtures include — that frame + // is the recognizer that distinguishes a genuine canary from a generic + // non-canary `toEqual([])` failure in some other provider's test. + const CANARY_ORIGIN = "\n at /repo/src/__tests__/drift/ws-realtime.drift.ts:108:69"; + + it("detects the single-glyph Unicode ellipsis `…(1)` truncation (as a flag, not a sentinel id)", () => { + const msg = + "AssertionError: expected [ 'gpt-realtime-3', …(1) ] to deeply equal []" + CANARY_ORIGIN; + const result = parseKnownModelsCanary(msg); + expect(result).not.toBeNull(); + expect(result!.ids[0]).toBe("gpt-realtime-3"); + // CLASS 3: no prose sentinel in the id list; truncation is a boolean. + expect(result!.truncated).toBe(true); + expect(result!.ids.every((id) => !id.startsWith("("))).toBe(true); + }); + + it("also detects the three-dot ASCII ellipsis `... (1)` truncation", () => { + const msg = + "AssertionError: expected [ 'gpt-realtime-3', ... (1) ] to deeply equal []" + CANARY_ORIGIN; + const result = parseKnownModelsCanary(msg); + expect(result!.truncated).toBe(true); + expect(result!.ids.every((id) => !id.startsWith("("))).toBe(true); + }); + + it("parses a small untruncated printed array", () => { + const msg = + "AssertionError: expected [ 'gpt-realtime-3', 'gpt-realtime-3-mini' ] to deeply equal []" + + CANARY_ORIGIN; + const result = parseKnownModelsCanary(msg); + expect(result!.ids).toEqual(["gpt-realtime-3", "gpt-realtime-3-mini"]); + expect(result!.truncated).toBeFalsy(); + }); + + it("returns null for an empty printed array (genuinely no unknown models)", () => { + const msg = "AssertionError: expected [] to deeply equal []"; + expect(parseKnownModelsCanary(msg)).toBeNull(); + }); + + it("flags truncation-only content (glyph present, no extractable id) without inventing a prose id", () => { + // Inner had a truncation glyph but no quoted ids we could extract. Carries + // the ws-realtime canary origin path so the fallback gate recognizes it as + // a genuine canary failure (a real canary failure ALWAYS carries this + // frame). Without a canary-origin token the fallback must NOT fire. + const msg = + "AssertionError: expected [ …(4) ] to deeply equal []\n" + + " at /repo/src/__tests__/drift/ws-realtime.drift.ts:108:69"; + const result = parseKnownModelsCanary(msg); + expect(result).not.toBeNull(); + // CLASS 3: no non-model prose id — the fact lives entirely in `truncated`. + expect(result!.ids.every((id) => !id.startsWith("("))).toBe(true); + expect(result!.truncated).toBe(true); + }); + }); + + // ------------------------------------------------------------------------- + // MISATTRIBUTION GUARD (bucket (a) finding): the printed-array fallback must + // fire ONLY for a genuine ws-realtime known-models canary failure. A generic + // `expected [...] to deeply equal []` from ANY OTHER provider/test (no + // realtime-canary marker AND not originating from ws-realtime.drift.ts) must + // NOT be claimed as OpenAI-Realtime known-models drift — its arbitrary array + // contents (which could be a leaked secret, an object shape, anything) must + // never be relabeled as "unknown model ids". + // ------------------------------------------------------------------------- + describe("fallback gating — non-canary toEqual([]) is NOT misattributed", () => { + it("returns null for a non-canary toEqual([]) failure carrying arbitrary array contents (RED before gate)", () => { + // A DIFFERENT provider's test asserted `toEqual([])` and the array held an + // arbitrary value — here a leaked-looking secret. NO realtime-canary marker + // and NO ws-realtime.drift.ts origin: this is not the canary and must not + // be parsed as one. + const msg = + "AssertionError: expected [ 'sk-leaked' ] to deeply equal []\n" + + " at /repo/src/__tests__/drift/openai-chat.drift.ts:42:30"; + expect(parseKnownModelsCanary(msg)).toBeNull(); + }); + + it("returns null for a bare non-canary toEqual([]) failure with no origin frame at all", () => { + const msg = "AssertionError: expected [ 'sk-leaked' ] to deeply equal []"; + expect(parseKnownModelsCanary(msg)).toBeNull(); + }); + + it("still parses a genuine marker-less canary failure that carries the ws-realtime.drift.ts origin", () => { + // No structured marker (mangled/stripped), but the stack frame identifies + // the canary — the fallback SHOULD still recover the id. + const result = parseKnownModelsCanary(CANARY_FALLBACK_TRUNCATED); + expect(result).not.toBeNull(); + expect(result!.ids[0]).toBe("gpt-realtime-9"); + expect(result!.truncated).toBe(true); + }); + }); + + // ------------------------------------------------------------------------- + // CLASS 2 — the NO_GA_REALTIME_MODELS marker (hasGA-false mode) + // ------------------------------------------------------------------------- + describe("NO_GA_REALTIME_MODELS marker (hasGA-false)", () => { + it("recognizes the marker and returns the observed model ids with noGA=true", () => { + const result = parseKnownModelsCanary(CANARY_NO_GA_MARKER); + expect(result).not.toBeNull(); + expect(result!.noGA).toBe(true); + expect(result!.ids).toEqual(["gpt-foo", "gpt-bar"]); + }); + + it("recognizes an EMPTY NO_GA marker (no realtime models observed at all) as noGA=true", () => { + const result = parseKnownModelsCanary(CANARY_NO_GA_EMPTY); + expect(result).not.toBeNull(); + expect(result!.noGA).toBe(true); + expect(result!.ids).toEqual([]); + expect(result!.unknownIds ?? []).toEqual([]); + }); + + it("preserves the unknown-model list carried alongside the NO_GA marker (info-loss fix)", () => { + // Combined case: no-GA AND new unknown models. The hasGA assertion + // short-circuits the unknown-models assertion, so the NO_GA marker carries + // BOTH lists. The observed and unknown lists must be split cleanly. + const result = parseKnownModelsCanary(CANARY_NO_GA_WITH_UNKNOWN); + expect(result).not.toBeNull(); + expect(result!.noGA).toBe(true); + expect(result!.ids).toEqual(["gpt-foo", "gpt-bar"]); + expect(result!.unknownIds).toEqual(["gpt-realtime-99", "gpt-realtime-99-mini"]); + }); + }); +}); + +// --------------------------------------------------------------------------- +// classifyUnparseableAsInfra (A3 — symmetric normalization safety net) +// --------------------------------------------------------------------------- + +describe("classifyUnparseableAsInfra", () => { + it("returns false for an EMPTY evidence array — no evidence is NOT proof of infra (CLASS 1)", () => { + // Vacuous `.every` on [] returns true; the fix must NOT treat "no evidence" + // as "all clear". Unrecognized ⇒ fail loud, never a false all-clear. + expect(classifyUnparseableAsInfra([])).toBe(false); + }); + + it("does NOT swallow a failure whose only infra token is confined to a stack frame (A3)", () => { + // Pre-fix: the raw scan saw ECONNREFUSED in the frame → allInfraErrors true + // → swallowed. The fix strips frames for BOTH scans, so the token is gone + // and the failure is not classified as infra. + expect(classifyUnparseableAsInfra([INFRA_TOKEN_IN_STACKFRAME_ONLY])).toBe(false); + }); + + it("does NOT swallow genuine drift carried in an AssertionError with a leading blank line", () => { + expect(classifyUnparseableAsInfra([GENUINE_DRIFT_WITH_STACK])).toBe(false); + }); + + it("does NOT treat a bare AssertionError as benign infra", () => { + const msg = "AssertionError: expected [ 'x' ] to deeply equal []\n at foo (file:///x)"; + expect(classifyUnparseableAsInfra([msg])).toBe(false); + }); + + it("still classifies genuine infra errors (token in the body) as infra", () => { + expect(classifyUnparseableAsInfra([REAL_INFRA_BODY])).toBe(true); + expect(classifyUnparseableAsInfra(["INFRA_ERROR: upstream down\n at foo (file:///x)"])).toBe( + true, + ); + expect(classifyUnparseableAsInfra(["API returned 503 Service Unavailable"])).toBe(true); + }); + + it("does NOT classify a drift body whose VALUE contains 'status 200' as infra (CLASS 1 anchoring)", () => { + // A real drift value like "status 200" appearing anywhere in the body must + // not trip the infra gate. The infra 'status \\d{3}' indicator must anchor + // to the failure reason/line, not a bare substring inside a drift value. + const msg = + "AssertionError: \nAPI DRIFT DETECTED: OpenAI Chat (non-streaming text)\n\n" + + " 1. [critical] LLMOCK DRIFT — mismatch detected\n" + + " Path: choices[0].message.content\n" + + " SDK: status 200\n" + + " Real: status 200\n" + + " Mock: \n"; + expect(classifyUnparseableAsInfra([msg])).toBe(false); + }); + + it("does NOT classify a labelled 'Real: API returned 503' drift VALUE as infra (CLASS 1 anchoring)", () => { + // Symmetric to the 'status 200' anchoring case above, and to the already- + // anchored 'status \\d{3}' sibling. A drift *value* like "API returned 503" + // appearing AFTER a `Field:` label must NOT trip the infra gate. The + // 'API returned \\d{3}' indicator must anchor to the failure reason/line + // (line start, optional `HTTP ` prefix) exactly like 'status \\d{3}' does — + // an anchoring-defeating `(?:.*:\\s*)?` prefix lets a labelled value match + // and silently swallow genuine drift. This message is deliberately NOT + // drift-like (no "drift"/"mismatch"/"expected…to" markers) so the + // anchoring of the infra indicator is the SOLE determinant of the outcome. + const msg = + " Path: choices[0].message.content\n" + + " SDK: n/a\n" + + " Real: API returned 503\n" + + " Mock: \n"; + expect(classifyUnparseableAsInfra([msg])).toBe(false); + }); + + it("still classifies a bare line-start 'API returned 503' reason as infra (anchoring preserved)", () => { + // The anchoring fix must NOT break the genuine infra case: a line whose + // reason IS "API returned " (optionally `HTTP `-prefixed, at line + // start) is still infra. Guards against over-tightening the anchor. + expect(classifyUnparseableAsInfra(["API returned 503 Service Unavailable"])).toBe(true); + expect(classifyUnparseableAsInfra([" HTTP API returned 500"])).toBe(true); + }); + + it("does not false-positive drift from a stack-trace filename like ws-realtime.drift.ts", () => { + // A recognized infra error (token in BODY) whose stack frame mentions + // "ws-realtime.drift.ts" stays infra — the frame filename is stripped. + const msg = "fetch failed\n at handler (file:///repo/src/ws-realtime.drift.ts:5:1)"; + expect(classifyUnparseableAsInfra([msg])).toBe(true); + }); + + // ------------------------------------------------------------------------- + // PROPERTY-BASED uniform-anchoring test — the NON-RECURRING deliverable. + // + // Iterates the REAL exported infra-indicator list (INFRA_INDICATOR_SOURCES), + // NOT a hand-copied subset. For EVERY indicator it asserts through the REAL + // exported classifyUnparseableAsInfra that: + // (a) a labelled drift-body line `Real: ` (no drift marker) is + // NOT classified as infra — genuine drift is surfaced/fail-loud; and + // (b) a bare line-start `` failure reason IS classified as infra. + // + // This is what makes the class non-recurring: if a future indicator is added + // to INFRA_INDICATOR_SPECS but individually mis-anchored (e.g. with the + // old `(?:.*:\s*)?` prefix or an unanchored `/i`), row (a) fails automatically + // for that indicator — no one has to remember to add a bespoke test. + // + // RED before the uniform-anchoring fix: at minimum the `empty response`, + // `returned no SSE events`, and `returned empty body` rows fail case (a) + // (they were `(?:.*:\s*)?`-prefixed or unanchored, so a labelled value matched + // and swallowed genuine drift). GREEN after: all rows pass both cases. + // ------------------------------------------------------------------------- + describe("uniform anchoring across the REAL infra-indicator list (property)", () => { + it("exports a non-empty indicator list to iterate", () => { + expect(INFRA_INDICATOR_SOURCES.length).toBeGreaterThan(0); + }); + + for (const source of INFRA_INDICATOR_SOURCES) { + const sample = infraIndicatorSample(source); + + it(`[${source}] a labelled drift-body value "Real: ${sample}" is NOT swallowed as infra (a)`, () => { + // A labelled body line carrying the phrase as a drift VALUE. No drift + // marker present, so the infra-indicator anchoring is the SOLE + // determinant: if the indicator is properly line-anchored it does NOT + // match here (the phrase follows a `Real:` label), so the batch is not + // all-infra and the failure is surfaced (classify → false). + const msg = + " Path: choices[0].message.content\n" + + " SDK: n/a\n" + + ` Real: ${sample}\n` + + " Mock: \n"; + expect(classifyUnparseableAsInfra([msg])).toBe(false); + }); + + it(`[${source}] a bare line-start "${sample}" failure reason IS classified as infra (b)`, () => { + // The phrase AS the failure reason at line start must still be infra — + // the anchoring fix must not over-tighten and break genuine infra. + expect(classifyUnparseableAsInfra([sample])).toBe(true); + }); + + it(`[${source}] taxonomy (c): a bare "${sample}" reason → collector exit 0 (benign, no quarantine)`, () => { + // A1.4 extension: tie the infra classification to the exit-code taxonomy + // at the REAL collector surface. A bare infra-reason failure must be a + // benign exit 0 — NOT quarantined (exit 5) and NOT a crash. + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + title: "non-streaming text matches real API", + failureMessages: [sample], + }), + ]); + expect(() => collectDriftEntries(result)).not.toThrow(); + expect(entriesOf(result)).toEqual([]); + expect(quarantineOf(result)).toEqual([]); + expect(exitCodeOf(result)).toBe(0); + }); + + it(`[${source}] taxonomy (c'): a labelled "Real: ${sample}" drift value → NOT exit 0 (quarantined, exit 5)`, () => { + // Symmetric to (a) at the collector surface: a labelled body value that + // merely CONTAINS the infra phrase must never be swallowed as a green. + // It is not a full parseable drift block, so it is quarantined (exit 5). + const msg = + " Path: choices[0].message.content\n" + + " SDK: n/a\n" + + ` Real: ${sample}\n` + + " Mock: \n"; + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + title: "non-streaming text matches real API", + failureMessages: [msg], + }), + ]); + expect(exitCodeOf(result)).not.toBe(0); + expect(exitCodeOf(result)).toBe(5); + }); + } + }); +}); + +// --------------------------------------------------------------------------- +// D6.2 — per-item `id` field on ParsedDiff +// +// The delta layer (D6.1) keys findings by `provider+id`. For N distinct unknown +// model ids, the collector must produce N DISTINCT per-item `id` values so that +// a downstream `provider+id` keying yields N distinct keys — not 1 collapsed +// key under `path:"knownModels"` (pre-fix behaviour when `id` was absent/undefined). +// +// RED (pre-fix): `id` is unset on every ParsedDiff produced by the canary path, +// so all 3 diffs have `id === undefined` → only 1 distinct key. +// GREEN (post-fix): each diff carries `id` = the model id stored in `diff.real`, +// so 3 distinct unknown ids → 3 distinct `id` values → 3 distinct keys. +// --------------------------------------------------------------------------- + +describe("D6.2 — per-item id on ParsedDiff", () => { + // Three distinct hypothetical unknown model ids (not in the knownModels set + // in ws-realtime.drift.ts — A4 note: use future/hypothetical ids only). + const THREE_UNKNOWN_IDS_CANARY = + "AssertionError: UNKNOWN_REALTIME_MODELS=gpt-realtime-x1,gpt-realtime-x2,gpt-realtime-x3: " + + "expected [ 'gpt-realtime-x1', …(2) ] to deeply equal []\n" + + " at /repo/src/__tests__/drift/ws-realtime.drift.ts:108:69"; + + it("D6.2 RED→GREEN: 3 distinct canary model ids produce 3 DISTINCT per-item id fields (not collapsed under undefined)", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "canary: GA realtime models available", + failureMessages: [THREE_UNKNOWN_IDS_CANARY], + }), + ]); + + const entries = entriesOf(result); + expect(entries).toHaveLength(1); + const entry = entries[0]; + expect(entry.provider).toBe("OpenAI Realtime"); + + // There should be exactly 3 diffs — one per unknown model id. + expect(entry.diffs).toHaveLength(3); + + // D6.2 core assertion: each diff must carry a populated `id` field equal to + // the model id in `diff.real`, and all three must be distinct. + const ids = entry.diffs.map((d) => d.id); + expect(ids).toEqual(["gpt-realtime-x1", "gpt-realtime-x2", "gpt-realtime-x3"]); + + // All three ids are defined (not undefined). + expect(ids.every((id) => id !== undefined)).toBe(true); + + // All three ids are DISTINCT — a downstream provider+id key would yield 3 + // different keys, not 1 collapsed key under undefined/absent id. + const distinctIds = new Set(ids); + expect(distinctIds.size).toBe(3); + + // The model ids must match what's in `diff.real` (the source of truth). + for (const diff of entry.diffs) { + expect(diff.id).toBe(diff.real); + } + }); + + it("D6.2: parseDriftBlock-path diffs carry a stable id derived from path", () => { + // For regular drift-block diffs (not canary), `id` is derived from + // the `path` field so different paths → different ids. + const formatted = formatDriftReport("OpenAI Chat (non-streaming text)", [ + { ...SAMPLE_DIFF, path: "choices[0].message.refusal" }, + { ...SAMPLE_DIFF, path: "choices[0].message.content", severity: "warning" as const }, + ]); + const parsed = parseDriftBlock(formatted); + expect(parsed).not.toBeNull(); + expect(parsed!.diffs).toHaveLength(2); + + const ids = parsed!.diffs.map((d) => d.id); + // Both diffs must have a non-empty id. + expect(ids.every((id) => id !== undefined && id !== "")).toBe(true); + // The two paths are different → two distinct ids. + expect(new Set(ids).size).toBe(2); + // Each id must be derived from (or equal to) the path. + for (const diff of parsed!.diffs) { + expect(diff.id).toBe(diff.path); + } + }); }); diff --git a/src/__tests__/drift-delta.test.ts b/src/__tests__/drift-delta.test.ts new file mode 100644 index 00000000..833e08a3 --- /dev/null +++ b/src/__tests__/drift-delta.test.ts @@ -0,0 +1,195 @@ +import { describe, it, expect } from "vitest"; + +import { computeDelta, isBaseReportReusable } from "../../scripts/drift-delta.js"; +import type { DeltaKey } from "../../scripts/drift-delta.js"; +import { DriftClass } from "../../scripts/drift-types.js"; +import type { DriftReport, ParsedDiff } from "../../scripts/drift-types.js"; + +// --------------------------------------------------------------------------- +// drift-delta: the delta-gating core. +// +// The gate must BLOCK only on drift attributable to the PR diff (new-in-head), +// treat drift already present on main as ADVISORY (environmental / world drift), +// and report base-only drift as FIXED. The block/advisory decision is by KEY +// PRESENCE alone — `DriftClass` is annotation and must NEVER route. +// --------------------------------------------------------------------------- + +function diff(overrides: Partial = {}): ParsedDiff { + return { + path: "knownModels", + severity: "critical", + issue: "model drift", + expected: "x", + real: "y", + mock: "z", + ...overrides, + }; +} + +function report( + provider: string, + diffs: ParsedDiff[], + timestamp = "2026-07-14T00:00:00.000Z", +): DriftReport { + return { + timestamp, + entries: [ + { + provider, + scenario: "s", + builderFile: "b.ts", + builderFunctions: ["f"], + typesFile: null, + sdkShapesFile: "shapes.ts", + diffs, + }, + ], + }; +} + +function keys(list: DeltaKey[]): string[] { + return list.map((k) => `${k.provider}:${k.id}`).sort(); +} + +// --------------------------------------------------------------------------- +// The M-1 golden regression (#292). +// +// A real-drift/critical failure that is NEW-in-head MUST BLOCK. The old broken +// rule routed by CLASS (real-drift/critical → advisory), which would have +// greenlit #292. This test proves the class-routed rule fails and computeDelta +// blocks regardless of class. +// --------------------------------------------------------------------------- +describe("M-1 golden: new-in-head critical MUST block regardless of class", () => { + // Head introduces a critical/real-drift finding that base does not have. + const base = report("anthropic", [diff({ id: "claude-3-opus", class: DriftClass.None })]); + const head = report("anthropic", [ + diff({ id: "claude-3-opus", class: DriftClass.None }), + diff({ id: "claude-4-new-model", class: DriftClass.Critical }), + ]); + + // Simulate the OLD broken rule: route purely by class. A critical drift is + // sent to advisory, so a new-in-head #292 failure is NOT blocked. This stub + // encodes the pre-fix behavior we are regressing against. + const classRouted = (r: DriftReport) => { + const block: DeltaKey[] = []; + const advisory: DeltaKey[] = []; + for (const entry of r.entries) { + for (const d of entry.diffs) { + const dk: DeltaKey = { provider: entry.provider, id: d.id ?? d.path, class: d.class }; + if (d.class === DriftClass.Critical) advisory.push(dk); + else block.push(dk); + } + } + return { block, advisory }; + }; + + it("RED regression: the CLASS-ROUTED rule fails to block the new-in-head critical (would greenlight #292)", () => { + const result = classRouted(head); + const newCriticalBlocked = result.block.some((k) => k.id === "claude-4-new-model"); + // The broken rule sends the critical to advisory, NOT block. If this ever + // starts blocking, the class-routed regression is no longer being exercised. + expect(newCriticalBlocked).toBe(false); + expect(result.advisory.some((k) => k.id === "claude-4-new-model")).toBe(true); + }); + + it("GREEN: computeDelta blocks the new-in-head critical regardless of class", () => { + const { block, advisory } = computeDelta(base, head); + expect(keys(block)).toEqual(["anthropic:claude-4-new-model"]); + expect(keys(advisory)).toEqual(["anthropic:claude-3-opus"]); + // The blocked key is critical — proving class did not route it to advisory. + expect(block[0].class).toBe(DriftClass.Critical); + }); +}); + +describe("computeDelta routing by key presence", () => { + it("same key in base+head → advisory (even critical)", () => { + const base = report("openai", [diff({ id: "gpt-4", class: DriftClass.Critical })]); + const head = report("openai", [diff({ id: "gpt-4", class: DriftClass.Critical })]); + const { block, advisory, fixed } = computeDelta(base, head); + expect(block).toEqual([]); + expect(keys(advisory)).toEqual(["openai:gpt-4"]); + expect(fixed).toEqual([]); + }); + + it("head-only transient → block (keyed by provider+id)", () => { + const base = report("openai", []); + const head = report("openai", [diff({ id: "gpt-5-preview", class: DriftClass.Critical })]); + const { block, advisory, fixed } = computeDelta(base, head); + expect(keys(block)).toEqual(["openai:gpt-5-preview"]); + expect(advisory).toEqual([]); + expect(fixed).toEqual([]); + }); + + it("base-only failure → fixed (informational, not block/advisory)", () => { + const base = report("openai", [diff({ id: "gpt-3.5", class: DriftClass.Critical })]); + const head = report("openai", []); + const { block, advisory, fixed } = computeDelta(base, head); + expect(block).toEqual([]); + expect(advisory).toEqual([]); + expect(keys(fixed)).toEqual(["openai:gpt-3.5"]); + }); + + it("keys by provider+id (path bucket must NOT collapse N distinct ids into one)", () => { + const base = report("anthropic", []); + const head = report("anthropic", [ + diff({ path: "knownModels", id: "a" }), + diff({ path: "knownModels", id: "b" }), + diff({ path: "knownModels", id: "c" }), + ]); + const { block } = computeDelta(base, head); + expect(keys(block)).toEqual(["anthropic:a", "anthropic:b", "anthropic:c"]); + }); + + it("same id across different providers → distinct keys", () => { + const base = report("openai", [diff({ id: "shared" })]); + const head = { + timestamp: base.timestamp, + entries: [...base.entries, ...report("anthropic", [diff({ id: "shared" })]).entries], + }; + const { block, advisory } = computeDelta(base, head); + expect(keys(block)).toEqual(["anthropic:shared"]); + expect(keys(advisory)).toEqual(["openai:shared"]); + }); + + it("falls back to path when id is absent (legacy diffs still participate)", () => { + const base = report("cohere", []); + const head = report("cohere", [diff({ path: "legacyBucket" })]); // no id + const { block } = computeDelta(base, head); + expect(keys(block)).toEqual(["cohere:legacyBucket"]); + }); +}); + +describe("isBaseReportReusable (O-2)", () => { + const good = report("openai", [diff({ id: "gpt-4" })]); + + it("accepts a non-empty, known-good, same-UTC-day report", () => { + expect(isBaseReportReusable(good, "clean", true)).toBe(true); + expect(isBaseReportReusable(good, "success", true)).toBe(true); + }); + + it("rejects an empty-entries report (malformed cached base)", () => { + const empty: DriftReport = { timestamp: "2026-07-14T00:00:00.000Z", entries: [] }; + expect(isBaseReportReusable(empty, "clean", true)).toBe(false); + }); + + it("rejects a null / malformed report object", () => { + expect(isBaseReportReusable(null, "clean", true)).toBe(false); + expect(isBaseReportReusable(undefined, "clean", true)).toBe(false); + // Malformed: entries missing entirely. + expect(isBaseReportReusable({ timestamp: "t" } as unknown as DriftReport, "clean", true)).toBe( + false, + ); + }); + + it("rejects an unknown / bad conclusion (crash, quarantine, empty)", () => { + expect(isBaseReportReusable(good, "failure", true)).toBe(false); + expect(isBaseReportReusable(good, "quarantine", true)).toBe(false); + expect(isBaseReportReusable(good, "", true)).toBe(false); + expect(isBaseReportReusable(good, null, true)).toBe(false); + expect(isBaseReportReusable(good, undefined, true)).toBe(false); + }); + + it("rejects a stale (different-UTC-day) report", () => { + expect(isBaseReportReusable(good, "clean", false)).toBe(false); + }); +}); diff --git a/src/__tests__/drift-retry.test.ts b/src/__tests__/drift-retry.test.ts index 373a0b23..9e34c13f 100644 --- a/src/__tests__/drift-retry.test.ts +++ b/src/__tests__/drift-retry.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from "vitest"; -import { retryUntilStable } from "../../scripts/drift-retry.js"; +import { EXIT_QUARANTINE, retryUntilStable } from "../../scripts/drift-retry.js"; import type { RetryAttempt, RetryOptions, RetryResult } from "../../scripts/drift-retry.js"; // --------------------------------------------------------------------------- @@ -137,4 +137,21 @@ describe("retryUntilStable", () => { expect(result.attempts.map((a: RetryAttempt) => a.exitCode)).toEqual([2, 2, 0]); }); + + it("treats exit 5 (quarantine) as a distinct terminal outcome — no retry, quarantine:true", () => { + // exit 5 = collector quarantined unparseable output; must propagate immediately + // without retrying and mark the result with quarantine:true. + const runner = fakeRunner([EXIT_QUARANTINE, 0, 0]); + const opts = makeOptions({ runCollector: runner.run }); + const result: RetryResult = retryUntilStable(opts); + + // Propagated as exit 5, not swallowed into the crash branch + expect(result.exitCode).toBe(EXIT_QUARANTINE); + // Quarantine flag set + expect(result.quarantine).toBe(true); + // No retry — only one attempt + expect(runner.calls()).toBe(1); + // Not treated as a transient drift event + expect(result.transient).toBe(false); + }); }); diff --git a/src/__tests__/drift-scripts.test.ts b/src/__tests__/drift-scripts.test.ts index 268fb68d..6ce34527 100644 --- a/src/__tests__/drift-scripts.test.ts +++ b/src/__tests__/drift-scripts.test.ts @@ -19,7 +19,7 @@ import { import { summarizeDriftReport } from "../../scripts/drift-slack-summary.js"; -import type { DriftEntry, DriftReport } from "../../scripts/drift-types.js"; +import type { DriftEntry, DriftReport, QuarantineEntry } from "../../scripts/drift-types.js"; // --------------------------------------------------------------------------- // Helpers @@ -356,7 +356,9 @@ describe("summarizeDriftReport", () => { expect(summary).toContain("*OpenAI Chat*"); expect(summary).toContain("1 critical"); expect(summary).toContain("`choices[0].message.refusal`"); - expect(summary.startsWith("•")).toBe(true); + // The first line is now the classification header; bullet lines follow. + const bulletLines = summary.split("\n").filter((l) => l.startsWith("•")); + expect(bulletLines.length).toBeGreaterThan(0); }); it("merges multiple entries for the same provider into one line with combined counts", () => { @@ -404,11 +406,12 @@ describe("summarizeDriftReport", () => { makeEntry(), // OpenAI Chat with a critical ], }); - const lines = summary.split("\n"); - expect(lines).toHaveLength(2); + // First line is the classification header; provider bullets follow. + const bulletLines = summary.split("\n").filter((l) => l.startsWith("•")); + expect(bulletLines).toHaveLength(2); // Provider with a critical diff sorts before the warning-only provider - expect(lines[0]).toContain("*OpenAI Chat*"); - expect(lines[1]).toContain("*Anthropic*"); + expect(bulletLines[0]).toContain("*OpenAI Chat*"); + expect(bulletLines[1]).toContain("*Anthropic*"); }); it("caps example paths per provider and reports the remainder", () => { @@ -439,3 +442,147 @@ describe("summarizeDriftReport", () => { expect(summary).not.toContain("\\n"); }); }); + +// --------------------------------------------------------------------------- +// summarizeDriftReport — headline class + per-item detail (C5.2) +// --------------------------------------------------------------------------- + +function makeQuarantineEntry(overrides?: Partial): QuarantineEntry { + return { + provider: "OpenAI Chat", + testName: "OpenAI Chat > non-streaming text > response shape", + rawLocation: "src/__tests__/drift/openai-chat.drift.ts:42", + message: "Cannot read properties of undefined (reading 'choices')", + ...overrides, + }; +} + +describe("summarizeDriftReport — headline class", () => { + it("class: real-drift — report has entries with critical diffs", () => { + const report: DriftReport = { + timestamp: "t", + entries: [ + makeEntry({ + diffs: [ + { + severity: "critical", + issue: "field missing from mock", + path: "choices[0].message.refusal", + expected: "null", + real: "null", + mock: "", + }, + ], + }), + ], + }; + const summary = summarizeDriftReport(report); + expect(summary).toContain("real-drift"); + // per-item: provider name present + expect(summary).toContain("OpenAI Chat"); + // per-item: offending path or id + expect(summary).toContain("choices[0].message.refusal"); + // per-item: one-line issue + expect(summary).toContain("field missing from mock"); + // per-item: file reference (builderFile) + expect(summary).toContain("src/helpers.ts"); + }); + + it("class: quarantine — report has quarantine[] entries", () => { + const report: DriftReport = { + timestamp: "t", + entries: [], + quarantine: [makeQuarantineEntry()], + }; + const summary = summarizeDriftReport(report); + expect(summary).toContain("quarantine"); + // quarantine: provider + expect(summary).toContain("OpenAI Chat"); + // quarantine: rawLocation (file:line) + expect(summary).toContain("src/__tests__/drift/openai-chat.drift.ts:42"); + // quarantine: one-line message + expect(summary).toContain("Cannot read properties of undefined"); + }); + + it("class: stale-key — InfraError status 401", () => { + const report: DriftReport = { timestamp: "t", entries: [] }; + const summary = summarizeDriftReport(report, { infraErrorStatus: 401 }); + expect(summary).toContain("stale-key"); + }); + + it("class: stale-key — InfraError status 403", () => { + const report: DriftReport = { timestamp: "t", entries: [] }; + const summary = summarizeDriftReport(report, { infraErrorStatus: 403 }); + expect(summary).toContain("stale-key"); + }); + + it("class: infra-transient — InfraError status 429", () => { + const report: DriftReport = { timestamp: "t", entries: [] }; + const summary = summarizeDriftReport(report, { infraErrorStatus: 429 }); + expect(summary).toContain("infra-transient"); + }); + + it("class: infra-transient — InfraError status 503", () => { + const report: DriftReport = { timestamp: "t", entries: [] }; + const summary = summarizeDriftReport(report, { infraErrorStatus: 503 }); + expect(summary).toContain("infra-transient"); + }); + + it("class: test-infra-false-positive — exit 0, empty entries, no infraError", () => { + const report: DriftReport = { timestamp: "t", entries: [] }; + const summary = summarizeDriftReport(report, { exitCode: 0 }); + expect(summary).toContain("test-infra-false-positive"); + }); + + it("quarantine entries appear on their own distinct line with testName", () => { + const report: DriftReport = { + timestamp: "t", + entries: [], + quarantine: [ + makeQuarantineEntry({ provider: "Anthropic", testName: "Anthropic > streaming > shape" }), + ], + }; + const summary = summarizeDriftReport(report); + const lines = summary.split("\n"); + const quarantineLine = lines.find((l) => l.includes("Anthropic")); + expect(quarantineLine).toBeDefined(); + expect(quarantineLine).toContain("Anthropic > streaming > shape"); + }); + + it("mixed: real-drift entries + quarantine both appear in summary", () => { + const report: DriftReport = { + timestamp: "t", + entries: [makeEntry()], + quarantine: [makeQuarantineEntry({ provider: "Gemini" })], + }; + const summary = summarizeDriftReport(report); + // Class is real-drift when entries have critical diffs (quarantine is secondary) + expect(summary).toContain("real-drift"); + // Quarantine section still present + expect(summary).toContain("Gemini"); + expect(summary).toContain("quarantine"); + }); + + it("per-item id is used over path when present", () => { + const report: DriftReport = { + timestamp: "t", + entries: [ + makeEntry({ + diffs: [ + { + severity: "critical", + issue: "model removed from mock", + path: "knownModels", + id: "gpt-4o-mini", + expected: "present", + real: "present", + mock: "", + }, + ], + }), + ], + }; + const summary = summarizeDriftReport(report); + expect(summary).toContain("gpt-4o-mini"); + }); +}); diff --git a/src/__tests__/drift/model-family.test.ts b/src/__tests__/drift/model-family.test.ts new file mode 100644 index 00000000..da230e95 --- /dev/null +++ b/src/__tests__/drift/model-family.test.ts @@ -0,0 +1,35 @@ +/** + * Unit test for the shared `normalizeModelFamily` primitive. + */ +import { describe, it, expect } from "vitest"; +import { normalizeModelFamily } from "./model-family.js"; + +describe("normalizeModelFamily", () => { + it("strips a trailing dated snapshot suffix", () => { + expect(normalizeModelFamily("gpt-audio-2025-08-28", "openai")).toBe("gpt-audio"); + }); + + it("strips a trailing build-tag suffix", () => { + expect(normalizeModelFamily("tts-1-1106", "openai")).toBe("tts-1"); + }); + + it("does not strip a single-digit suffix", () => { + expect(normalizeModelFamily("gpt-live-1", "openai")).toBe("gpt-live-1"); + }); + + it("strips a trailing Anthropic contiguous 8-digit snapshot for anthropic", () => { + expect(normalizeModelFamily("claude-3-5-sonnet-20241022", "anthropic")).toBe( + "claude-3-5-sonnet", + ); + expect(normalizeModelFamily("claude-3-7-sonnet-20250219", "anthropic")).toBe( + "claude-3-7-sonnet", + ); + }); + + it("does NOT strip a contiguous 8-digit tail for openai/gemini", () => { + // The 8-digit rule is Anthropic-only; a non-date 8-digit tail on another + // provider must survive so it is not silently over-stripped. + expect(normalizeModelFamily("gpt-weird-12345678", "openai")).toBe("gpt-weird-12345678"); + expect(normalizeModelFamily("gemini-weird-12345678", "gemini")).toBe("gemini-weird-12345678"); + }); +}); diff --git a/src/__tests__/drift/model-family.ts b/src/__tests__/drift/model-family.ts new file mode 100644 index 00000000..6489f64a --- /dev/null +++ b/src/__tests__/drift/model-family.ts @@ -0,0 +1,59 @@ +/** + * Shared, side-effect-free `normalizeModelFamily` primitive (no + * `describe`/`beforeAll`) reducing a model id to its FAMILY KEY by stripping the + * trailing version/snapshot suffixes that providers append to already-known + * families. + * + * New dated snapshots of an existing family land constantly (`tts-1-1106`, + * `gpt-audio-2025-08-28`, `gpt-4o-mini-tts-2025-12-15`, …); appending every one + * to a known-ID set never converges and turns the daily drift job permanently + * red on false positives. Comparing the NORMALIZED family instead means only a + * genuinely new family (e.g. `gpt-live`) is ever flagged. + * + * Two suffix shapes are stripped, repeatedly, from the END of the id for ALL + * providers (the SHARED CORE): + * - a dated snapshot `-YYYY-MM-DD` (e.g. `-2025-08-28`) + * - a build/version tag `-NNN` or `-NNNN` (3–4 digits, e.g. `-1106`) + * + * Both are anchored to the end and applied in a loop so a trailing dated + * snapshot that itself follows a build tag is fully reduced. A short numeric + * suffix like `gpt-live-1`'s trailing `-1` is a SINGLE digit and is deliberately + * NOT stripped, so `gpt-live-1` normalizes to `gpt-live-1` — an unknown family — + * and stays flagged (the whole point of the canary). + * + * The `provider` argument selects a per-provider EXTRA rule on top of the shared + * core: + * - `anthropic` additionally strips a CONTIGUOUS 8-digit snapshot `-YYYYMMDD` + * (e.g. `claude-3-5-sonnet-20241022` → `claude-3-5-sonnet`). Anthropic dates + * its ids with an undelimited `-YYYYMMDD` suffix rather than the dashed + * `-YYYY-MM-DD` form, so without this every dated Claude id would + * false-positive as drift (the incident-2 class, for Anthropic). + * - `openai` / `gemini` keep ONLY the shared core — the 8-digit strip is NOT + * applied to them, so a non-date 8-digit tail is never over-stripped, and + * `normalizeModelFamily(id, "openai")` stays byte-identical to the historical + * `normalizeVoiceModelFamily(id)`. + * + * The per-provider rule runs inside the SAME reduction loop as the shared core, + * so it stays idempotent: a dated snapshot following a build tag still fully + * reduces. + */ +const DATED_SNAPSHOT_SUFFIX = /-\d{4}-\d{2}-\d{2}$/; +const BUILD_TAG_SUFFIX = /-\d{3,4}$/; +/** Anthropic contiguous `-YYYYMMDD` snapshot suffix (undelimited date). */ +const ANTHROPIC_DATE_SUFFIX = /-\d{8}$/; + +export function normalizeModelFamily( + id: string, + provider: "openai" | "anthropic" | "gemini", +): string { + let family = id; + for (;;) { + let stripped = family.replace(DATED_SNAPSHOT_SUFFIX, "").replace(BUILD_TAG_SUFFIX, ""); + if (provider === "anthropic") { + stripped = stripped.replace(ANTHROPIC_DATE_SUFFIX, ""); + } + if (stripped === family) break; + family = stripped; + } + return family; +} diff --git a/src/__tests__/drift/model-registry.test.ts b/src/__tests__/drift/model-registry.test.ts new file mode 100644 index 00000000..b6e02703 --- /dev/null +++ b/src/__tests__/drift/model-registry.test.ts @@ -0,0 +1,189 @@ +/** + * Unit test for the per-provider model-family registry shape, plus the §6.1c + * builder/fixture cross-check. + * + * Part 1 (registry shape): the minimal invariant probe for model-registry.ts: + * - seeded families are normalization-consistent (membership works), + * - the provider-mode allowlist carries `gemini-interactions`, + * - no family appears in BOTH include and exclude for any provider. + * + * Part 2 (builder/fixture cross-check — §6.1c): every model id that aimock's + * builders and fixtures reference must resolve, via normalizeModelFamily, to a + * family that is either in includeFamilies for its provider (aimock mocks it) + * or in excludeFamilies (retired/non-text/preview). Non-model "provider mode" + * tokens (e.g. `gemini-interactions`) must appear on NON_MODEL_TOKENS. A stray + * builder model id or a misclassified prose token must fail this test — never a + * live crash in the drift job. + * + * The cross-check table is derived from: + * - `DEFAULT_MODELS` in src/server.ts (aimock's advertised /v1/models list) + * - Dated/versioned ids used in conformance and fixture-blocks tests + * - The `gemini-interactions` provider-mode token documented in README + * + * The table is intentionally static (no source scraping) so it exercises the + * EXACT normalizeModelFamily + registry membership path the live drift check + * relies on, with no live-key dependency. + */ +import { describe, it, expect } from "vitest"; +import { normalizeModelFamily } from "./model-family.js"; +import { includeFamilies, excludeFamilies, NON_MODEL_TOKENS } from "./model-registry.js"; + +// ─── Part 1: registry shape invariants ─────────────────────────────────────── + +describe("model-registry", () => { + it("include contains the normalized family of a mocked model id", () => { + expect(includeFamilies.gemini.has(normalizeModelFamily("gemini-2.5-flash", "gemini"))).toBe( + true, + ); + }); + + it("allowlists the gemini-interactions provider-mode token", () => { + expect(NON_MODEL_TOKENS.has("gemini-interactions")).toBe(true); + }); + + it("no family appears in both include and exclude for any provider", () => { + for (const provider of ["openai", "anthropic", "gemini"] as const) { + const inc = includeFamilies[provider]; + const exc = excludeFamilies[provider]; + const overlap = [...inc].filter((f) => exc.has(f)); + expect( + overlap, + `provider ${provider} has families on both lists: ${overlap.join(", ")}`, + ).toEqual([]); + } + }); + + it("seeds are idempotent under normalization (already family keys)", () => { + for (const provider of ["openai", "anthropic", "gemini"] as const) { + for (const family of includeFamilies[provider]) { + expect(normalizeModelFamily(family, provider)).toBe(family); + } + for (const family of excludeFamilies[provider]) { + expect(normalizeModelFamily(family, provider)).toBe(family); + } + } + }); +}); + +// ─── Part 2: §6.1c builder/fixture cross-check ─────────────────────────────── + +type Provider = "openai" | "anthropic" | "gemini"; + +/** + * The static cross-check table. Each entry is a model id that aimock's + * builders, fixtures, or DEFAULT_MODELS reference, tagged with its provider. + * Every entry MUST normalize to a family in includeFamilies[provider] or + * excludeFamilies[provider]. Non-model tokens go in NON_MODEL_TOKENS_UNDER_TEST. + * + * Sources: + * - DEFAULT_MODELS (src/server.ts): the ids aimock advertises at /v1/models + * - Conformance / fixture-blocks tests: dated snapshots, family aliases + * - gemini-interactions: provider-mode token (NON_MODEL_TOKENS, not a real id) + */ +const BUILDER_FIXTURE_MODEL_IDS: Array<{ id: string; provider: Provider }> = [ + // ── DEFAULT_MODELS (src/server.ts) ───────────────────────────────────────── + { id: "gpt-4", provider: "openai" }, + { id: "gpt-4o", provider: "openai" }, + { id: "claude-3-5-sonnet-20241022", provider: "anthropic" }, // dated snapshot + { id: "gemini-2.0-flash", provider: "gemini" }, + // text-embedding-3-small is in excludeFamilies (non-text-generation) — see below + + // ── OpenAI: additional families referenced in conformance/fixture tests ──── + { id: "gpt-3.5-turbo", provider: "openai" }, + { id: "gpt-4-turbo", provider: "openai" }, + { id: "gpt-4-turbo-2024-04-09", provider: "openai" }, // dated → gpt-4-turbo + { id: "gpt-4o-2024-08-06", provider: "openai" }, // dated → gpt-4o + { id: "gpt-4o-mini", provider: "openai" }, + { id: "gpt-4.1", provider: "openai" }, + { id: "gpt-4.1-mini", provider: "openai" }, + { id: "gpt-4.1-nano", provider: "openai" }, + { id: "gpt-5", provider: "openai" }, + { id: "gpt-5-mini", provider: "openai" }, + + // ── OpenAI: excluded families (embeddings, image, voice/audio) ──────────── + { id: "text-embedding-3-small", provider: "openai" }, // exclude: embeddings + { id: "gpt-image-1", provider: "openai" }, // exclude: image + { id: "gpt-audio", provider: "openai" }, // exclude: voice canary + { id: "gpt-audio-mini", provider: "openai" }, // exclude: voice canary + { id: "gpt-audio-2025-08-28", provider: "openai" }, // dated → gpt-audio (exclude) + { id: "tts-1", provider: "openai" }, // exclude: tts + { id: "gpt-4o-mini-tts", provider: "openai" }, // exclude: tts + { id: "gpt-4o-mini-tts-2025-12-15", provider: "openai" }, // dated → gpt-4o-mini-tts (exclude) + { id: "gpt-4o-transcribe", provider: "openai" }, // exclude: transcribe + { id: "gpt-4o-mini-transcribe", provider: "openai" }, // exclude: transcribe + { id: "gpt-realtime", provider: "openai" }, // exclude: realtime canary + { id: "gpt-realtime-mini", provider: "openai" }, // exclude: realtime canary + { id: "gpt-4o-realtime-preview", provider: "openai" }, // exclude: preview realtime + { id: "gpt-4o-mini-realtime-preview", provider: "openai" }, // exclude: preview realtime + + // ── Anthropic ───────────────────────────────────────────────────────────── + { id: "claude-3-opus", provider: "anthropic" }, + { id: "claude-3-opus-20240229", provider: "anthropic" }, // dated → claude-3-opus + { id: "claude-3-sonnet", provider: "anthropic" }, + { id: "claude-3-haiku", provider: "anthropic" }, + { id: "claude-3-5-sonnet", provider: "anthropic" }, + { id: "claude-3-5-haiku", provider: "anthropic" }, + { id: "claude-3-5-haiku-20241022", provider: "anthropic" }, // dated → claude-3-5-haiku + { id: "claude-3-7-sonnet", provider: "anthropic" }, + { id: "claude-3-7-sonnet-20250219", provider: "anthropic" }, // dated → claude-3-7-sonnet + { id: "claude-opus-4", provider: "anthropic" }, + { id: "claude-opus-4-20250514", provider: "anthropic" }, // dated → claude-opus-4 + { id: "claude-sonnet-4", provider: "anthropic" }, + { id: "claude-haiku-4", provider: "anthropic" }, + + // ── Gemini ──────────────────────────────────────────────────────────────── + { id: "gemini-1.5-pro", provider: "gemini" }, + { id: "gemini-1.5-pro-2024-05-14", provider: "gemini" }, // dated → gemini-1.5-pro + { id: "gemini-1.5-flash", provider: "gemini" }, + { id: "gemini-2.5-flash", provider: "gemini" }, + { id: "gemini-2.5-pro", provider: "gemini" }, + { id: "gemini-2.0-flash-exp", provider: "gemini" }, // exclude: experimental + { id: "gemini-2.0-flash-thinking-exp", provider: "gemini" }, // exclude: experimental +]; + +/** + * Provider-mode tokens that aimock uses internally to route to a real upstream + * provider but that are NOT model ids any provider's /models endpoint exposes. + * These MUST be on NON_MODEL_TOKENS — they must never be mistaken for real ids. + */ +const NON_MODEL_TOKENS_UNDER_TEST: string[] = ["gemini-interactions"]; + +describe("§6.1c builder/fixture cross-check (no live keys)", () => { + it("every referenced builder/fixture model id resolves to a classified family", () => { + const failures: string[] = []; + + for (const { id, provider } of BUILDER_FIXTURE_MODEL_IDS) { + const family = normalizeModelFamily(id, provider); + const inInclude = includeFamilies[provider].has(family); + const inExclude = excludeFamilies[provider].has(family); + + if (!inInclude && !inExclude) { + failures.push( + `${id} (${provider}): normalized to "${family}" which is in NEITHER includeFamilies NOR excludeFamilies`, + ); + } + } + + expect( + failures, + `Stray builder/fixture model ids not in registry:\n${failures.join("\n")}`, + ).toEqual([]); + }); + + it("every non-model provider-mode token is on NON_MODEL_TOKENS", () => { + const failures: string[] = []; + + for (const token of NON_MODEL_TOKENS_UNDER_TEST) { + if (!NON_MODEL_TOKENS.has(token)) { + failures.push( + `"${token}" is not on NON_MODEL_TOKENS — a greedy scrape would false-positive`, + ); + } + } + + expect( + failures, + `Provider-mode tokens missing from NON_MODEL_TOKENS:\n${failures.join("\n")}`, + ).toEqual([]); + }); +}); diff --git a/src/__tests__/drift/model-registry.ts b/src/__tests__/drift/model-registry.ts new file mode 100644 index 00000000..cd8444f2 --- /dev/null +++ b/src/__tests__/drift/model-registry.ts @@ -0,0 +1,165 @@ +/** + * Per-provider model-family REGISTRY for text-generation drift detection. + * + * Side-effect-free data module (no `describe`/`beforeAll`, like `voice-models.ts`) + * so both the live drift check (`models.drift.ts`) and its unit test can import + * the same seed data without transitively registering a drift suite. + * + * The drift check works by NORMALIZED FAMILY, not raw id: a provider's live + * `GET /models` list is normalized through `normalizeModelFamily(id, provider)` + * and each resulting family is subtracted against `include ∪ exclude`. Anything + * left over is an UNCLASSIFIED family — the drift signal. (The subtract/delta + * itself is `models.drift.ts`'s job; this module only owns the seed sets.) + * + * Two curated sets per provider, both keyed by the NORMALIZED family: + * + * - `include` — families aimock actually MOCKS. Derived from the model ids + * aimock's conformance tests, README, and fixtures reference (chat / text + * completion families). A live family that normalizes into this set is known + * and generates no drift. Seeds are already family keys, so dated snapshots + * of an included family (`gpt-4o-2024-08-06` → `gpt-4o`) collapse onto them. + * + * - `exclude` — families we deliberately DO NOT treat as text-generation drift: + * retired/legacy ids, preview-only ids, and non-text families (embeddings, + * image, tts/audio/transcribe voice families — the last are the realtime + * canary's responsibility in `voice-models.ts`, not this text check). A live + * family in this set is expected and generates no drift. + * + * A family MUST NOT appear in both `include` and `exclude` for the same provider + * (asserted in the unit test) — the two sets partition the "already classified" + * space; their union is what the drift check subtracts. + * + * `NON_MODEL_TOKENS` is a provider-agnostic allowlist of aimock "provider mode" + * names — internal routing names that reuse a real upstream API key but are NOT + * model ids any provider's `/models` endpoint exposes (e.g. `gemini-interactions` + * reuses the Gemini upstream key). The README documents them, so a greedy source + * scrape or a builder cross-check would otherwise treat them as unknown model + * ids and flag guaranteed false positives. They live here so both the drift + * check and the builder cross-check (B4.3) exclude the exact same tokens. + * + * Every set is built THROUGH `normalizeModelFamily` so membership tests are + * normalization-consistent and the seeds are provably idempotent (a seed carrying + * a stray dated/build suffix would silently normalize to a different key — the + * `.map(normalize)` makes that impossible). + */ + +import { normalizeModelFamily } from "./model-family.js"; + +type Provider = "openai" | "anthropic" | "gemini"; + +/** Build a family Set for a provider, seeding each entry through the normalizer. */ +function familySet(provider: Provider, families: string[]): Set { + return new Set(families.map((f) => normalizeModelFamily(f, provider))); +} + +/** + * Families aimock MOCKS, per provider. These are the canonical text-generation + * family keys referenced by aimock's conformance tests, README, and fixtures. + * Dated/versioned variants normalize onto these (e.g. `gpt-4o-2024-08-06` → + * `gpt-4o`, `claude-3-5-sonnet-20241022` → `claude-3-5-sonnet`). + */ +export const includeFamilies: Record> = { + openai: familySet("openai", [ + // Chat / completion families + "gpt-3.5-turbo", + "gpt-4", + "gpt-4-turbo", + "gpt-4o", + "gpt-4o-mini", + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4.1-nano", + "gpt-5", + "gpt-5-mini", + ]), + anthropic: familySet("anthropic", [ + // Claude 3 / 3.5 / 3.7 families + "claude-3-opus", + "claude-3-sonnet", + "claude-3-haiku", + "claude-3-5-sonnet", + "claude-3-5-haiku", + "claude-3-7-sonnet", + // Claude 4 families + "claude-opus-4", + "claude-sonnet-4", + "claude-haiku-4", + ]), + gemini: familySet("gemini", [ + // Gemini 1.5 / 2.0 / 2.5 text families + "gemini-1.5-pro", + "gemini-1.5-flash", + "gemini-2.0-flash", + "gemini-2.5-flash", + "gemini-2.5-pro", + ]), +}; + +/** + * Families we deliberately DO NOT count as text-generation drift, per provider: + * retired/legacy ids, preview-only ids, and non-text families (embeddings, + * image, and the voice/audio/tts/transcribe families the realtime canary in + * `voice-models.ts` owns). A live family here is expected, not drift. + */ +export const excludeFamilies: Record> = { + openai: familySet("openai", [ + // Retired / legacy chat + "gpt-3", + "gpt-3.5", + // Embeddings (non-text-generation) + "text-embedding-3-small", + "text-embedding-3-large", + "text-embedding-ada-002", + // Image models + "dall-e-2", + "dall-e-3", + "gpt-image-1", + // Voice / audio / tts / transcribe — owned by the realtime canary + "tts-1", + "tts-1-hd", + "whisper-1", + "gpt-audio", + "gpt-audio-mini", + "gpt-audio-1.5", + "gpt-4o-mini-tts", + "gpt-4o-transcribe", + "gpt-4o-mini-transcribe", + "gpt-4o-transcribe-diarize", + "gpt-realtime", + "gpt-realtime-mini", + "gpt-realtime-2", + "gpt-realtime-2.1", + "gpt-realtime-2.1-mini", + "gpt-realtime-1.5", + "gpt-realtime-translate", + "gpt-realtime-whisper", + // Preview-only realtime + "gpt-4o-realtime-preview", + "gpt-4o-mini-realtime-preview", + ]), + anthropic: familySet("anthropic", [ + // Retired / legacy Claude ids + "claude-v3", + "claude-2", + "claude-instant-1", + ]), + gemini: familySet("gemini", [ + // Retired / legacy + "gemini-pro", + // Embeddings (non-text-generation) + "text-embedding-004", + // Experimental / preview / thinking-preview + "gemini-2.0-flash-exp", + "gemini-2.0-flash-thinking-exp", + // Live/full-duplex voice — owned by the realtime canary, not this text check + "gemini-live", + ]), +}; + +/** + * aimock "provider mode" names: internal routing names that reuse a real + * upstream provider key but are NOT model ids any provider's `/models` endpoint + * exposes. Provider-agnostic allowlist so both the drift check and the builder + * cross-check exclude the exact same tokens (never false-positive drift). + */ +export const NON_MODEL_TOKENS: Set = new Set(["gemini-interactions"]); diff --git a/src/__tests__/drift/models.drift.ts b/src/__tests__/drift/models.drift.ts index 73e4f4e0..d2737a37 100644 --- a/src/__tests__/drift/models.drift.ts +++ b/src/__tests__/drift/models.drift.ts @@ -1,103 +1,185 @@ /** - * Model deprecation checks — verify that models referenced in aimock's - * tests, docs, and examples still exist at each provider. + * Model-family drift check — verify that each provider's LIVE `GET /models` + * list contains no UNCLASSIFIED text-generation family. + * + * How it works (no source scraping — that path caused incident 5): + * 1. Fetch the provider's live model list (`list*Models`). + * 2. Normalize every id to its FAMILY key (`normalizeModelFamily`), so dated + * snapshots and build tags collapse onto their family + * (`gpt-4o-2024-08-06` → `gpt-4o`, `tts-1-1106` → `tts-1`). + * 3. Subtract the already-classified space: `includeFamilies[provider] ∪ + * excludeFamilies[provider]`, and drop the provider-agnostic + * `NON_MODEL_TOKENS` allowlist. + * 4. Whatever remains is an UNCLASSIFIED family — the drift signal. It is + * surfaced as a FAILING assertion wrapped in a `formatDriftReport` block so + * the collector (`scripts/drift-report-collector.ts`) routes it: the + * `API DRIFT DETECTED` block parses into critical entries (exit-2 auto-fix + * lane); anything it cannot map falls to exit-5 quarantine. + * + * Comparing NORMALIZED families (not raw ids) is what makes this converge: + * appending every new dated snapshot to a known-id set never stabilizes and + * turns the daily job permanently red on false positives (incident 2). Only a + * genuinely NEW family (e.g. `gpt-live`) is ever flagged. + * + * Because nothing is scraped from source, an aimock "provider mode" prose token + * (e.g. `gemini-interactions`) can never enter the pipeline as a candidate id — + * the only inputs are the provider's own `/models` payload. */ import { describe, it, expect } from "vitest"; -import * as fs from "node:fs"; -import * as path from "node:path"; import { listOpenAIModels, listAnthropicModels, listGeminiModels } from "./providers.js"; +import { normalizeModelFamily } from "./model-family.js"; +import { includeFamilies, excludeFamilies, NON_MODEL_TOKENS } from "./model-registry.js"; +import { formatDriftReport } from "./schema.js"; -// --------------------------------------------------------------------------- -// Scrape referenced models from the codebase -// --------------------------------------------------------------------------- +type Provider = "openai" | "anthropic" | "gemini"; -const PROJECT_ROOT = path.resolve(import.meta.dirname, "..", "..", ".."); - -function scrapeModels(pattern: RegExp, files: string[]): string[] { - const models = new Set(); - for (const file of files) { - const filePath = path.join(PROJECT_ROOT, file); - if (!fs.existsSync(filePath)) continue; - const content = fs.readFileSync(filePath, "utf-8"); - pattern.lastIndex = 0; - let match; - while ((match = pattern.exec(content)) !== null) { - models.add(match[1]); - } +/** + * Reduce a live `/models` list to the UNCLASSIFIED families: normalize each id, + * then drop everything already in `include ∪ exclude` or on the non-model + * allowlist. The returned list (sorted, de-duplicated) is the drift signal. + * + * Exported so the co-located regression suite can exercise the EXACT + * enumerate→normalize→subtract pipeline the live check relies on, with an + * injected payload — no reimplementation. + */ +export function unclassifiedFamilies(modelIds: string[], provider: Provider): string[] { + const known = new Set([...includeFamilies[provider], ...excludeFamilies[provider]]); + const unclassified = new Set(); + for (const id of modelIds) { + const family = normalizeModelFamily(id, provider); + if (known.has(family)) continue; + if (NON_MODEL_TOKENS.has(family) || NON_MODEL_TOKENS.has(id)) continue; + unclassified.add(family); } - return [...models]; + return [...unclassified].sort(); } -const sourceFiles = [ - "src/__tests__/api-conformance.test.ts", - "src/__tests__/ws-api-conformance.test.ts", - "README.md", - "fixtures/example-greeting.json", - "fixtures/example-multi-turn.json", - "fixtures/example-tool-call.json", -]; +/** + * Assert that a live `/models` list has zero unclassified families. On failure, + * emit one critical drift diff per unclassified family inside a + * `formatDriftReport` block so the collector routes it to the exit-2 auto-fix + * lane (provider names match `PROVIDER_MAP` keys in the collector). + */ +function assertNoUnclassifiedFamilies( + modelIds: string[], + provider: Provider, + context: string, +): void { + const unclassified = unclassifiedFamilies(modelIds, provider); + const report = + unclassified.length > 0 + ? formatDriftReport( + context, + unclassified.map((family) => ({ + path: `models/${family}`, + severity: "critical" as const, + issue: + `Unclassified model family "${family}" in ${provider} /models — ` + + `add it to includeFamilies (aimock mocks it) or excludeFamilies ` + + `(non-text / retired / preview) in model-registry.ts`, + expected: "(family in includeFamilies ∪ excludeFamilies)", + real: family, + mock: "", + })), + ) + : `No drift detected: ${context}`; + expect(unclassified, report).toEqual([]); +} // --------------------------------------------------------------------------- -// OpenAI +// Regression suite (no live keys) — exercises the REAL pipeline with injected +// `/models` payloads. Runs unconditionally so the drift job proves the +// enumerate→normalize→subtract behavior even when live keys are absent. // --------------------------------------------------------------------------- -describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI model availability", () => { - it("models used in aimock tests are still available", async () => { - const models = await listOpenAIModels(process.env.OPENAI_API_KEY!); - const referenced = scrapeModels(/\b(gpt-4o(?:-mini)?|gpt-4|gpt-3\.5-turbo)\b/g, sourceFiles); +describe("model-family pipeline (injected /models)", () => { + it("incident 2: dated snapshots of included families produce ZERO drift", () => { + // Payload of dated/build-tag snapshots whose FAMILIES are all in + // includeFamilies/excludeFamilies. The old scrape+substring path would have + // flagged these dated ids as unknown; the normalize+subtract path collapses + // each onto its known family, so the unclassified delta must be empty. + const openaiPayload = [ + "gpt-4o-2024-08-06", // → gpt-4o (include) + "gpt-4o-mini-2024-07-18", // → gpt-4o-mini (include) + "gpt-4.1-2025-04-14", // → gpt-4.1 (include) + "gpt-audio-2025-08-28", // → gpt-audio (exclude) + "tts-1-1106", // → tts-1 (exclude) + "gpt-4o-mini-tts-2025-12-15", // → gpt-4o-mini-tts (exclude) + ]; + expect(unclassifiedFamilies(openaiPayload, "openai")).toEqual([]); + + // Gemini dated variants collapse via the same dated-snapshot strip. + const geminiPayload = [ + "gemini-2.5-flash", // include + "gemini-2.0-flash", // include + "gemini-1.5-pro-2024-05-14", // → gemini-1.5-pro (include) + ]; + expect(unclassifiedFamilies(geminiPayload, "gemini")).toEqual([]); + + // Anthropic uses a CONTIGUOUS 8-digit snapshot suffix (`-YYYYMMDD`), not the + // dashed `-YYYY-MM-DD` form. These must collapse onto their included family + // via the anthropic-specific strip, or every dated Claude id false-positives + // as drift (the incident-2 class, for Anthropic). + const anthropicPayload = [ + "claude-3-5-sonnet-20241022", // → claude-3-5-sonnet (include) + "claude-3-7-sonnet-20250219", // → claude-3-7-sonnet (include) + "claude-3-5-haiku-20241022", // → claude-3-5-haiku (include) + ]; + expect(unclassifiedFamilies(anthropicPayload, "anthropic")).toEqual([]); + }); - if (referenced.length === 0) return; // no models found to check + it("a prose provider-mode token can never enter as a candidate", () => { + // Nothing is scraped from source, so a `gemini-interactions`-style token can + // only appear if a provider's own /models returned it — and even then it is + // on NON_MODEL_TOKENS and never becomes drift. + expect(unclassifiedFamilies(["gemini-interactions"], "gemini")).toEqual([]); + }); - for (const m of referenced) { - // OpenAI model list may include versioned variants — check prefix match - const found = models.some((available) => available === m || available.startsWith(`${m}-`)); - expect(found, `Model ${m} no longer available at OpenAI`).toBe(true); - } + it("a genuinely new family IS flagged as unclassified drift", () => { + // Guard the other side: the canary must still fire for a real new family. + expect(unclassifiedFamilies(["gpt-live"], "openai")).toEqual(["gpt-live"]); + // Single-digit trailing suffix is NOT a build tag, so it stays unknown. + expect(unclassifiedFamilies(["gpt-live-1"], "openai")).toEqual(["gpt-live-1"]); }); }); // --------------------------------------------------------------------------- -// Anthropic +// OpenAI // --------------------------------------------------------------------------- -describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic model availability", () => { - it("models used in aimock tests are still available", async () => { - const models = await listAnthropicModels(process.env.ANTHROPIC_API_KEY!); - const referenced = scrapeModels( - /\b(claude-3(?:\.\d+)?-(?:opus|sonnet|haiku)(?:-\d{8})?)\b/g, - sourceFiles, - ); - - if (referenced.length === 0) return; - - for (const m of referenced) { - const found = models.some((available) => available === m || available.startsWith(m)); - expect(found, `Model ${m} no longer available at Anthropic`).toBe(true); - } +describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Chat model-family availability", () => { + it("live /models contains no unclassified family", async () => { + const models = await listOpenAIModels(process.env.OPENAI_API_KEY!); + assertNoUnclassifiedFamilies(models, "openai", "OpenAI Chat (live /models family canary)"); }); }); // --------------------------------------------------------------------------- -// Gemini +// Anthropic // --------------------------------------------------------------------------- -describe.skipIf(!process.env.GOOGLE_API_KEY)("Gemini model availability", () => { - it("models used in aimock tests are still available", async () => { - const models = await listGeminiModels(process.env.GOOGLE_API_KEY!); - const referenced = scrapeModels(/\b(gemini-(?:[\w.-]+))\b/g, sourceFiles); - - if (referenced.length === 0) return; +describe.skipIf(!process.env.ANTHROPIC_API_KEY)( + "Anthropic Claude model-family availability", + () => { + it("live /models contains no unclassified family", async () => { + const models = await listAnthropicModels(process.env.ANTHROPIC_API_KEY!); + assertNoUnclassifiedFamilies( + models, + "anthropic", + "Anthropic Claude (live /models family canary)", + ); + }); + }, +); - // Skip experimental models, live-only models, and anchor-link fragments - // scraped from markdown (e.g., "gemini-live-bidigeneratecontent") - const stable = referenced.filter( - (m) => !m.includes("-exp") && !m.includes("-live") && !m.includes("bidigeneratecontent"), - ); +// --------------------------------------------------------------------------- +// Gemini +// --------------------------------------------------------------------------- - for (const m of stable) { - const found = models.some((available) => available === m || available.startsWith(m)); - expect(found, `Model ${m} no longer available at Gemini`).toBe(true); - } +describe.skipIf(!process.env.GOOGLE_API_KEY)("Google Gemini model-family availability", () => { + it("live /models contains no unclassified family", async () => { + const models = await listGeminiModels(process.env.GOOGLE_API_KEY!); + assertNoUnclassifiedFamilies(models, "gemini", "Google Gemini (live /models family canary)"); }); }); diff --git a/src/__tests__/drift/providers-infra-error.test.ts b/src/__tests__/drift/providers-infra-error.test.ts new file mode 100644 index 00000000..34019683 --- /dev/null +++ b/src/__tests__/drift/providers-infra-error.test.ts @@ -0,0 +1,77 @@ +/** + * Unit test for structured numeric `status` on InfraError (C5.1). + * + * Verifies that errors thrown through the `assertOk` → `withInfraErrorTag` + * pipeline carry a first-class numeric `.status` field, so downstream + * classification (C5.2 Slack, C5.3a preflight) can key off + * `status === 401 || status === 403` → stale-key vs `429`/`5xx` → infra-transient + * WITHOUT parsing prose message strings. + * + * RED captured (pre-change): `e.status` was `undefined` — no numeric status. + * GREEN (post-change): `e.status` is the exact HTTP status code. + */ +import { describe, it, expect } from "vitest"; +import { listOpenAIModels } from "./providers.js"; + +// Helper: build a minimal Response-like object +function fakeResponse(status: number, body = "Unauthorized"): Response { + return new Response(body, { status }); +} + +describe("InfraError structured status (C5.1)", () => { + it("GREEN: caught error exposes status === 401 (stale-key class)", async () => { + const origFetch = globalThis.fetch; + globalThis.fetch = async () => fakeResponse(401, "Unauthorized"); + try { + await listOpenAIModels("fake-key"); + throw new Error("should have thrown"); + } catch (err: unknown) { + expect(err).toBeInstanceOf(Error); + const e = err as Error & { status?: number }; + // Must be numeric 401, not undefined + expect(typeof e.status).toBe("number"); + expect(e.status).toBe(401); + // Human-readable INFRA_ERROR prose must still be present in message + expect((e as Error).message).toMatch(/INFRA_ERROR/); + } finally { + globalThis.fetch = origFetch; + } + }); + + it("GREEN: caught error exposes status === 403 (stale-key class)", async () => { + const origFetch = globalThis.fetch; + globalThis.fetch = async () => fakeResponse(403, "Forbidden"); + try { + await listOpenAIModels("fake-key"); + throw new Error("should have thrown"); + } catch (err: unknown) { + expect(err).toBeInstanceOf(Error); + const e = err as Error & { status?: number }; + expect(typeof e.status).toBe("number"); + expect(e.status).toBe(403); + expect((e as Error).message).toMatch(/INFRA_ERROR/); + } finally { + globalThis.fetch = origFetch; + } + }); + + it("GREEN: caught error exposes status === 503 (infra-transient class)", async () => { + const origFetch = globalThis.fetch; + // 503 is in RETRYABLE_STATUSES; fetchWithRetry retries attempts 0 and 1, + // then returns the response on attempt 2 (the last) without further retry. + // assertOk then fires on the 503 response. + globalThis.fetch = async () => fakeResponse(503, "Service Unavailable"); + try { + await listOpenAIModels("fake-key"); + throw new Error("should have thrown"); + } catch (err: unknown) { + expect(err).toBeInstanceOf(Error); + const e = err as Error & { status?: number }; + expect(typeof e.status).toBe("number"); + expect(e.status).toBe(503); + expect((e as Error).message).toMatch(/INFRA_ERROR/); + } finally { + globalThis.fetch = origFetch; + } + }); +}); diff --git a/src/__tests__/drift/providers.ts b/src/__tests__/drift/providers.ts index 03c80967..c2b2c309 100644 --- a/src/__tests__/drift/providers.ts +++ b/src/__tests__/drift/providers.ts @@ -27,6 +27,27 @@ interface StreamResult { rawEvents: { type: string; data: unknown }[]; } +/** + * Structured error carrying the HTTP status code as a first-class numeric field. + * + * Downstream classifiers (Slack alerts, preflight checks) MUST key off + * `error.status` — never parse the prose message string. + * + * 401 | 403 → stale-key (replace the API key) + * 429 → rate-limited (back off) + * 5xx → infra-transient (retry / alert ops) + * network → no status field (instanceof check first) + */ +export class InfraError extends Error { + readonly status: number; + + constructor(message: string, status: number) { + super(message); + this.name = "InfraError"; + this.status = status; + } +} + // --------------------------------------------------------------------------- // Retry helper // --------------------------------------------------------------------------- @@ -71,7 +92,10 @@ function redactUrl(url: string): string { function assertOk(raw: string, status: number, context: string, url?: string): void { if (status >= 400) { const urlSuffix = url ? ` (${redactUrl(url)})` : ""; - throw new Error(`${context}: API returned ${status}${urlSuffix}: ${raw.slice(0, 300)}`); + throw new InfraError( + `${context}: API returned ${status}${urlSuffix}: ${raw.slice(0, 300)}`, + status, + ); } } @@ -157,7 +181,8 @@ function toSSEEventShapes(events: { type: string; data: unknown }[]): SSEEventSh function withInfraErrorTag(provider: string, fn: () => Promise): Promise { return fn().catch((err: unknown) => { const msg = err instanceof Error ? err.message : String(err); - throw new Error(`INFRA_ERROR: ${provider} — ${msg}`); + const status = err instanceof InfraError ? err.status : 0; + throw new InfraError(`INFRA_ERROR: ${provider} — ${msg}`, status); }); } diff --git a/src/__tests__/drift/voice-models.ts b/src/__tests__/drift/voice-models.ts new file mode 100644 index 00000000..57b24895 --- /dev/null +++ b/src/__tests__/drift/voice-models.ts @@ -0,0 +1,144 @@ +/** + * Known voice/audio model FAMILIES + drift detection for the OpenAI realtime + * canary. Detection compares each candidate's NORMALIZED family (trailing dated + * snapshot / build-tag suffixes stripped) against a known-family set, so dated + * snapshots of a known family don't churn as false-positive drift. + * + * Extracted into its own side-effect-free module (no `describe`/`beforeAll`) so + * both the live canary in ws-realtime.drift.ts AND its unit test can import the + * SAME detection code path without the unit test transitively registering the + * drift suite (which would spin up the drift server). + */ + +import { normalizeModelFamily } from "./model-family.js"; + +/** + * The GA realtime model FAMILIES. At least one voice/audio model whose + * normalized family (see `normalizeVoiceModelFamily`) is one of these MUST + * appear in the account's model list, otherwise the family was renamed/removed + * (NO_GA drift). Entries are already normalized family keys — dated snapshots + * such as `gpt-realtime-2025-08-28` collapse onto `gpt-realtime` and so match + * here without being listed separately. + */ +export const gaRealtimeModels = [ + "gpt-realtime", + "gpt-realtime-2", + "gpt-realtime-2.1", + "gpt-realtime-2.1-mini", + "gpt-realtime-1.5", + "gpt-realtime-mini", +]; + +/** + * Normalize a voice/audio model id to its FAMILY KEY by stripping trailing + * version/snapshot suffixes that OpenAI appends to already-known families. New + * dated snapshots of an existing family land constantly (`tts-1-1106`, + * `gpt-audio-2025-08-28`, `gpt-4o-mini-tts-2025-12-15`, …); appending every one + * to a known-ID set never converges and turns the daily drift job permanently + * red on false positives. Comparing the NORMALIZED family instead means only a + * genuinely new family (e.g. `gpt-live`) is ever flagged. + * + * This is a thin OpenAI-provider wrapper over the shared `normalizeModelFamily` + * primitive (see `model-family.ts`), which owns the dated-snapshot/build-tag + * strip loop. Behavior is byte-identical to the historical inline implementation + * — a short numeric suffix like `gpt-live-1`'s trailing `-1` is a SINGLE digit + * and is deliberately NOT stripped, so `gpt-live-1` normalizes to `gpt-live-1` + * — an unknown family — and stays flagged (the whole point of the canary). + */ +export function normalizeVoiceModelFamily(id: string): string { + return normalizeModelFamily(id, "openai"); +} + +/** + * The set of voice/audio model FAMILIES we already know about, keyed by the + * normalized family (see `normalizeVoiceModelFamily`). A voice/audio model whose + * NORMALIZED family is not in this set is surfaced as new/unknown drift, so a + * newly-shipped family (e.g. `gpt-live`) is flagged the first time it appears — + * while dated snapshots of a known family (e.g. `gpt-audio-2025-08-28`) collapse + * onto their family and stay green. + * + * The listed ids are the family keys; the seed values are already normalized + * (they carry no dated/build suffix), so building the set through + * `normalizeVoiceModelFamily` is idempotent and keeps the two in lockstep. + */ +export const knownVoiceModelFamilies = new Set( + [ + // GA realtime family (dated/versioned variants normalize onto these). + "gpt-realtime", + "gpt-realtime-2", + "gpt-realtime-2.1", + "gpt-realtime-2.1-mini", + "gpt-realtime-1.5", + "gpt-realtime-mini", + // Translate/whisper realtime variants + "gpt-realtime-translate", + "gpt-realtime-whisper", + // Audio models also valid in realtime sessions + "gpt-audio", + "gpt-audio-1.5", + "gpt-audio-mini", + // Transcription/translation models + "gpt-4o-transcribe", + "gpt-4o-mini-transcribe", + "gpt-4o-transcribe-diarize", + "whisper-1", + // Legacy preview models (may still appear) + "gpt-4o-realtime-preview", + "gpt-4o-mini-realtime-preview", + // TTS / speech-out models (voice family, no "realtime" substring) + "gpt-4o-mini-tts", + "tts-1", + "tts-1-hd", + ].map(normalizeVoiceModelFamily), +); + +/** + * Match a model id that belongs to the voice/audio family the realtime canary + * is responsible for. This is DELIBERATELY broader than the old + * `id.includes("realtime")` filter: a new full-duplex voice family whose id + * lacks the "realtime" substring (e.g. OpenAI's `gpt-live-1` / `gpt-live-1-mini`) + * would previously never enter the unknown-model computation and so slip past + * the canary silently. Matching on the broader voice/audio vocabulary closes + * that blind spot generally — the point is "a new audio/voice model family the + * account hasn't seen before gets flagged", not a one-off hardcode of gpt-live. + * + * Chat/text/image/embedding models (gpt-4o, gpt-5, dall-e, text-embedding-*, + * etc.) do NOT match, so they never become false-positive "unknown voice" drift. + */ +export function isVoiceModelId(id: string): boolean { + return /(?:realtime|audio|\blive\b|-live|transcribe|whisper|voice|\btts\b|-tts)/i.test(id); +} + +/** + * Result of running the known-voice-models drift detection over a model list. + */ +export interface VoiceModelDriftResult { + /** Every model id that matched the voice/audio family matcher. */ + candidateModels: string[]; + /** + * Voice/audio ids whose NORMALIZED family is not in knownVoiceModelFamilies — + * new/unknown drift. Dated snapshots of a known family (e.g. + * `gpt-audio-2025-08-28`) collapse onto their family and are NOT listed. + */ + unknown: string[]; + /** Whether at least one GA realtime model (by family) is present. */ + hasGA: boolean; +} + +/** + * The single detection code path shared by the live canary AND its unit test. + * Given a raw `GET /v1/models` id list, compute the voice/audio candidates, the + * unknown (new-family) subset, and GA presence. Keeping this pure lets the unit + * test drive the EXACT logic the live canary runs against a representative + * payload without a network call. + */ +export function detectVoiceModelDrift(models: string[]): VoiceModelDriftResult { + const candidateModels = models.filter(isVoiceModelId); + const unknown = candidateModels.filter( + (m) => !knownVoiceModelFamilies.has(normalizeVoiceModelFamily(m)), + ); + const hasGA = candidateModels.some((m) => + gaRealtimeModels.includes(normalizeVoiceModelFamily(m)), + ); + return { candidateModels, unknown, hasGA }; +} diff --git a/src/__tests__/drift/ws-realtime.drift.ts b/src/__tests__/drift/ws-realtime.drift.ts index 0fd9253f..fdb271cb 100644 --- a/src/__tests__/drift/ws-realtime.drift.ts +++ b/src/__tests__/drift/ws-realtime.drift.ts @@ -11,6 +11,7 @@ import { extractShape, compareSSESequences, formatDriftReport } from "./schema.j import { openaiRealtimeTextEventShapes, openaiRealtimeToolCallEventShapes } from "./sdk-shapes.js"; import { openaiRealtimeWS } from "./ws-providers.js"; import { listOpenAIModels } from "./providers.js"; +import { detectVoiceModelDrift } from "./voice-models.js"; import { startDriftServer, stopDriftServer, collectMockWSMessages } from "./helpers.js"; import { connectWebSocket } from "../ws-test-client.js"; @@ -35,7 +36,18 @@ const BETA_SUPPRESSED_EVENTS = new Set(["conversation.item.done"]); // --------------------------------------------------------------------------- let instance: ServerInstance; +// The known-models canary — the entire reason this suite exists — fetches the +// model list via GET /v1/models, which returns ALL models for ANY valid OpenAI +// key regardless of realtime access. So it uses OPENAI_API_KEY (the credential +// CI actually provides to the drift job) and is gated ONLY on OPENAI_API_KEY, +// guaranteeing it runs in CI and cannot silently skip. +// +// The real realtime WS *session* tests connect a live socket that legitimately +// requires realtime access, so they gate on OPENAI_REALTIME_KEY and pass that +// same credential in their session config — using the chat-only OPENAI_API_KEY +// there could produce a spurious auth-failure "drift" if the two keys differ. const OPENAI_API_KEY = process.env.OPENAI_API_KEY; +const OPENAI_REALTIME_KEY = process.env.OPENAI_REALTIME_KEY; beforeAll(async () => { instance = await startDriftServer(); @@ -50,55 +62,59 @@ afterAll(async () => { // --------------------------------------------------------------------------- describe.skipIf(!OPENAI_API_KEY)("OpenAI Realtime API drift", () => { - const config = { apiKey: OPENAI_API_KEY! }; + // Session config for the real WS realtime tests uses the realtime credential, + // NOT the chat-only OPENAI_API_KEY, so that a differing realtime key can't + // cause a spurious auth-failure "drift". The canary below does NOT use this. + const config = { apiKey: OPENAI_REALTIME_KEY! }; it("canary: GA realtime models available", async () => { - const models = await listOpenAIModels(config.apiKey); - - const gaModels = [ - "gpt-realtime", - "gpt-realtime-2", - "gpt-realtime-2025-08-28", - "gpt-realtime-1.5", - "gpt-realtime-mini", - "gpt-realtime-mini-2025-10-06", - "gpt-realtime-mini-2025-12-15", - ]; - const knownModels = new Set([ - ...gaModels, - // Translate/whisper models (also contain "realtime" in some variants) - "gpt-realtime-translate", - "gpt-realtime-whisper", - // Audio models also valid in realtime sessions - "gpt-audio-1.5", - "gpt-audio-mini", - "gpt-audio-mini-2025-10-06", - "gpt-audio-mini-2025-12-15", - // Transcription/translation models - "gpt-4o-transcribe", - "gpt-4o-mini-transcribe", - "whisper-1", - // Legacy preview models (may still appear) - "gpt-4o-realtime-preview", - "gpt-4o-mini-realtime-preview", - "gpt-4o-realtime-preview-2024-10-01", - "gpt-4o-realtime-preview-2024-12-17", - "gpt-4o-realtime-preview-2025-06-03", - "gpt-4o-mini-realtime-preview-2024-12-17", - ]); - - const realtimeModels = models.filter((m) => m.includes("realtime")); - - // At least one GA model should exist - const hasGA = realtimeModels.some((m) => gaModels.includes(m)); - expect(hasGA).toBe(true); - - // Flag unknown realtime models - const unknown = realtimeModels.filter((m) => !knownModels.has(m)); + // Fetch via GET /v1/models, which lists ALL models for ANY valid key. Use + // OPENAI_API_KEY (guaranteed present by the describe.skipIf above and the + // credential CI provides) so the canary ALWAYS runs in CI and never skips — + // gating this on OPENAI_REALTIME_KEY (which CI does NOT provide) would have + // silently skipped the one check this whole suite exists to run. + const models = await listOpenAIModels(OPENAI_API_KEY!); + + // Run the SHARED detection code path (also driven directly by the unit test + // in ws-realtime-canary.test.ts). The voice/audio family matcher is broader + // than the old `includes("realtime")` filter so a NEW voice family whose id + // lacks the "realtime" substring (e.g. gpt-live-1) is still flagged. + const { candidateModels: realtimeModels, unknown, hasGA } = detectVoiceModelDrift(models); + + // Compute the unknown-model list BEFORE the hasGA assertion. A run can be + // BOTH GA-family-gone AND carry new unknown models; because the hasGA + // assertion below throws first, the later unknown-models assertion would + // never run and its list would be lost from the NO_GA failure message (and + // therefore from the auto-fix prompt). So we carry the unknown list into the + // NO_GA marker too — no information is lost in the combined case. if (unknown.length > 0) { - console.warn(`[DRIFT] Unknown realtime models detected: ${unknown.join(", ")}`); + console.warn(`[DRIFT] Unknown voice/audio models detected: ${unknown.join(", ")}`); } - expect(unknown).toEqual([]); + + // At least one GA model should exist. Carry the OBSERVED realtime models in + // a stable custom assertion message (symmetric to UNKNOWN_REALTIME_MODELS= + // below) so that when the GA family is renamed/removed — or the credential + // cannot see any realtime models — the drift collector recognizes the + // NO_GA_REALTIME_MODELS= marker and emits a CRITICAL OpenAI-Realtime entry + // (exit 2, auto-remediated) instead of crashing to exit 1. Without the + // marker, "expected false to be true" is an unrecognized shape that the + // collector would treat as unparseable and throw on. + // + // The message ALSO carries the unknown list after a ` | UNKNOWN_REALTIME_MODELS=` + // segment so the combined (no-GA AND unknown-models-present) case does not + // lose the unknown list when this assertion short-circuits the one below. + // The collector splits the two markers apart; each list stays clean. + expect( + hasGA, + `NO_GA_REALTIME_MODELS=${realtimeModels.join(",")} | UNKNOWN_REALTIME_MODELS=${unknown.join(",")}`, + ).toBe(true); + + // Carry the FULL unknown-model list in a stable custom assertion message. + // vitest truncates the printed array in failureMessages (`…(N)`), so the + // drift collector cannot recover ids beyond the first from the array. The + // custom message is emitted verbatim and is NOT truncated, so the collector + // parses the UNKNOWN_REALTIME_MODELS= marker as its source of truth. + expect(unknown, `UNKNOWN_REALTIME_MODELS=${unknown.join(",")}`).toEqual([]); }); it.skipIf(!process.env.OPENAI_REALTIME_KEY)( diff --git a/src/__tests__/provider-auth.test.ts b/src/__tests__/provider-auth.test.ts new file mode 100644 index 00000000..b93755ca --- /dev/null +++ b/src/__tests__/provider-auth.test.ts @@ -0,0 +1,435 @@ +import { describe, it, expect, afterEach, beforeEach } from "vitest"; +import * as http from "node:http"; +import type { RecordProviderKey } from "../types.js"; +import { createServer, type ServerInstance } from "../server.js"; + +// --------------------------------------------------------------------------- +// This suite exercises the REAL forwarded-header surface: a fake upstream HTTP +// server records the auth headers aimock forwards to it on a fixture-miss +// passthrough (proxy-only). We assert on what the fake upstream actually saw — +// not on a mock of aimock's internals. +// --------------------------------------------------------------------------- + +function post( + url: string, + body: unknown, + headers?: Record, +): Promise<{ status: number; body: string }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify(body); + const parsed = new URL(url); + const req = http.request( + { + hostname: parsed.hostname, + port: parsed.port, + path: parsed.pathname, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => + resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString() }), + ); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +interface FakeUpstream { + server: http.Server; + url: string; + /** Auth-relevant headers observed on the most recent request. */ + last: () => http.IncomingHttpHeaders; + requestCount: () => number; +} + +/** + * A fake upstream that records the headers it receives and returns a minimal + * OpenAI-shaped chat completion so aimock's collapse/record path is happy. + */ +function createFakeUpstream(): Promise { + return new Promise((resolve) => { + let seen: http.IncomingHttpHeaders = {}; + let count = 0; + const server = http.createServer((req, res) => { + count++; + seen = req.headers; + // Drain the request body before responding. + req.on("data", () => {}); + req.on("end", () => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + id: "chatcmpl-fake", + object: "chat.completion", + created: Date.now(), + model: "gpt-4", + choices: [ + { + index: 0, + message: { role: "assistant", content: "hi" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }), + ); + }); + }); + server.listen(0, "127.0.0.1", () => { + const addr = server.address(); + const port = typeof addr === "object" && addr ? addr.port : 0; + resolve({ + server, + url: `http://127.0.0.1:${port}`, + last: () => seen, + requestCount: () => count, + }); + }); + }); +} + +const DUMMY = "sk-aimock-dev-ci-only"; +const REAL = "sk-real-test-123"; + +let upstream: FakeUpstream | undefined; +let recorder: ServerInstance | undefined; + +afterEach(async () => { + if (recorder) { + await new Promise((r) => recorder!.server.close(() => r())); + recorder = undefined; + } + if (upstream) { + await new Promise((r) => upstream!.server.close(() => r())); + upstream = undefined; + } +}); + +/** + * Stand up the fake upstream + an aimock proxy-only recorder pointed at it for a + * single provider, optionally with a built-in key configured for that provider. + */ +async function setup( + provider: RecordProviderKey, + providerKey?: string, +): Promise<{ recorderUrl: string }> { + upstream = await createFakeUpstream(); + recorder = await createServer([], { + port: 0, + record: { + providers: { [provider]: upstream.url }, + providerKeys: providerKey ? { [provider]: providerKey } : undefined, + proxyOnly: true, + }, + }); + return { recorderUrl: recorder.url }; +} + +const CHAT_PATH = "/v1/chat/completions"; +const CHAT_BODY = { model: "gpt-4", messages: [{ role: "user", content: "unmatched-miss" }] }; + +describe("applyProviderAuth — aimock owns the upstream key", () => { + describe("OpenAI (bearer)", () => { + it("RED baseline: with NO built-in key, a dummy caller key is forwarded verbatim", async () => { + const { recorderUrl } = await setup("openai"); // feature OFF + const res = await post(recorderUrl + CHAT_PATH, CHAT_BODY, { + Authorization: `Bearer ${DUMMY}`, + }); + expect(res.status).toBe(200); + // Today's behavior (and case (a)): dummy forwarded unchanged. + expect(upstream!.last().authorization).toBe(`Bearer ${DUMMY}`); + }); + + it("GREEN: built-in key set + caller sends dummy → aimock injects its own key", async () => { + const { recorderUrl } = await setup("openai", REAL); + const res = await post(recorderUrl + CHAT_PATH, CHAT_BODY, { + Authorization: `Bearer ${DUMMY}`, + }); + expect(res.status).toBe(200); + expect(upstream!.last().authorization).toBe(`Bearer ${REAL}`); + }); + + it("case (b): built-in key set + caller sends a REAL key → caller overrides", async () => { + const callerReal = "sk-caller-owns-this-999"; + const { recorderUrl } = await setup("openai", REAL); + await post(recorderUrl + CHAT_PATH, CHAT_BODY, { + Authorization: `Bearer ${callerReal}`, + }); + expect(upstream!.last().authorization).toBe(`Bearer ${callerReal}`); + }); + + it("no caller credential + built-in key set → aimock injects its own key", async () => { + const { recorderUrl } = await setup("openai", REAL); + await post(recorderUrl + CHAT_PATH, CHAT_BODY); + expect(upstream!.last().authorization).toBe(`Bearer ${REAL}`); + }); + }); + + describe("Anthropic (x-api-key)", () => { + it("built-in key set + caller sends dummy → injects x-api-key", async () => { + const { recorderUrl } = await setup("anthropic", REAL); + await post(recorderUrl + "/v1/messages", CHAT_BODY, { "x-api-key": DUMMY }); + expect(upstream!.last()["x-api-key"]).toBe(REAL); + }); + + it("caller sends a real x-api-key → forwarded unchanged", async () => { + const { recorderUrl } = await setup("anthropic", REAL); + await post(recorderUrl + "/v1/messages", CHAT_BODY, { "x-api-key": "real-anthropic-key" }); + expect(upstream!.last()["x-api-key"]).toBe("real-anthropic-key"); + }); + }); + + describe("Gemini (x-goog-api-key)", () => { + it("built-in key set + caller sends dummy → injects x-goog-api-key", async () => { + const { recorderUrl } = await setup("gemini", REAL); + await post(recorderUrl + "/v1beta/models/gemini-pro:generateContent", CHAT_BODY, { + "x-goog-api-key": DUMMY, + }); + expect(upstream!.last()["x-goog-api-key"]).toBe(REAL); + }); + }); + + describe("Cohere (bearer, via /v2/chat)", () => { + const COHERE_PATH = "/v2/chat"; + const COHERE_BODY = { + model: "command-r", + messages: [{ role: "user", content: "unmatched-miss" }], + }; + + it("built-in key set + caller sends dummy → injects Bearer", async () => { + const { recorderUrl } = await setup("cohere", REAL); + await post(recorderUrl + COHERE_PATH, COHERE_BODY, { + Authorization: `Bearer ${DUMMY}`, + }); + expect(upstream!.last().authorization).toBe(`Bearer ${REAL}`); + }); + + it("caller sends a real Bearer key → forwarded unchanged", async () => { + const callerReal = "co-caller-real-abc"; + const { recorderUrl } = await setup("cohere", REAL); + await post(recorderUrl + COHERE_PATH, COHERE_BODY, { + Authorization: `Bearer ${callerReal}`, + }); + expect(upstream!.last().authorization).toBe(`Bearer ${callerReal}`); + }); + + it("inert when no built-in key set → dummy forwarded verbatim", async () => { + const { recorderUrl } = await setup("cohere"); // feature OFF + await post(recorderUrl + COHERE_PATH, COHERE_BODY, { + Authorization: `Bearer ${DUMMY}`, + }); + expect(upstream!.last().authorization).toBe(`Bearer ${DUMMY}`); + }); + }); + + describe("Ollama (bearer, via /api/chat)", () => { + const OLLAMA_PATH = "/api/chat"; + const OLLAMA_BODY = { + model: "llama3", + messages: [{ role: "user", content: "unmatched-miss" }], + }; + + it("built-in key set + caller sends dummy → injects Bearer", async () => { + const { recorderUrl } = await setup("ollama", REAL); + await post(recorderUrl + OLLAMA_PATH, OLLAMA_BODY, { + Authorization: `Bearer ${DUMMY}`, + }); + expect(upstream!.last().authorization).toBe(`Bearer ${REAL}`); + }); + + it("inert when no built-in key set → dummy forwarded verbatim", async () => { + const { recorderUrl } = await setup("ollama"); // feature OFF + await post(recorderUrl + OLLAMA_PATH, OLLAMA_BODY, { + Authorization: `Bearer ${DUMMY}`, + }); + expect(upstream!.last().authorization).toBe(`Bearer ${DUMMY}`); + }); + }); + + describe("Azure OpenAI (api-key, via /openai/deployments/{id}/chat/completions)", () => { + const AZURE_PATH = "/openai/deployments/my-deploy/chat/completions"; + + it("built-in key set + caller sends dummy → injects api-key", async () => { + const { recorderUrl } = await setup("azure", REAL); + await post(recorderUrl + AZURE_PATH, CHAT_BODY, { "api-key": DUMMY }); + expect(upstream!.last()["api-key"]).toBe(REAL); + }); + + it("caller sends a real api-key → forwarded unchanged", async () => { + const callerReal = "azure-caller-real-xyz"; + const { recorderUrl } = await setup("azure", REAL); + await post(recorderUrl + AZURE_PATH, CHAT_BODY, { "api-key": callerReal }); + expect(upstream!.last()["api-key"]).toBe(callerReal); + }); + + it("inert when no built-in key set → dummy forwarded verbatim", async () => { + const { recorderUrl } = await setup("azure"); // feature OFF + await post(recorderUrl + AZURE_PATH, CHAT_BODY, { "api-key": DUMMY }); + expect(upstream!.last()["api-key"]).toBe(DUMMY); + }); + }); + + describe("ElevenLabs (xi-api-key, via /v1/text-to-speech/{voice})", () => { + const EL_PATH = "/v1/text-to-speech/voice-123"; + const EL_BODY = { text: "unmatched-miss speak this" }; + + it("built-in key set + caller sends dummy → injects xi-api-key", async () => { + const { recorderUrl } = await setup("elevenlabs", REAL); + await post(recorderUrl + EL_PATH, EL_BODY, { "xi-api-key": DUMMY }); + expect(upstream!.last()["xi-api-key"]).toBe(REAL); + }); + + it("caller sends a real xi-api-key → forwarded unchanged", async () => { + const callerReal = "el-caller-real-777"; + const { recorderUrl } = await setup("elevenlabs", REAL); + await post(recorderUrl + EL_PATH, EL_BODY, { "xi-api-key": callerReal }); + expect(upstream!.last()["xi-api-key"]).toBe(callerReal); + }); + + it("inert when no built-in key set → dummy forwarded verbatim", async () => { + const { recorderUrl } = await setup("elevenlabs"); // feature OFF + await post(recorderUrl + EL_PATH, EL_BODY, { "xi-api-key": DUMMY }); + expect(upstream!.last()["xi-api-key"]).toBe(DUMMY); + }); + }); + + describe("fal.ai (Authorization: Key, via /fal/queue/submit)", () => { + const FAL_PATH = "/fal/queue/submit/fal-ai/some-model"; + const FAL_BODY = { prompt: "unmatched-miss render this" }; + + it("built-in key set + caller sends dummy → injects Authorization: Key", async () => { + const { recorderUrl } = await setup("fal", REAL); + await post(recorderUrl + FAL_PATH, FAL_BODY, { + Authorization: `Key ${DUMMY}`, + }); + expect(upstream!.last().authorization).toBe(`Key ${REAL}`); + }); + + it("caller sends a real Key credential → forwarded unchanged", async () => { + const callerReal = "fal-caller-real-555"; + const { recorderUrl } = await setup("fal", REAL); + await post(recorderUrl + FAL_PATH, FAL_BODY, { + Authorization: `Key ${callerReal}`, + }); + expect(upstream!.last().authorization).toBe(`Key ${callerReal}`); + }); + + it("inert when no built-in key set → dummy forwarded verbatim", async () => { + const { recorderUrl } = await setup("fal"); // feature OFF + await post(recorderUrl + FAL_PATH, FAL_BODY, { + Authorization: `Key ${DUMMY}`, + }); + expect(upstream!.last().authorization).toBe(`Key ${DUMMY}`); + }); + }); + + describe("fixture MATCH short-circuits injection", () => { + it("a matched request never reaches the proxy, so no injection happens", async () => { + upstream = await createFakeUpstream(); + // Serve a fixture that matches the request so proxyAndRecord never runs. + recorder = await createServer( + [ + { + match: { userMessage: "matched-hit" }, + response: { content: "served from fixture" }, + }, + ], + { + port: 0, + record: { + providers: { openai: upstream.url }, + providerKeys: { openai: REAL }, + proxyOnly: true, + }, + }, + ); + const res = await post( + recorder.url + CHAT_PATH, + { model: "gpt-4", messages: [{ role: "user", content: "matched-hit" }] }, + { Authorization: `Bearer ${DUMMY}` }, + ); + expect(res.status).toBe(200); + expect(res.body).toContain("served from fixture"); + // The fake upstream was never contacted → injection path never ran. + expect(upstream!.requestCount()).toBe(0); + }); + }); +}); + +describe("readProviderKeysFromEnv", () => { + const ALL_ENV_VARS = [ + "AIMOCK_PROVIDER_OPENAI_KEY", + "AIMOCK_PROVIDER_ANTHROPIC_KEY", + "AIMOCK_PROVIDER_GEMINI_KEY", + "AIMOCK_PROVIDER_OPENROUTER_KEY", + "AIMOCK_PROVIDER_COHERE_KEY", + "AIMOCK_PROVIDER_GROK_KEY", + "AIMOCK_PROVIDER_OLLAMA_KEY", + "AIMOCK_PROVIDER_VEO_KEY", + "AIMOCK_PROVIDER_AZURE_KEY", + "AIMOCK_PROVIDER_ELEVENLABS_KEY", + "AIMOCK_PROVIDER_FAL_KEY", + ] as const; + const saved = { ...process.env }; + beforeEach(() => { + for (const name of ALL_ENV_VARS) delete process.env[name]; + }); + afterEach(() => { + process.env = { ...saved }; + }); + + it("returns undefined when no provider key env vars are set", async () => { + const { readProviderKeysFromEnv } = await import("../provider-auth.js"); + expect(readProviderKeysFromEnv({})).toBeUndefined(); + }); + + it("reads every wired provider key from its env var", async () => { + const { readProviderKeysFromEnv } = await import("../provider-auth.js"); + const keys = readProviderKeysFromEnv({ + AIMOCK_PROVIDER_OPENAI_KEY: "o", + AIMOCK_PROVIDER_ANTHROPIC_KEY: "a", + AIMOCK_PROVIDER_GEMINI_KEY: "g", + AIMOCK_PROVIDER_OPENROUTER_KEY: "or", + AIMOCK_PROVIDER_COHERE_KEY: "co", + AIMOCK_PROVIDER_GROK_KEY: "gr", + AIMOCK_PROVIDER_OLLAMA_KEY: "ol", + AIMOCK_PROVIDER_VEO_KEY: "ve", + AIMOCK_PROVIDER_AZURE_KEY: "az", + AIMOCK_PROVIDER_ELEVENLABS_KEY: "el", + AIMOCK_PROVIDER_FAL_KEY: "fa", + } as NodeJS.ProcessEnv); + expect(keys).toEqual({ + openai: "o", + anthropic: "a", + gemini: "g", + openrouter: "or", + cohere: "co", + grok: "gr", + ollama: "ol", + veo: "ve", + azure: "az", + elevenlabs: "el", + fal: "fa", + }); + }); + + it("treats an empty-string env var as unset (feature stays inert)", async () => { + const { readProviderKeysFromEnv } = await import("../provider-auth.js"); + const keys = readProviderKeysFromEnv({ + AIMOCK_PROVIDER_OPENAI_KEY: "o", + AIMOCK_PROVIDER_COHERE_KEY: "", + AIMOCK_PROVIDER_FAL_KEY: "", + } as NodeJS.ProcessEnv); + expect(keys).toEqual({ openai: "o" }); + }); +}); diff --git a/src/__tests__/ws-realtime-canary.test.ts b/src/__tests__/ws-realtime-canary.test.ts new file mode 100644 index 00000000..ac61ee1a --- /dev/null +++ b/src/__tests__/ws-realtime-canary.test.ts @@ -0,0 +1,177 @@ +/** + * Unit test for the ws-realtime known-voice-models drift canary. + * + * This drives the SAME detection code path the live canary runs + * (`detectVoiceModelDrift` from ws-realtime.drift.ts) against a representative + * `GET /v1/models` id list — no network call, but NOT a reimplemented fake: the + * exported function under test IS the one the live canary invokes. + * + * Regression guard for the voice-model-family blind spot: the canary used to + * filter the model list with `models.filter((m) => m.includes("realtime"))`, so + * a NEW full-duplex voice family whose id lacks the "realtime" substring (e.g. + * OpenAI's gpt-live-1 / gpt-live-1-mini) never entered the unknown-model + * computation and slipped past the canary silently. The broadened voice/audio + * matcher closes that blind spot generally. + */ + +import { describe, it, expect } from "vitest"; +import { + detectVoiceModelDrift, + isVoiceModelId, + knownVoiceModelFamilies, + normalizeVoiceModelFamily, +} from "./drift/voice-models.js"; +import { normalizeModelFamily } from "./drift/model-family.js"; + +// Dated-snapshot / build-tag variants of families that are ALREADY known. These +// are the exact ids the live Drift Tests run (28968203340) flagged as false +// positives before family normalization: each normalizes onto a known family +// (tts-1, tts-1-hd, gpt-audio, gpt-4o-mini-transcribe, gpt-4o-mini-tts) and so +// MUST NOT be flagged. +const KNOWN_FAMILY_SNAPSHOTS = [ + "tts-1-1106", + "tts-1-hd-1106", + "gpt-audio-2025-08-28", + "gpt-4o-mini-transcribe-2025-12-15", + "gpt-4o-mini-transcribe-2025-03-20", + "gpt-4o-mini-tts-2025-03-20", + "gpt-4o-mini-tts-2025-12-15", +]; + +// A representative GET /v1/models id list: the known GA realtime family, some +// known audio/transcribe/tts models (incl. dated snapshots of known families +// that MUST normalize onto their family), a batch of non-voice +// chat/image/embedding models that MUST NOT be flagged, and the newly-shipped +// gpt-live-* full-duplex voice family that is a genuinely new family. +const REPRESENTATIVE_MODELS = [ + // --- known voice/audio family (should stay green) --- + "gpt-realtime", + "gpt-realtime-2.1", + "gpt-realtime-mini", + "gpt-audio-mini", + "gpt-4o-transcribe", + "whisper-1", + "gpt-4o-realtime-preview", + "tts-1", + // --- dated-snapshot / build-tag variants of KNOWN families (stay green) --- + ...KNOWN_FAMILY_SNAPSHOTS, + // --- non-voice models (must NOT be flagged as voice drift) --- + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "o1", + "o3-mini", + "chatgpt-4o-latest", + "dall-e-3", + "text-embedding-3-large", + "text-embedding-3-small", + "omni-moderation-latest", + // --- NEW voice family with no "realtime" substring (the blind spot) --- + "gpt-live-1", + "gpt-live-1-mini", +]; + +describe("ws-realtime known-voice-models canary detection", () => { + it("flags a new voice family whose id lacks the 'realtime' substring (gpt-live-*)", () => { + const { unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS); + + // The whole point: the new gpt-live-* family is surfaced as unknown drift. + expect(unknown).toContain("gpt-live-1"); + expect(unknown).toContain("gpt-live-1-mini"); + }); + + it("does not flag legitimately-known voice/audio models (no false positives)", () => { + const { unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS); + + // Every known-voice model in the payload stays green. + for (const known of [ + "gpt-realtime", + "gpt-realtime-2.1", + "gpt-realtime-mini", + "gpt-audio-mini", + "gpt-4o-transcribe", + "whisper-1", + "gpt-4o-realtime-preview", + "tts-1", + ]) { + expect(knownVoiceModelFamilies.has(normalizeVoiceModelFamily(known))).toBe(true); + expect(unknown).not.toContain(known); + } + }); + + it("does not flag dated-snapshot / build-tag variants of known families", () => { + // Regression guard for live Drift Tests run 28968203340: these 7 real ids + // were flagged as critical drift by the pre-normalization id-set matcher. + // Family normalization collapses each onto an already-known family, so none + // may appear in `unknown`. + const { unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS); + for (const snapshot of KNOWN_FAMILY_SNAPSHOTS) { + expect(unknown).not.toContain(snapshot); + } + }); + + it("normalizes dated snapshots and build tags onto their family", () => { + // Dated snapshot `-YYYY-MM-DD` and build tag `-NNN`/`-NNNN` are stripped. + expect(normalizeVoiceModelFamily("tts-1-1106")).toBe("tts-1"); + expect(normalizeVoiceModelFamily("tts-1-hd-1106")).toBe("tts-1-hd"); + expect(normalizeVoiceModelFamily("gpt-audio-2025-08-28")).toBe("gpt-audio"); + expect(normalizeVoiceModelFamily("gpt-4o-mini-transcribe-2025-12-15")).toBe( + "gpt-4o-mini-transcribe", + ); + expect(normalizeVoiceModelFamily("gpt-4o-mini-tts-2025-03-20")).toBe("gpt-4o-mini-tts"); + // A single-digit trailing tag is NOT a build tag: gpt-live-1 stays a new + // family and must remain flaggable. + expect(normalizeVoiceModelFamily("gpt-live-1")).toBe("gpt-live-1"); + expect(normalizeVoiceModelFamily("gpt-live-1-mini")).toBe("gpt-live-1-mini"); + expect(knownVoiceModelFamilies.has("gpt-live-1")).toBe(false); + }); + + it("is byte-identical to the shared normalizeModelFamily(id, 'openai') primitive", () => { + // Characterization guard for the A3.2 equivalence refactor: normalizeVoiceModelFamily + // is now a thin wrapper over normalizeModelFamily(id, "openai"), and the migrated + // wrapper must yield the SAME normalization for every representative id AND an + // identical knownVoiceModelFamilies set as re-seeding through the shared primitive. + for (const id of [...REPRESENTATIVE_MODELS, ...knownVoiceModelFamilies]) { + expect(normalizeVoiceModelFamily(id)).toBe(normalizeModelFamily(id, "openai")); + } + const reseeded = new Set( + [...knownVoiceModelFamilies].map((f) => normalizeModelFamily(f, "openai")), + ); + expect([...knownVoiceModelFamilies].sort()).toEqual([...reseeded].sort()); + }); + + it("does not flag non-voice chat/image/embedding models as voice drift", () => { + const { candidateModels, unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS); + + for (const nonVoice of [ + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "o1", + "o3-mini", + "chatgpt-4o-latest", + "dall-e-3", + "text-embedding-3-large", + "text-embedding-3-small", + "omni-moderation-latest", + ]) { + expect(isVoiceModelId(nonVoice)).toBe(false); + expect(candidateModels).not.toContain(nonVoice); + expect(unknown).not.toContain(nonVoice); + } + }); + + it("reports GA realtime presence when a GA model exists", () => { + const { hasGA } = detectVoiceModelDrift(REPRESENTATIVE_MODELS); + expect(hasGA).toBe(true); + }); + + it("gpt-live-* is the ONLY unknown in the representative payload", () => { + // Confirms the matcher is neither too narrow (misses gpt-live) nor too broad + // (drags in non-voice models). The unknown set is exactly the new family. + const { unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS); + expect([...unknown].sort()).toEqual(["gpt-live-1", "gpt-live-1-mini"]); + }); +}); diff --git a/src/cli.ts b/src/cli.ts index 4e366e49..7056472d 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -8,6 +8,7 @@ import { Logger, type LogLevel } from "./logger.js"; import { watchFixtures } from "./watcher.js"; import { AGUIMock } from "./agui-mock.js"; import { resolveFixturesValue } from "./fixtures-remote.js"; +import { readProviderKeysFromEnv } from "./provider-auth.js"; import type { Fixture, ChaosConfig, RecordConfig } from "./types.js"; const HELP = ` @@ -276,6 +277,10 @@ if (values.record || values["proxy-only"]) { } record = { providers, + // aimock's own upstream keys, sourced from AIMOCK_PROVIDER_*_KEY env vars + // (not CLI flags — secrets must not appear in `ps`). Injected on a + // fixture-miss passthrough when the caller sent no/dummy credential. + providerKeys: readProviderKeysFromEnv(), // In proxy-only mode with only URL sources, fixturePath is never consumed // (recorder.ts skips disk writes when proxyOnly is set). Leave it undefined // rather than resolving a URL string as a filesystem path. diff --git a/src/config-loader.ts b/src/config-loader.ts index a02e4500..871904ca 100644 --- a/src/config-loader.ts +++ b/src/config-loader.ts @@ -90,6 +90,10 @@ export interface VectorConfig { export interface AimockConfig { llm?: { fixtures?: string; + latency?: number; + chunkSize?: number; + replaySpeed?: number; + logLevel?: "silent" | "warn" | "info" | "debug"; chaos?: ChaosConfig; record?: RecordConfig; }; @@ -115,10 +119,22 @@ export async function startFromConfig( ): Promise<{ llmock: LLMock; url: string }> { const logger = new Logger("info"); + // A non-positive replaySpeed fails calculateDelay's `speed > 0` check and applies the + // full delay rather than none. Mirrors the fixture-level guard in fixture-loader.ts. + let replaySpeed = config.llm?.replaySpeed; + if (replaySpeed != null && (!Number.isFinite(replaySpeed) || replaySpeed <= 0)) { + logger.warn(`llm.replaySpeed must be a positive number, got ${replaySpeed}. Ignoring.`); + replaySpeed = undefined; + } + // Load fixtures if specified const llmock = new LLMock({ port: overrides?.port ?? config.port ?? 0, host: overrides?.host ?? config.host ?? "127.0.0.1", + latency: config.llm?.latency, + chunkSize: config.llm?.chunkSize, + replaySpeed, + logLevel: config.llm?.logLevel, chaos: config.llm?.chaos, record: config.llm?.record, metrics: config.metrics, diff --git a/src/fal-audio.ts b/src/fal-audio.ts index 62aa6a12..f79a8a64 100644 --- a/src/fal-audio.ts +++ b/src/fal-audio.ts @@ -577,6 +577,9 @@ async function tryRecordAudioQueueWalk(args: { submitPath: pathname, body, headers: buildForwardHeaders(req), + // aimock's built-in fal key (Authorization: Key), injected by the walk on + // a no/dummy caller credential; a real caller key overrides. + builtinKey: record.providerKeys?.fal, pollIntervalMs: record.fal?.pollIntervalMs, timeoutMs: record.fal?.timeoutMs, upstreamTimeoutMs: record.upstreamTimeoutMs, diff --git a/src/fal.ts b/src/fal.ts index 9fb16ba9..268aace6 100644 --- a/src/fal.ts +++ b/src/fal.ts @@ -36,6 +36,7 @@ import { sanitizeHeaderValue, } from "./recorder.js"; import { resolveUpstreamUrl } from "./url.js"; +import { applyProviderAuth } from "./provider-auth.js"; import type { Journal } from "./journal.js"; import { audioToFalFile } from "./fal-audio.js"; @@ -885,6 +886,14 @@ export async function walkFalQueue(args: { fallbackResultPath: (requestId: string) => string; /** Warn sink for the same-origin envelope-URL gate (omitting it only mutes the warns). */ logger?: Logger; + /** + * aimock's built-in fal key, when configured. Injected as `Authorization: Key + * ` on every walk fetch (submit/status/result) if the caller sent no + * credential or a dummy placeholder; a real caller credential overrides. This + * is the single chokepoint for all fal queue walks — the queue path doesn't + * route through `proxyAndRecord`, so own-key injection lives here. + */ + builtinKey?: string; }): Promise { const { upstreamBase, @@ -897,8 +906,14 @@ export async function walkFalQueue(args: { fallbackStatusPath, fallbackResultPath, logger, + builtinKey, } = args; + // Inject aimock's own fal credential onto the walk headers up front so every + // fetch below (submit + polls) carries it. fal's scheme ignores the target + // URL, so any resolvable URL suffices. + applyProviderAuth(headers, resolveUpstreamUrl(upstreamBase, submitPath), "fal", builtinKey); + const deadline = Date.now() + timeoutMs; const perFetchTimeoutMs = clampTimeout(upstreamTimeoutMs, DEFAULT_FAL_FETCH_TIMEOUT_MS); // Bound every upstream fetch: the per-fetch timeout, additionally clamped @@ -1065,6 +1080,9 @@ async function proxyAndRecordFalQueueSubmit(args: { submitPath: strippedPath, body, headers: buildForwardHeaders(req), + // aimock's built-in fal key (Authorization: Key), injected by the walk on + // a no/dummy caller credential; a real caller key overrides. + builtinKey: record.providerKeys?.fal, pollIntervalMs: record.fal?.pollIntervalMs, timeoutMs: record.fal?.timeoutMs, upstreamTimeoutMs: record.upstreamTimeoutMs, diff --git a/src/provider-auth.ts b/src/provider-auth.ts new file mode 100644 index 00000000..b00f436e --- /dev/null +++ b/src/provider-auth.ts @@ -0,0 +1,238 @@ +import type { RecordProviderKey } from "./types.js"; + +/** + * Default marker prefix identifying a "dummy" credential that a caller sends + * only to satisfy an SDK's non-empty API-key requirement (e.g. `sk-aimock-...`). + * A caller credential that is absent OR begins with this prefix is treated as a + * placeholder that aimock is free to override with its own built-in upstream key. + * A caller credential that does NOT begin with this prefix is treated as a real + * key and forwarded verbatim (the caller overrides aimock). + * + * Overridable via the `AIMOCK_DUMMY_KEY_MARKER` env var for setups that mint + * placeholder keys under a different prefix. + */ +export const DEFAULT_DUMMY_KEY_MARKER = "sk-aimock-"; + +/** Resolve the active dummy-key marker, honoring the env override. */ +export function getDummyKeyMarker(): string { + const override = process.env.AIMOCK_DUMMY_KEY_MARKER; + return override && override.length > 0 ? override : DEFAULT_DUMMY_KEY_MARKER; +} + +/** + * How a given provider carries its API credential on the wire. aimock injects + * its built-in key using the provider's native scheme so the upstream accepts + * it. Providers whose auth is a signed/exchanged credential (Bedrock SigV4, + * Vertex AI OAuth) are deliberately absent — those are NOT simple static-key + * schemes, so aimock never rewrites their auth and always forwards the caller's + * credential unchanged. + * + * Scheme kinds (all static long-lived keys): + * - bearer `Authorization: Bearer ` (OpenAI, OpenRouter, Cohere, Grok/xAI, Ollama) + * - fal-key `Authorization: Key ` (fal.ai — note the `Key ` prefix, NOT `Bearer`) + * - x-api-key `x-api-key: ` (Anthropic) + * - x-goog-api-key `x-goog-api-key: ` (Gemini, Gemini Interactions, Veo) + * - api-key `api-key: ` (Azure OpenAI static-key auth) + * - xi-api-key `xi-api-key: ` (ElevenLabs) + * + * Note on Azure: Azure OpenAI also supports Microsoft Entra ID `Authorization: + * Bearer ` OAuth. aimock only ever injects the static `api-key` header, + * and a real Entra bearer token from the caller is never dummy-prefixed, so it + * is always forwarded verbatim (the caller overrides aimock). + */ +type AuthScheme = + | { kind: "bearer" } // Authorization: Bearer + | { kind: "fal-key" } // Authorization: Key + | { kind: "x-api-key" } // x-api-key: + | { kind: "x-goog-api-key" } // x-goog-api-key: + | { kind: "api-key" } // api-key: + | { kind: "xi-api-key" }; // xi-api-key: + +const PROVIDER_AUTH_SCHEMES: Partial> = { + // Bearer-token providers. + openai: { kind: "bearer" }, + openrouter: { kind: "bearer" }, + cohere: { kind: "bearer" }, + grok: { kind: "bearer" }, // xAI + ollama: { kind: "bearer" }, // Ollama Cloud / bearer-gated Ollama servers + // Anthropic. + anthropic: { kind: "x-api-key" }, + // Google (Gemini + Veo) use the x-goog-api-key header scheme. + gemini: { kind: "x-goog-api-key" }, + "gemini-interactions": { kind: "x-goog-api-key" }, + veo: { kind: "x-goog-api-key" }, + // Azure OpenAI static-key auth. + azure: { kind: "api-key" }, + // ElevenLabs. + elevenlabs: { kind: "xi-api-key" }, + // fal.ai — uses the `Key ` prefix on Authorization, NOT `Bearer `. + fal: { kind: "fal-key" }, +}; + +/** + * Case-insensitively delete a header from a forward-header map, returning the + * value that was present (if any). Header maps preserve the original casing of + * incoming headers, so we can't assume a canonical key. + */ +function takeHeader(headers: Record, name: string): string | undefined { + const lowerName = name.toLowerCase(); + let found: string | undefined; + for (const key of Object.keys(headers)) { + if (key.toLowerCase() === lowerName) { + found = headers[key]; + delete headers[key]; + } + } + return found; +} + +/** + * Extract the caller's credential value for a given auth scheme from the + * forwarded headers, if present. For bearer, strips the `Bearer ` prefix. + */ +function readCallerCredential( + headers: Record, + scheme: AuthScheme, +): string | undefined { + if (scheme.kind === "bearer" || scheme.kind === "fal-key") { + const raw = scanHeader(headers, "authorization"); + if (raw === undefined) return undefined; + // Strip the scheme prefix (`Bearer ` for bearer, `Key ` for fal) if present, + // otherwise treat the whole value as the credential. + const prefix = scheme.kind === "fal-key" ? "Key" : "Bearer"; + const match = new RegExp(`^\\s*${prefix}\\s+(.+)$`, "i").exec(raw); + return match ? match[1].trim() : raw.trim(); + } + return scanHeader(headers, authHeaderName(scheme)); +} + +/** + * The wire header name a non-Authorization scheme carries its credential in. + * (bearer/fal-key both use `Authorization` and are handled separately.) + */ +function authHeaderName(scheme: AuthScheme): string { + switch (scheme.kind) { + case "x-api-key": + return "x-api-key"; + case "x-goog-api-key": + return "x-goog-api-key"; + case "api-key": + return "api-key"; + case "xi-api-key": + return "xi-api-key"; + case "bearer": + case "fal-key": + return "authorization"; + } +} + +/** Case-insensitive header lookup that does not mutate the map. */ +function scanHeader(headers: Record, name: string): string | undefined { + const lowerName = name.toLowerCase(); + for (const key of Object.keys(headers)) { + if (key.toLowerCase() === lowerName) return headers[key]; + } + return undefined; +} + +/** + * Decide whether aimock should override the caller's credential with its own + * built-in key. Override when the caller sent no credential OR sent a dummy + * placeholder (prefixed with the active marker). A real caller key is left + * untouched so the caller can always override aimock. + */ +function shouldOverride(callerCredential: string | undefined, marker: string): boolean { + if (callerCredential === undefined || callerCredential.length === 0) return true; + return callerCredential.startsWith(marker); +} + +/** + * Inject aimock's built-in upstream credential into the forwarded headers when + * appropriate (opt-in, backward-compatible). Mutates `forwardHeaders` in place + * and may mutate `target` (query-param schemes; none currently). + * + * Precedence: + * - No built-in key configured for this provider → no-op (forward verbatim). + * - Provider not a static-key scheme → no-op (Bedrock SigV4 / Vertex AI OAuth). + * - Caller credential absent or dummy-prefixed → inject built-in key. + * - Caller credential is a real key → forward it unchanged. + * + * @param forwardHeaders headers already built by buildForwardHeaders(req) + * @param target upstream URL (reserved for query-param auth schemes) + * @param providerKey which provider this request is routed to + * @param builtinKey aimock's configured key for this provider (if any) + */ +export function applyProviderAuth( + forwardHeaders: Record, + target: URL, + providerKey: RecordProviderKey, + builtinKey: string | undefined, +): void { + void target; // reserved for future query-param schemes (e.g. Gemini ?key=) + if (!builtinKey) return; // feature inert for this provider + + const scheme = PROVIDER_AUTH_SCHEMES[providerKey]; + if (!scheme) return; // signed/OAuth provider — never rewrite auth + + const marker = getDummyKeyMarker(); + const callerCredential = readCallerCredential(forwardHeaders, scheme); + if (!shouldOverride(callerCredential, marker)) return; // caller sent a real key + + // Strip any existing auth headers for this scheme (across casings) before + // setting aimock's key, so a dummy caller key can't linger alongside ours. + switch (scheme.kind) { + case "bearer": + takeHeader(forwardHeaders, "authorization"); + forwardHeaders["Authorization"] = `Bearer ${builtinKey}`; + break; + case "fal-key": + takeHeader(forwardHeaders, "authorization"); + forwardHeaders["Authorization"] = `Key ${builtinKey}`; + break; + case "x-api-key": + takeHeader(forwardHeaders, "x-api-key"); + forwardHeaders["x-api-key"] = builtinKey; + break; + case "x-goog-api-key": + takeHeader(forwardHeaders, "x-goog-api-key"); + forwardHeaders["x-goog-api-key"] = builtinKey; + break; + case "api-key": + takeHeader(forwardHeaders, "api-key"); + forwardHeaders["api-key"] = builtinKey; + break; + case "xi-api-key": + takeHeader(forwardHeaders, "xi-api-key"); + forwardHeaders["xi-api-key"] = builtinKey; + break; + } +} + +/** + * Read per-provider built-in keys from the environment. Returns undefined when + * no provider key is configured so the feature stays fully inert by default. + * + * Each wired static-key provider reads `AIMOCK_PROVIDER__KEY`. An + * empty-string value is treated as unset (existing truthiness pattern), keeping + * the feature inert. `gemini-interactions` is not read here: it reuses the + * Gemini key via the lookup-key remap in `proxyAndRecord`. Signed/OAuth + * providers (`bedrock`, `vertexai`) have no env var — aimock never rewrites + * their auth. + */ +export function readProviderKeysFromEnv( + env: NodeJS.ProcessEnv = process.env, +): Partial> | undefined { + const keys: Partial> = {}; + if (env.AIMOCK_PROVIDER_OPENAI_KEY) keys.openai = env.AIMOCK_PROVIDER_OPENAI_KEY; + if (env.AIMOCK_PROVIDER_ANTHROPIC_KEY) keys.anthropic = env.AIMOCK_PROVIDER_ANTHROPIC_KEY; + if (env.AIMOCK_PROVIDER_GEMINI_KEY) keys.gemini = env.AIMOCK_PROVIDER_GEMINI_KEY; + if (env.AIMOCK_PROVIDER_OPENROUTER_KEY) keys.openrouter = env.AIMOCK_PROVIDER_OPENROUTER_KEY; + if (env.AIMOCK_PROVIDER_COHERE_KEY) keys.cohere = env.AIMOCK_PROVIDER_COHERE_KEY; + if (env.AIMOCK_PROVIDER_GROK_KEY) keys.grok = env.AIMOCK_PROVIDER_GROK_KEY; + if (env.AIMOCK_PROVIDER_OLLAMA_KEY) keys.ollama = env.AIMOCK_PROVIDER_OLLAMA_KEY; + if (env.AIMOCK_PROVIDER_VEO_KEY) keys.veo = env.AIMOCK_PROVIDER_VEO_KEY; + if (env.AIMOCK_PROVIDER_AZURE_KEY) keys.azure = env.AIMOCK_PROVIDER_AZURE_KEY; + if (env.AIMOCK_PROVIDER_ELEVENLABS_KEY) keys.elevenlabs = env.AIMOCK_PROVIDER_ELEVENLABS_KEY; + if (env.AIMOCK_PROVIDER_FAL_KEY) keys.fal = env.AIMOCK_PROVIDER_FAL_KEY; + return Object.keys(keys).length > 0 ? keys : undefined; +} diff --git a/src/recorder.ts b/src/recorder.ts index eb299688..7b518271 100644 --- a/src/recorder.ts +++ b/src/recorder.ts @@ -20,6 +20,7 @@ import type { Logger } from "./logger.js"; import { collapseStreamingResponse, capturedRedactedData } from "./stream-collapse.js"; import { writeErrorResponse } from "./sse-writer.js"; import { resolveUpstreamUrl } from "./url.js"; +import { applyProviderAuth } from "./provider-auth.js"; import { getTestId, slugifyTestId, slugifyContext } from "./helpers.js"; import { DEFAULT_TEST_ID } from "./constants.js"; @@ -431,6 +432,12 @@ export async function proxyAndRecord( // Forward all request headers except hop-by-hop and client-set ones. const forwardHeaders = buildForwardHeaders(req); + // If aimock owns a built-in upstream key for this provider, inject it now + // (opt-in, backward-compatible). A real caller credential overrides it; an + // absent or dummy-prefixed caller credential is replaced. gemini-interactions + // reuses the Gemini key, mirroring the upstream-URL remap above. + applyProviderAuth(forwardHeaders, target, providerKey, record.providerKeys?.[lookupKey]); + const requestBody = rawBody ?? JSON.stringify(request); // Make upstream request diff --git a/src/types.ts b/src/types.ts index ab8f88ca..b7a0aebe 100644 --- a/src/types.ts +++ b/src/types.ts @@ -694,6 +694,19 @@ export type RecordProviderKey = export interface RecordConfig { providers: Partial>; + /** + * aimock's own built-in upstream API keys, keyed by provider. Opt-in and + * backward-compatible: when set for a provider, aimock injects the key on a + * fixture-miss passthrough IF the caller sent no credential or a dummy + * placeholder (see `applyProviderAuth`); a real caller key always overrides. + * Only static-key providers are eligible: OpenAI/OpenRouter/Cohere/Grok/Ollama + * (bearer), Anthropic (x-api-key), Gemini/Gemini-Interactions/Veo + * (x-goog-api-key), Azure OpenAI (api-key), ElevenLabs (xi-api-key), and fal + * (Authorization: Key). Signed/OAuth providers (Bedrock SigV4, Vertex AI + * OAuth) are never rewritten. Sourced from AIMOCK_PROVIDER_*_KEY env vars by + * the CLI (secrets stay out of `ps`). + */ + providerKeys?: Partial>; fixturePath?: string; /** Proxy unmatched requests without saving fixtures or caching in memory. */ proxyOnly?: boolean;