diff --git a/.claude-plugin/marketplace.json b/.claude-plugin/marketplace.json index 7ed79d85..a3571ced 100644 --- a/.claude-plugin/marketplace.json +++ b/.claude-plugin/marketplace.json @@ -9,7 +9,7 @@ "source": { "source": "npm", "package": "@copilotkit/aimock", - "version": "^1.35.0" + "version": "^1.35.1" }, "description": "Fixture authoring skill for @copilotkit/aimock — LLM, multimedia (image/TTS/transcription/video), MCP, A2A, AG-UI, vector, embeddings, structured output, sequential responses, streaming physics, record/replay, agent loop patterns, and debugging" } diff --git a/.claude-plugin/plugin.json b/.claude-plugin/plugin.json index 96a75c9d..0c719560 100644 --- a/.claude-plugin/plugin.json +++ b/.claude-plugin/plugin.json @@ -1,6 +1,6 @@ { "name": "aimock", - "version": "1.35.0", + "version": "1.35.1", "description": "Fixture authoring guidance for @copilotkit/aimock — LLM, multimedia, MCP, A2A, AG-UI, vector, and service mocking", "author": { "name": "CopilotKit" diff --git a/.github/dependabot.yml b/.github/dependabot.yml deleted file mode 100644 index e392e17e..00000000 --- a/.github/dependabot.yml +++ /dev/null @@ -1,32 +0,0 @@ -version: 2 - -updates: - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "daily" - groups: - minor-and-patch: - patterns: - - "*" - update-types: - - "minor" - - "patch" - # Workaround for dependabot/dependabot-core#14202: without an explicit - # major group, major updates matching the minor-and-patch pattern are - # silently suppressed. Remove this group when #14202 is fixed to get - # individual (ungrouped) PRs per major bump instead. - major: - patterns: - - "*" - update-types: - - "major" - labels: - - "dependencies" - - "github-actions" - commit-message: - prefix: "ci" - include: "scope" - open-pull-requests-limit: 10 - cooldown: - default-days: 1 diff --git a/.github/workflows/dependabot-auto-merge.yml b/.github/workflows/dependabot-auto-merge.yml deleted file mode 100644 index b7a6e555..00000000 --- a/.github/workflows/dependabot-auto-merge.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: Dependabot Auto-Merge (Minor/Patch) - -on: - pull_request_target: - types: [opened, synchronize] - -permissions: - contents: write - pull-requests: write - -jobs: - auto-merge: - runs-on: ubuntu-latest - if: github.event.pull_request.user.login == 'dependabot[bot]' - steps: - - name: Fetch Dependabot metadata - id: metadata - uses: dependabot/fetch-metadata@25dd0e34f4fe68f24cc83900b1fe3fe149efef98 # v3.1.0 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Auto-approve and merge minor/patch github-actions updates - if: >- - steps.metadata.outputs.package-ecosystem == 'github_actions' && - (steps.metadata.outputs.update-type == 'version-update:semver-minor' || - steps.metadata.outputs.update-type == 'version-update:semver-patch') - env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_URL: ${{ github.event.pull_request.html_url }} - run: | - gh pr review "$PR_URL" --approve - gh pr merge "$PR_URL" --auto --merge diff --git a/.github/workflows/dependabot-major-analysis.yml b/.github/workflows/dependabot-major-analysis.yml deleted file mode 100644 index c51ce55c..00000000 --- a/.github/workflows/dependabot-major-analysis.yml +++ /dev/null @@ -1,144 +0,0 @@ -name: Dependabot Major Version Analysis - -on: - pull_request_target: - types: [opened] - -permissions: - contents: read - pull-requests: write - -jobs: - analyze-major: - runs-on: ubuntu-latest - if: github.event.pull_request.user.login == 'dependabot[bot]' - steps: - - name: Fetch Dependabot metadata - id: metadata - uses: dependabot/fetch-metadata@25dd0e34f4fe68f24cc83900b1fe3fe149efef98 # v3.1.0 - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Analyze major version bump - if: >- - steps.metadata.outputs.package-ecosystem == 'github_actions' && - steps.metadata.outputs.update-type == 'version-update:semver-major' - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - env: - DEP_NAME: ${{ steps.metadata.outputs.dependency-names }} - PREV_VERSION: ${{ steps.metadata.outputs.previous-version }} - NEW_VERSION: ${{ steps.metadata.outputs.new-version }} - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const depName = process.env.DEP_NAME; - const prevVersion = process.env.PREV_VERSION; - const newVersion = process.env.NEW_VERSION; - const parts = depName.split('/'); - const owner = parts[0]; - const repo = parts[1]; - const repoSlug = `${owner}/${repo}`; - - let releases = []; - try { - const { data } = await github.rest.repos.listReleases({ owner, repo, per_page: 50 }); - releases = data; - } catch (err) { - core.warning(`Could not fetch releases for ${repoSlug}: ${err.message}`); - } - - const prevMajor = parseInt(prevVersion.replace(/^v/, ''), 10); - const newMajor = parseInt(newVersion.replace(/^v/, ''), 10); - - const relevantReleases = releases.filter(r => { - const major = parseInt(r.tag_name.replace(/^v/, ''), 10); - return major > prevMajor && major <= newMajor; - }); - - let releaseNotesSummary = ''; - let breakingChanges = ''; - - if (relevantReleases.length === 0) { - releaseNotesSummary = '_No releases found between these versions._'; - breakingChanges = `_Unable to determine breaking changes automatically. Please review the [full changelog](https://github.com/${repoSlug}/releases)._`; - } else { - for (const release of relevantReleases.slice(0, 10)) { - const body = (release.body || '_No release notes._').replace(/(?<=^|\s)@(?=[a-zA-Z0-9])(?![a-zA-Z0-9-]*\/)/gm, ''); - releaseNotesSummary += `### ${release.tag_name}${release.name && release.name !== release.tag_name ? ' — ' + release.name : ''}\n\n`; - releaseNotesSummary += body.substring(0, 2000); - if (body.length > 2000) releaseNotesSummary += '\n\n_...truncated_'; - releaseNotesSummary += '\n\n---\n\n'; - const lines = body.split('\n'); - for (const line of lines) { - if (/breaking|BREAKING|removed|deprecated|incompatible|migration/i.test(line)) { - breakingChanges += `- ${line.trim()}\n`; - } - } - } - } - - if (!breakingChanges) { - breakingChanges = '_No explicit breaking changes detected in release notes. Manual review recommended._'; - } - - let commentBody = `## :warning: Major Version Update — Manual Review Required - - | Field | Value | - |-------|-------| - | **Action** | [\`${depName}\`](https://github.com/${repoSlug}) | - | **Previous** | \`v${prevVersion}\` | - | **New** | \`v${newVersion}\` | - | **Type** | Major (\`v${prevMajor}\` → \`v${newMajor}\`) | - - ### Breaking Changes - - ${breakingChanges} - - ### Release Notes (v${prevMajor + 1} → v${newMajor}) - - ${releaseNotesSummary} - - ### Next Steps - - 1. Review breaking changes above - 2. Check if workflow inputs/outputs changed - 3. Verify compatibility with your CI/CD configuration - - > Full changelog: https://github.com/${repoSlug}/releases - - --- - _Generated automatically for Dependabot major version PRs._`.replace(/^ /gm, ''); - - if (commentBody.length > 64000) { - commentBody = commentBody.substring(0, 63900) + '\n\n_...comment truncated due to size limit._'; - } - - await github.rest.issues.createComment({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.pull_request.number, - body: commentBody, - }); - - try { - const labelsToAdd = ['major-update', 'needs-review']; - for (const label of labelsToAdd) { - try { - await github.rest.issues.getLabel({ owner: context.repo.owner, repo: context.repo.repo, name: label }); - } catch { - const colors = { 'major-update': 'B60205', 'needs-review': 'FBCA04' }; - await github.rest.issues.createLabel({ - owner: context.repo.owner, repo: context.repo.repo, - name: label, color: colors[label] || 'EDEDED', - }); - } - } - await github.rest.issues.addLabels({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.payload.pull_request.number, - labels: labelsToAdd, - }); - } catch (err) { - core.warning(`Could not add labels: ${err.message}`); - } diff --git a/.github/workflows/fix-drift.yml b/.github/workflows/fix-drift.yml index e5610bfd..8a03713f 100644 --- a/.github/workflows/fix-drift.yml +++ b/.github/workflows/fix-drift.yml @@ -23,11 +23,25 @@ jobs: permissions: contents: write pull-requests: write + checks: read + statuses: read # issues: write # removed — no longer creating issues on drift failure steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: persist-credentials: false + - name: Mint app token + id: app-token + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: "1108748" + private-key: ${{ secrets.DEVOPS_BOT_PRIVATE_KEY }} + permission-contents: write + permission-pull-requests: write + # Read check runs + commit statuses so the merge gate can assert + # the PR is truly green before merging. + permission-checks: read + permission-statuses: read - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 with: @@ -48,33 +62,49 @@ jobs: git config user.email "drift-bot@copilotkit.ai" git checkout -B "fix/drift-$(date +%Y-%m-%d)-${RUN_ID}" - # Step 1: Detect drift and produce report + # Step 1: Detect drift and produce report. + # + # FIX #F3 (round-4) — write the report OUTSIDE the repo checkout, to + # $RUNNER_TEMP. A collector output left in the repo cwd shows up as an + # untracked file in `git status --porcelain`, which the drift-success + # predicate scores as an UNSANCTIONED_CHANGE (fail-closed) and would break + # the happy path. $RUNNER_TEMP is outside the checkout, so neither the + # predicate's changed-file scan nor createPr's staging ever sees it. (The + # .gitignore also covers `drift-report*.json` as belt-and-suspenders.) - name: Collect drift report id: detect env: OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + PRE_FIX_REPORT: ${{ runner.temp }}/drift-report.json run: | set +e - npx tsx scripts/drift-report-collector.ts + npx tsx scripts/drift-report-collector.ts --out "$PRE_FIX_REPORT" EXIT_CODE=$? set -e - echo "exit_code=$EXIT_CODE" >> $GITHUB_OUTPUT + echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT" if [ "$EXIT_CODE" -eq 2 ]; then : # critical drift found, continue + elif [ "$EXIT_CODE" -eq 5 ]; then + : # quarantine: collector encountered unparseable output, continue without aborting elif [ "$EXIT_CODE" -ne 0 ]; then - echo "::error::Collector script crashed with exit code $EXIT_CODE" + # A Step-1 collector crash is a DISTINCT failure cause from an + # auto-fix step failure. Record it as an output BEFORE aborting so + # the dedicated collector-crash alert (not the "auto-fix STEP + # failed" alert) fires and names the real cause. + echo "collector_crashed=true" >> "$GITHUB_OUTPUT" + echo "::error::Drift collector (Step 1) crashed with exit code $EXIT_CODE — this is a collector failure, NOT an auto-fix step failure" exit $EXIT_CODE fi - # Always upload the report as an artifact + # Always upload the report as an artifact (from $RUNNER_TEMP — see Step 1). - name: Upload drift report if: always() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: drift-report - path: drift-report.json + path: ${{ runner.temp }}/drift-report.json if-no-files-found: warn retention-days: 30 @@ -84,15 +114,60 @@ jobs: env: DETECT_EXIT_CODE: ${{ steps.detect.outputs.exit_code }} run: | + set -euo pipefail if [ "$DETECT_EXIT_CODE" = "2" ]; then - echo "skip=false" >> $GITHUB_OUTPUT + echo "skip=false" >> "$GITHUB_OUTPUT" + echo "quarantine=false" >> "$GITHUB_OUTPUT" echo "Critical drift detected" + elif [ "$DETECT_EXIT_CODE" = "5" ]; then + # Unparseable / quarantine: NOT a silent skip — a human must look. + echo "skip=true" >> "$GITHUB_OUTPUT" + echo "quarantine=true" >> "$GITHUB_OUTPUT" + echo "::warning::Drift collector returned quarantine (exit 5) — unparseable output, routing to human alert" else - echo "skip=true" >> $GITHUB_OUTPUT + echo "skip=true" >> "$GITHUB_OUTPUT" + echo "quarantine=false" >> "$GITHUB_OUTPUT" echo "No critical drift detected (exit code: $DETECT_EXIT_CODE) — skipping fix" fi + # Step 2b: PIN the PRE-fix report OUTSIDE the repo before the autofix LLM + # runs (INTEGRITY GUARANTEE — CR round-3 F-A). + # + # The drift-success predicate derives its ALLOWLIST's sanctioned-target set + # (union of every entry's builderFile/typesFile) from the PRE-fix report. The + # autofix step below grants the Claude Code subprocess unrestricted Write/Edit + # scoped to the repo checkout (its cwd), so a repo-committed `drift-report.json` + # is LLM-writable: a malicious/confused fix could overwrite it to name an + # arbitrary fixture as a builderFile and thereby expand the allowlist. + # + # Pinning the report to $RUNNER_TEMP — which is OUTSIDE the repo checkout and + # therefore outside the autofix tool's writable scope — makes the sanctioned + # set immutable across the autofix. Both the "Assert" and "Create PR" steps + # read `--report` from this pinned copy, NEVER the in-repo file the LLM can + # touch. The POST-fix report is separately re-generated after autofix (Step + # 4c) and so needs no pin. Runs AFTER "Check for critical diffs" (so it is + # gated by the same skip guard) and BEFORE "Auto-fix drift" (so the pin is + # captured before the LLM can touch the on-disk report). + - name: Pin pre-fix drift report (integrity) + if: steps.check.outputs.skip != 'true' + env: + PRE_FIX_REPORT: ${{ runner.temp }}/drift-report.json + PINNED_REPORT: ${{ runner.temp }}/drift-report.pinned.json + run: | + set -euo pipefail + cp "$PRE_FIX_REPORT" "$PINNED_REPORT" + echo "Pinned pre-fix drift report to $PINNED_REPORT (outside the LLM-writable repo checkout)" + # Step 3: Invoke Claude Code to fix + # + # --report reads the PINNED pre-fix report (Step 2b), NOT the default + # in-repo `drift-report.json`. The collector writes its report to + # $RUNNER_TEMP (Step 1, FIX #F3), which is OUTSIDE the repo checkout, so + # the repo-root `drift-report.json` that fix-drift.ts defaults to does not + # exist — without this flag readDriftReport throws "Drift report not + # found" and the fixer never runs. Reading the pinned copy (same source + # the Assert and Create PR steps read) also keeps the fixer's view of the + # drift consistent with the downstream integrity gate. - name: Auto-fix drift id: autofix if: steps.check.outputs.skip != 'true' @@ -101,7 +176,8 @@ jobs: ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} - run: npx tsx scripts/fix-drift.ts + PINNED_REPORT: ${{ runner.temp }}/drift-report.pinned.json + run: npx tsx scripts/fix-drift.ts --report "${PINNED_REPORT}" # Upload Claude Code output for debugging - name: Upload Claude Code logs @@ -121,6 +197,10 @@ jobs: if: steps.autofix.outcome == 'success' && steps.check.outputs.skip != 'true' run: pnpm test + # `pnpm test:drift` is a FAST fail here, but it is NO LONGER the + # authoritative signal — it is the same suite the fixer was told to make + # pass, so a fixture-relaxation cheat sails through it. The authoritative + # signal is the fresh re-collect + drift-success predicate below. - name: Verify drift resolved if: steps.autofix.outcome == 'success' && steps.check.outputs.skip != 'true' env: @@ -129,34 +209,287 @@ jobs: GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} run: pnpm test:drift + # Step 4c: Re-collect drift AUTHORITATIVELY against the LIVE SDK, writing + # to a DISTINCT path so the pre-fix report is preserved for classification. + # This is a fresh collector invocation (same capture pattern as Step 1) — + # its exit code is the authoritative "is the drift gone?" signal that the + # predicate scores. A cheat that relaxed the SDK-shape fixture still + # reports clean HERE (its own SDK leg reads the relaxed fixture), which is + # exactly why the predicate ALSO requires a real production change. + # + # FIX #F3 (round-4) — write the post-fix report to $RUNNER_TEMP, OUTSIDE + # the repo checkout, for the same reason as Step 1: a post-fix report left + # in the repo cwd is an untracked file that the predicate's changed-file + # scan would score as UNSANCTIONED_CHANGE, breaking the happy path. Writing + # it outside the checkout (and out of the autofix LLM's writable scope) + # keeps it out of `git status --porcelain` entirely. + - name: Re-collect drift (authoritative) + id: recollect + if: steps.autofix.outcome == 'success' && steps.check.outputs.skip != 'true' + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + POST_FIX_REPORT: ${{ runner.temp }}/drift-report.post-fix.json + run: | + set +e + npx tsx scripts/drift-report-collector.ts --out "$POST_FIX_REPORT" + POST_FIX_EXIT=$? + set -e + echo "post_fix_exit=$POST_FIX_EXIT" >> "$GITHUB_OUTPUT" + echo "Post-fix collector exited with code $POST_FIX_EXIT" + + - name: Upload post-fix drift report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: drift-report-post-fix + path: ${{ runner.temp }}/drift-report.post-fix.json + if-no-files-found: warn + retention-days: 30 + + # Step 4d: Assert the drift was TRULY resolved (not a fixture relaxation). + # The predicate scores three independent signals — post-fix collector + # clean, a real production mock-builder change, and no comparison-leg + # relaxation — and exits non-zero (with a distinct reason) on a cheat. + # A non-zero exit blocks the pipeline: no PR is opened, so nothing merges. + - name: Assert drift truly resolved + id: assert + if: steps.autofix.outcome == 'success' && steps.check.outputs.skip != 'true' + run: | + # Run the predicate to a log file so we capture its REAL exit code + # directly (a `VAR="$(pipeline)"` assignment would report the + # assignment's own status via PIPESTATUS, never the predicate's). + # + # --report reads the PINNED pre-fix report (Step 1b), NOT the in-repo + # drift-report.json the autofix LLM could have overwritten — the + # allowlist's sanctioned-target set is derived from it, so it must be + # the untamperable copy (CR round-3 F-A). + # Write the predicate log OUTSIDE the repo checkout ($RUNNER_TEMP): + # the predicate scans `git status --porcelain` in this same working + # directory, so a predicate.log written into cwd would appear as an + # untracked file and be scored as UNSANCTIONED_CHANGE, breaking the + # happy path (FIX #F3). + set +e + npx tsx scripts/drift-success-predicate.ts \ + --report "${PINNED_REPORT}" \ + --post-fix-report "${POST_FIX_REPORT}" \ + --post-fix-exit "${POST_FIX_EXIT}" 2>&1 | tee "${PREDICATE_LOG}" + PRED_EXIT=${PIPESTATUS[0]} + set -e + REASON="$(grep '^reason=' "${PREDICATE_LOG}" | tail -n1)" + REASON="${REASON#reason=}" + echo "reason=${REASON}" >> "$GITHUB_OUTPUT" + if [ "$PRED_EXIT" -ne 0 ]; then + echo "::error::Drift-success predicate refused (exit ${PRED_EXIT}, reason=${REASON}) — not a real drift fix, blocking PR" + exit "$PRED_EXIT" + fi + echo "Drift-success predicate: drift truly resolved" + env: + POST_FIX_EXIT: ${{ steps.recollect.outputs.post_fix_exit }} + POST_FIX_REPORT: ${{ runner.temp }}/drift-report.post-fix.json + PINNED_REPORT: ${{ runner.temp }}/drift-report.pinned.json + PREDICATE_LOG: ${{ runner.temp }}/predicate.log + # Inject git credentials only when needed for push (persist-credentials: false above) - name: Configure git for push if: steps.autofix.outcome == 'success' && success() && steps.check.outputs.skip != 'true' run: git config --local url."https://x-access-token:${TOKEN}@github.com/".insteadOf "https://github.com/" env: - TOKEN: ${{ secrets.GITHUB_TOKEN }} + TOKEN: ${{ steps.app-token.outputs.token }} # Step 5: Create PR on success + # + # Select the PR by HEAD-SHA match (not `gh pr list ... .[0]`) so a stale + # or unrelated open PR on the same head branch cannot be mistaken for the + # one we just pushed. - name: Create PR id: pr if: steps.autofix.outcome == 'success' && success() && steps.check.outputs.skip != 'true' env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GH_TOKEN: ${{ steps.app-token.outputs.token }} + POST_FIX_EXIT: ${{ steps.recollect.outputs.post_fix_exit }} + POST_FIX_REPORT: ${{ runner.temp }}/drift-report.post-fix.json + PINNED_REPORT: ${{ runner.temp }}/drift-report.pinned.json + CREATE_PR_LOG: ${{ runner.temp }}/create-pr.log run: | - npx tsx scripts/fix-drift.ts --create-pr - PR_URL=$(gh pr list --head "$(git branch --show-current)" --json url --jq '.[0].url') - if [ -z "$PR_URL" ]; then echo "No PR URL found"; exit 1; fi - echo "url=$PR_URL" >> $GITHUB_OUTPUT + set -euo pipefail + # Defense-in-depth: the workflow already asserted the predicate in the + # "Assert drift truly resolved" step, but fix-drift.ts runs it AGAIN + # (via --post-fix-report/--post-fix-exit) as an in-script gate BEFORE + # any git add/commit, so a cheat opens no PR even if the step guard is + # ever bypassed. This opens the PR and STOPS — a human reviews and + # merges (see the "NO AUTO-MERGE" comment below); fix-drift.ts records + # the predicate verdict in the PR body for that reviewer. + # + # --report reads the PINNED pre-fix report (Step 1b), NOT the in-repo + # drift-report.json — same integrity guarantee as the Assert step: the + # in-script predicate's sanctioned-target set must come from the copy + # the autofix LLM could not overwrite (CR round-3 F-A). + # Capture the script's own exit code + emitted `reason=` line so a + # fail-closed exit (e.g. version-bump-failed) is NAMED in the failure + # alert rather than reported blank. Write the log OUTSIDE the repo + # checkout ($RUNNER_TEMP) so it is never scored by any git scan. + set +e + npx tsx scripts/fix-drift.ts --create-pr \ + --report "${PINNED_REPORT}" \ + --post-fix-report "${POST_FIX_REPORT}" \ + --post-fix-exit "${POST_FIX_EXIT}" 2>&1 | tee "${CREATE_PR_LOG}" + PR_EXIT=${PIPESTATUS[0]} + set -e + if [ "$PR_EXIT" -ne 0 ]; then + PR_REASON="$(grep '^reason=' "${CREATE_PR_LOG}" | tail -n1)" + PR_REASON="${PR_REASON#reason=}" + echo "::error::Create-PR step failed (exit ${PR_EXIT}, reason=${PR_REASON:-unknown})" + echo "reason=${PR_REASON}" >> "$GITHUB_OUTPUT" + exit "$PR_EXIT" + fi + HEAD_SHA="$(git rev-parse HEAD)" + echo "Pushed head SHA: $HEAD_SHA" + # Find the open PR whose headRefOid matches the exact SHA we pushed. + PR_URL="" + PR_NUMBER="" + for attempt in $(seq 1 6); do + MATCH="$(gh pr list --state open --json url,number,headRefOid \ + --jq ".[] | select(.headRefOid == \"$HEAD_SHA\") | {url, number}" 2>/dev/null || true)" + if [ -n "$MATCH" ]; then + PR_URL="$(printf '%s' "$MATCH" | jq -r '.url')" + PR_NUMBER="$(printf '%s' "$MATCH" | jq -r '.number')" + break + fi + echo "PR for head SHA not indexed yet (attempt $attempt/6), waiting 10s..." + sleep 10 + done + if [ -z "$PR_URL" ]; then + echo "::error::No open PR found whose head matches pushed SHA $HEAD_SHA" + echo "reason=no-pr-match" >> "$GITHUB_OUTPUT" + exit 1 + fi + echo "Matched PR #$PR_NUMBER: $PR_URL" + { + echo "url=$PR_URL" + echo "number=$PR_NUMBER" + echo "head_sha=$HEAD_SHA" + } >> "$GITHUB_OUTPUT" - # Step 5.5: Auto-merge the drift fix PR - - name: Auto-merge PR - if: success() && steps.pr.outputs.url != '' + # NO AUTO-MERGE FOR THE DRIFT PATH (WS-2, round-4 — user-approved design). + # + # The drift-success predicate is a strong AUTO-FILTER, NOT a provable merge + # gate. Its authoritative "is the drift gone?" signal comes from the in- + # workflow RE-COLLECT (Step 4c), which is NOT independent of the fix: the + # collector's SDK/expected leg reads the very fixtures the autofix touched, + # so a run that relaxed an input leg reports clean here too. The allowlist + + # always-block-on-leg-edit rules make that clean signal trustworthy enough + # to FILTER OUT cheats and never-fixed drift, but they cannot PROVE the mock + # was genuinely fixed (that requires an independent SDK leg — WS-2b, out of + # scope). Given that fundamental residual, the drift path opens a PR and + # STOPS: a HUMAN reviews CI + the committed diff + the predicate verdict + # (recorded in the PR body) and merges. The merge gate is the human plus the + # branch ruleset's one-approval requirement — NOT an unattended in-workflow + # merge command. The predicate having already filtered the PR down to + # genuine-looking fixes means the human review is a confirmation step, not a + # from-scratch triage. + # + # Accepted residuals under this backstop (documented, not fixed): + # • F6 — working-tree-vs-committed TOCTOU: the predicate scores the working + # tree; the committed diff is a gated subset. Mitigated by human review + # of the COMMITTED diff and CI running on the pushed SHA. + # • WS-2b — no independent SDK leg: the re-collect is not independent of the + # fix. Mitigated by the human confirming the diff is a real mock change. + + # Alert: the Step-1 drift collector crashed (exit code other than + # 0/2/5). This is a DISTINCT cause from an auto-fix step failure — the + # fixer never ran because we could not even produce a drift report — so it + # gets its own message instead of the misleading "auto-fix STEP failed". + - name: Alert on collector crash + if: always() && steps.detect.outputs.collector_crashed == 'true' env: - GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - PR_URL: ${{ steps.pr.outputs.url }} - run: gh pr merge "$PR_URL" --merge --auto + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + COLLECTOR_EXIT: ${{ steps.detect.outputs.exit_code }} + run: | + set -euo pipefail + echo "::error::Drift collector (Step 1) crashed (exit ${COLLECTOR_EXIT}) — no drift report produced, auto-fix never ran" + if [ -z "${SLACK_WEBHOOK:-}" ]; then + echo "::error::SLACK_WEBHOOK not set — cannot send collector-crash alert" + else + MSG="💥❌ *Drift collector crashed* (Step 1, exit ${COLLECTOR_EXIT}) — no drift report was produced and no auto-fix was attempted. This is a collector failure, not an auto-fix failure. Manual intervention needed.\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" + PAYLOAD="$(jq -n --arg text "$MSG" '{text: $text}')" + curl -fsS --show-error -X POST "$SLACK_WEBHOOK" \ + -H "Content-Type: application/json" \ + -d "$PAYLOAD" + fi + exit 1 + + # Alert: the autofix step itself failed. Because that step is + # `continue-on-error: true`, on failure the job would otherwise sail on + # green with no signal. Detect outcome != 'success' (and not the skip + # case), alert distinctly, then fail the job. + # NOTE: also require the `check` step to have RUN (skip output is one of + # 'true'/'false', never empty). Without that guard this step would ALSO + # fire on an EARLY-infra failure (checkout/token/pnpm/clone/git-config — + # all before `detect`/`check`), mislabelling it "auto-fix STEP failed" + # when the fixer never ran. The end-of-job catch-all owns that window. + - name: Alert on autofix step failure + if: >- + always() && steps.check.outputs.skip == 'false' && + steps.autofix.outcome != 'success' && + steps.detect.outputs.collector_crashed != 'true' + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + AUTOFIX_OUTCOME: ${{ steps.autofix.outcome }} + run: | + set -euo pipefail + echo "::error::Auto-fix step outcome was '${AUTOFIX_OUTCOME}' (not success) — failing the job" + if [ -z "${SLACK_WEBHOOK:-}" ]; then + echo "::error::SLACK_WEBHOOK not set — cannot send autofix-failure alert" + else + MSG="🛠️❌ *Drift auto-fix STEP failed* (outcome: ${AUTOFIX_OUTCOME}) — the fixer errored before producing a PR. Manual intervention needed.\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" + PAYLOAD="$(jq -n --arg text "$MSG" '{text: $text}')" + curl -fsS --show-error -X POST "$SLACK_WEBHOOK" \ + -H "Content-Type: application/json" \ + -d "$PAYLOAD" + fi + exit 1 + + # Alert: drift collector returned quarantine (exit 5) — unparseable + # output that a human must triage. Routed here instead of the silent + # no-drift skip path. + - name: Alert on drift quarantine + if: always() && steps.check.outputs.quarantine == 'true' + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + run: | + set -euo pipefail + # A missing webhook must NOT swallow a quarantine — it is a real + # needs-human event. Emit a VISIBLE ::error:: and FAIL the step (same + # discipline as the autofix-failure alert), never a silent exit 0. + if [ -z "${SLACK_WEBHOOK:-}" ]; then + echo "::error::Drift quarantine (exit 5) — a human must inspect the drift report — AND SLACK_WEBHOOK is not set, so no alert could be sent. See run https://github.com/${REPO}/actions/runs/${RUN_ID}" + exit 1 + fi + MSG="⚠️ *Drift quarantine — needs human* — the drift collector produced unparseable output (exit 5). No auto-fix attempted; a human must inspect the drift report artifact.\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" + PAYLOAD="$(jq -n --arg text "$MSG" '{text: $text}')" + curl -fsS --show-error -X POST "$SLACK_WEBHOOK" \ + -H "Content-Type: application/json" \ + -d "$PAYLOAD" + # Quarantine is a needs-human outcome — FAIL the job so a human + # watching CI status (not just Slack) sees it (matching the + # collector-crash / autofix-failure alerts which also exit 1). The + # other alert steps are mutually exclusive with quarantine (fix-failure + # requires skip != 'true'; catch-all requires skip == ''; both false on + # quarantine where skip == 'true'), so this does not double-alert. + exit 1 - # Step 7: Slack notification on successful fix + # Slack notification on success. The drift path NEVER auto-merges (see the + # "NO AUTO-MERGE" comment above): a passing predicate opens a PR that a + # human must review and merge. The message says exactly that — it never + # claims "merged to main". - name: Notify Slack on fix success if: success() && steps.pr.outputs.url != '' env: @@ -165,24 +498,104 @@ jobs: REPO: ${{ github.repository }} RUN_ID: ${{ github.run_id }} run: | - if [ -z "$SLACK_WEBHOOK" ]; then echo "SLACK_WEBHOOK not set, skipping"; exit 0; fi - MSG="✅ *Drift auto-fix PR created with auto-merge enabled*\nPR: ${PR_URL}\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" - PAYLOAD=$(jq -n --arg text "$MSG" '{text: $text}') - curl -sf -X POST "$SLACK_WEBHOOK" \ + set -euo pipefail + if [ -z "${SLACK_WEBHOOK:-}" ]; then + echo "::error::SLACK_WEBHOOK not set — cannot send fix-success alert" + exit 0 + fi + MSG="✅ *Drift-fix PR opened — needs human review + merge*\nThe drift-success predicate passed (verdict in the PR body); a human must review CI + the diff and merge.\nPR: ${PR_URL}\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" + PAYLOAD="$(jq -n --arg text "$MSG" '{text: $text}')" + curl -fsS --show-error -X POST "$SLACK_WEBHOOK" \ -H "Content-Type: application/json" \ -d "$PAYLOAD" - # Step 8: Slack notification on fix failure + # Slack notification on failure. A missing SLACK_WEBHOOK is a VISIBLE + # ::error:: annotation (not a silent exit 0). The drift path has no merge + # step, so the failure reasons are the PR-step reason and the drift-success + # predicate's reason (the "Assert drift truly resolved" step). - name: Notify Slack on fix failure - if: failure() && steps.check.outputs.skip != 'true' + if: failure() && steps.check.outputs.skip != 'true' && steps.autofix.outcome == 'success' env: SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} REPO: ${{ github.repository }} RUN_ID: ${{ github.run_id }} + PR_REASON: ${{ steps.pr.outputs.reason }} + ASSERT_REASON: ${{ steps.assert.outputs.reason }} run: | - if [ -z "$SLACK_WEBHOOK" ]; then echo "SLACK_WEBHOOK not set, skipping"; exit 0; fi - MSG="❌ *Drift auto-fix failed* — manual intervention needed\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" - PAYLOAD=$(jq -n --arg text "$MSG" '{text: $text}') - curl -sf -X POST "$SLACK_WEBHOOK" \ + set -euo pipefail + # The PR-step reason takes precedence, then the drift-success + # predicate's reason (the "Assert drift truly resolved" step) so a + # blocked cheat is named rather than reported blank. The predicate + # reasons below (WS-6 needs-human tie-in) name the attempted-cheat / + # not-actually-fixed cause explicitly. + REASON="${PR_REASON:-${ASSERT_REASON:-}}" + case "$REASON" in + no-pr-match) DETAIL=" (no open PR matched the pushed head SHA)";; + # Drift-success predicate reasons — NEEDS HUMAN. The LOUD cheat + # reasons (comparison-leg-only / suppression-suspected) mean the + # auto-fix tried to game the drift detector rather than fix the mock. + comparison-leg-only) DETAIL=" (🚨 NEEDS HUMAN: auto-fix changed ONLY comparison-leg fixtures — a fixture-relaxation cheat, NOT a real mock fix. See run logs for the offending files)";; + suppression-suspected) DETAIL=" (🚨 NEEDS HUMAN: auto-fix edited the drift allowlist or a *.drift.ts assertion — silencing the detector, never a valid fix. See run logs)";; + unsanctioned-change) DETAIL=" (🚨 NEEDS HUMAN: auto-fix changed a file NOT on the sanctioned allowlist — deps/config/manifests, unrelated tests, or an unnamed fixture that could game the collector. See run logs for the offending files)";; + no-production-change) DETAIL=" (NEEDS HUMAN: auto-fix changed zero production mock-builder files — nothing shippable)";; + still-dirty) DETAIL=" (NEEDS HUMAN: post-fix collector still reports critical drift — the fix did not resolve it)";; + quarantine-after-fix) DETAIL=" (NEEDS HUMAN: post-fix collector returned quarantine — unparseable/untrusted output after the fix)";; + collector-infra) DETAIL=" (NEEDS HUMAN: post-fix collector infra failure — cannot trust a clean signal)";; + production-change-off-target) DETAIL=" (NEEDS HUMAN: production change did not touch any file the drift report named as a fix target — may be a legit shared-helper fix)";; + version-bump-failed) DETAIL=" (NEEDS HUMAN: the version bump / CHANGELOG step failed — refused to open an UNVERSIONED PR that would merge a fix which never publishes a release. See run logs)";; + post-fix-parse-error) DETAIL=" (NEEDS HUMAN: the post-fix collector result could not be parsed/read — empty/non-integer --post-fix-exit (e.g. a skipped recollect) or an unreadable post-fix report. Failed closed: no PR opened. See run logs)";; + git-push-failed) DETAIL=" (NEEDS HUMAN: a git checkout/add/commit/push failed while opening the PR — failed closed, no PR opened. See run logs)";; + config-error) DETAIL=" (drift-success predicate CONFIG error — missing/unreadable report or bad args; the gate itself needs attention)";; + *) DETAIL="";; + esac + if [ -z "${SLACK_WEBHOOK:-}" ]; then + echo "::error::Drift auto-fix failed${DETAIL} AND SLACK_WEBHOOK is not set — no alert could be sent. See run https://github.com/${REPO}/actions/runs/${RUN_ID}" + exit 0 + fi + MSG="❌ *Drift auto-fix failed*${DETAIL} — manual intervention needed\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" + PAYLOAD="$(jq -n --arg text "$MSG" '{text: $text}')" + curl -fsS --show-error -X POST "$SLACK_WEBHOOK" \ + -H "Content-Type: application/json" \ + -d "$PAYLOAD" + + # End-of-job CATCH-ALL for ANY job failure the specific alerts above did + # NOT cover — chiefly the EARLY-infra window (checkout / mint-app-token / + # pnpm install / clone ag-ui / git config) that runs BEFORE `detect`/ + # `check` set their outputs. Those early steps leave collector_crashed, + # quarantine, and check.outputs.skip all EMPTY, so none of the four + # specific alerts fire and the job would otherwise die red with ZERO Slack + # signal. This step is UNCONDITIONAL on the earlier step outputs (only + # `failure()` + the anti-double-alert guard below), so it fires whenever + # nothing else did. + # + # Anti-double-alert: the four specific alerts fire iff collector_crashed + # or quarantine is set, OR the fixer actually ran (check.outputs.skip is + # 'true'/'false' — never empty — once `check` executed). So this catch-all + # only fires when NONE of those hold: collector_crashed != 'true' AND + # quarantine != 'true' AND check.outputs.skip is empty (check never ran). + # That is exactly the early-infra window, and it never overlaps with the + # specific alerts. + - name: Alert on early-infra failure (catch-all) + if: >- + failure() && steps.detect.outputs.collector_crashed != 'true' && + steps.check.outputs.quarantine != 'true' && + steps.check.outputs.skip == '' + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK }} + REPO: ${{ github.repository }} + RUN_ID: ${{ github.run_id }} + run: | + set -euo pipefail + # This is an INFRA/SETUP failure — the fixer never ran (checkout / + # token mint / pnpm install / ag-ui clone / git config failed), so it + # is distinct from a drift-fix failure. A missing webhook must NOT + # swallow it: emit a VISIBLE ::error:: and fail the step. + if [ -z "${SLACK_WEBHOOK:-}" ]; then + echo "::error::Drift-fix job failed during INFRA/SETUP (before the collector ran) AND SLACK_WEBHOOK is not set — no alert could be sent. See run https://github.com/${REPO}/actions/runs/${RUN_ID}" + exit 1 + fi + MSG="🧱❌ *Drift-fix job — INFRA/SETUP failure* — the job failed during setup (checkout / token mint / pnpm install / ag-ui clone / git config) BEFORE the drift collector ran; this is not a drift-fix failure. Manual intervention needed.\nRun: https://github.com/${REPO}/actions/runs/${RUN_ID}" + PAYLOAD="$(jq -n --arg text "$MSG" '{text: $text}')" + curl -fsS --show-error -X POST "$SLACK_WEBHOOK" \ -H "Content-Type: application/json" \ -d "$PAYLOAD" diff --git a/.github/workflows/sentinel.yml b/.github/workflows/sentinel.yml new file mode 100644 index 00000000..eb9070d9 --- /dev/null +++ b/.github/workflows/sentinel.yml @@ -0,0 +1,19 @@ +name: Sentinel +on: + pull_request: + push: + branches: [main] +permissions: + contents: read +jobs: + scan: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - uses: jpr5/sentinel@3001b71ad3fc6998649be5a18e39585479a3ef5b # v1.3.4 + with: + severity: high + fail-on-findings: true diff --git a/.github/workflows/test-drift.yml b/.github/workflows/test-drift.yml index 18b099be..ed190275 100644 --- a/.github/workflows/test-drift.yml +++ b/.github/workflows/test-drift.yml @@ -6,6 +6,11 @@ on: paths: - "src/agui-types.ts" - "src/__tests__/drift/agui-schema.drift.ts" + # PR-scoped live drift leg (drift-live-pr job): trigger when the drift + # scripts, any drift test, or a drift workflow itself changes. + - "scripts/drift-*.ts" + - "src/__tests__/drift/**" + - ".github/workflows/*drift*" workflow_dispatch: # Manual trigger permissions: @@ -44,6 +49,11 @@ jobs: # persisted across every retry). Lets the alert say "confirmed across N # runs". Empty on the common green/transient path. runs: ${{ steps.drift.outputs.drift_runs }} + # Final collector exit code (0 clean/transient, 2 critical drift, 5 + # quarantine, other = crash). Promoted to a job output so the `notify` + # job can classify an exit-5 quarantine distinctly instead of folding it + # into the generic "job failed → HTTP drift" fallback (M5 backstop). + exit_code: ${{ steps.drift.outputs.exit_code }} steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 with: { persist-credentials: false } @@ -54,6 +64,89 @@ jobs: cache: pnpm - run: pnpm install --frozen-lockfile + # Per-provider key-freshness PREFLIGHT. Before spending 15 minutes on the + # full suite, do a cheap GET /models per provider. A dead/expired/revoked + # key returns 401/403 — classify that as `stale-key` and fail the job LOUD + # with a "rotate _API_KEY" message, so a dead key surfaces up + # front instead of as a late, generic streaming failure buried in the + # suite (or worse, a misleading green skip). Classification keys off the + # numeric InfraError.status (401/403), NEVER the prose message. Transient + # statuses (429/5xx/network) are NOT stale keys: the preflight warns and + # continues, since the retry-wrapped collector below handles those. + # (C5.3b, a separate task, adds the exit_code job-output + notify + # plumbing — this step is only the preflight.) + - name: Preflight — provider key freshness + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + run: | + # Written at the checkout root (cwd) so the relative provider import + # below resolves against the repo. ESM relative specifiers key off the + # module's own location, not the process cwd — a file under + # $RUNNER_TEMP would look for src/ beside itself and fail. Cleaned up + # on exit either way. + PREFLIGHT_FILE="$GITHUB_WORKSPACE/.drift-preflight.mts" + trap 'rm -f "$PREFLIGHT_FILE"' EXIT + cat > "$PREFLIGHT_FILE" <<'PREFLIGHT' + import { + InfraError, + listOpenAIModels, + listAnthropicModels, + listGeminiModels, + } from "./src/__tests__/drift/providers.js"; + + const PROVIDERS: { + name: string; + keyEnv: string; + list: (key: string) => Promise; + }[] = [ + { name: "OpenAI", keyEnv: "OPENAI_API_KEY", list: listOpenAIModels }, + { name: "Anthropic", keyEnv: "ANTHROPIC_API_KEY", list: listAnthropicModels }, + { name: "Gemini", keyEnv: "GOOGLE_API_KEY", list: listGeminiModels }, + ]; + + const stale: string[] = []; + + for (const p of PROVIDERS) { + const key = process.env[p.keyEnv]; + if (!key) { + console.error(`::error::${p.keyEnv} is not set — cannot run drift for ${p.name}`); + stale.push(p.keyEnv); + continue; + } + try { + await p.list(key); + console.log(`preflight: ${p.name} key OK`); + } catch (err) { + const status = err instanceof InfraError ? err.status : 0; + if (status === 401 || status === 403) { + // stale-key: the key is dead/expired/revoked. LOUD, classified. + console.error( + `::error title=stale-key::${p.name} preflight got HTTP ${status} — rotate ${p.keyEnv}`, + ); + stale.push(p.keyEnv); + } else { + // Transient (429/5xx/network) at preflight time is NOT a stale + // key — don't block; the retry-wrapped collector handles it. + const msg = err instanceof Error ? err.message : String(err); + console.warn( + `preflight: ${p.name} non-auth error (status ${status}), continuing: ${msg}`, + ); + } + } + } + + if (stale.length > 0) { + console.error( + `::error::Preflight failed (stale-key): rotate ${stale.join(", ")} before drift can run`, + ); + process.exit(1); + } + console.log("preflight: all provider keys fresh"); + PREFLIGHT + npx tsx "$PREFLIGHT_FILE" + # Run the collector behind a "retry before alert" wrapper. A single # critical run can be a transient real-API hiccup (a streaming call # failing mid-flight), NOT a format change — so the wrapper re-runs the @@ -76,6 +169,9 @@ jobs: echo "exit_code=$EXIT_CODE" >> "$GITHUB_OUTPUT" if [ "$EXIT_CODE" -eq 2 ]; then : # critical drift persisted across retries, continue + elif [ "$EXIT_CODE" -eq 5 ]; then + echo "::error::Drift collector quarantined unparseable output (exit 5) — manual triage required" + exit "$EXIT_CODE" elif [ "$EXIT_CODE" -ne 0 ]; then echo "::error::Collector script crashed with exit code $EXIT_CODE" exit "$EXIT_CODE" @@ -129,6 +225,7 @@ jobs: AGUI_RESULT: ${{ needs.agui-schema-drift.result }} DRIFT_SUMMARY: ${{ needs.drift.outputs.summary }} DRIFT_RUNS: ${{ needs.drift.outputs.runs }} + DRIFT_EXIT_CODE: ${{ needs.drift.outputs.exit_code }} REPO: ${{ github.repository }} RUN_ID: ${{ github.run_id }} run: | @@ -137,9 +234,17 @@ jobs: HTTP_DRIFT=false AGUI_DRIFT=false INFRA_ERROR=false + QUARANTINE=false - # Determine what happened in each job - if [ "$DRIFT_RESULT" = "failure" ]; then + # Determine what happened in each job. An exit_code of 5 means the + # collector QUARANTINED unparseable output — the drift job concludes + # `failure`, but this is NOT real HTTP API drift: it needs human + # triage, not a "providers changed formats" alert. Classify it as + # quarantine FIRST, before the generic failure→HTTP_DRIFT fallback, + # so an exit-5 quarantine is never misreported as real drift. + if [ "$DRIFT_EXIT_CODE" = "5" ]; then + QUARANTINE=true + elif [ "$DRIFT_RESULT" = "failure" ]; then HTTP_DRIFT=true elif [ "$DRIFT_RESULT" != "success" ] && [ "$DRIFT_RESULT" != "skipped" ]; then INFRA_ERROR=true @@ -175,8 +280,15 @@ jobs: CONFIRMED="${NL}_(confirmed across ${DRIFT_RUNS} runs)_" fi + # Quarantine (collector exit 5): unparseable output was quarantined + # and needs manual triage. Classified distinctly — NOT real HTTP API + # drift — so nobody chases a phantom provider format change. The + # DRIFT_SUMMARY already carries the quarantine detail block (C5.2). + if [ "$QUARANTINE" = "true" ]; then + EMOJI="🔬" + MSG="*Drift collector quarantined unparseable output* in aimock — manual triage required (not confirmed API drift).${DETAIL}${NL}${RUN_URL}" # Both types of drift - if [ "$HTTP_DRIFT" = "true" ] && [ "$AGUI_DRIFT" = "true" ]; then + elif [ "$HTTP_DRIFT" = "true" ] && [ "$AGUI_DRIFT" = "true" ]; then EMOJI="🚨" MSG="*Drift detected* in aimock — HTTP API drift + AG-UI schema drift.${DETAIL}${CONFIRMED}${NL}${RUN_URL}" # HTTP API drift only @@ -204,3 +316,451 @@ jobs: curl -sf -X POST "$SLACK_WEBHOOK" \ -H "Content-Type: application/json" \ -d "$PAYLOAD" + + # PR-scoped live drift leg (D6.3a skeleton). Triggered ONLY on `pull_request` + # — deliberately NOT `pull_request_target` (O3): a `pull_request` run on a + # fork gets a read-only GITHUB_TOKEN and NO repo secrets, so untrusted fork + # code can never exfiltrate the provider keys. The trade-off (a fork PR has + # no live provider keys and so cannot run the live leg) is handled by the + # fork→maintainer-dry-run fallback that D6.3b documents; this skeleton only + # produces the two reports on the trusted (non-fork) path. + # + # The job runs the live drift collector on BOTH the base (`main`) and the + # head (the PR ref), emitting two artifacts — `drift-report-base` and + # `drift-report-head`. For base it first tries to REUSE the most recent + # same-UTC-day, well-formed `main` scheduled report (via + # `isBaseReportReusable`); only when no reusable report exists does it run a + # fresh live base. The delta comparison / block-vs-advisory decision that + # consumes these two reports is intentionally NOT here — that is D6.3b, which + # plugs in at the "D6.3b SEAM" marker below. + drift-live-pr: + if: github.event_name == 'pull_request' + runs-on: ubuntu-latest + timeout-minutes: 30 + env: + OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} + ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }} + GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }} + steps: + # HEAD checkout: the PR ref (default checkout behavior on pull_request). + # persist-credentials:false — this leg never pushes; keeps the token off + # disk on the fork path. + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + persist-credentials: false + # Need main's history reachable so the base leg can check it out + # into a sibling worktree without a second full clone. + fetch-depth: 0 + - uses: pnpm/action-setup@0ebf47130e4866e96fce0953f49152a61190b271 # v6.0.9 + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: 24 + cache: pnpm + - run: pnpm install --frozen-lockfile + + # ---- PR preflight — per-provider key triage (graceful degradation) --- + # The PR-scoped live leg must NOT hard-block a drift-code PR merely because + # a provider key is absent or a provider is transiently down — that would + # re-introduce the live-key coupling the whole delta design set out to + # reduce. So classify each provider independently, keying off the numeric + # InfraError.status (401/403), NEVER the prose message: + # + # - NO key configured → SKIP that provider's live leg with a + # NEUTRAL ::notice:: advisory. Not a failure. + # - key present but 401/403 → LOUD `stale-key` ::error:: → block. A + # configured key that is dead is a real + # failure (rotate it) — this is the one case + # that stays red (the InfraError.status + # contract, mirroring the daily preflight). + # - transient (429/5xx/net) → warn + continue; the collector's own retry + # path handles a transient blip. + # + # `live_legs_ran` records whether ANY provider had a usable key. When it is + # `false` (all skipped for no-key — e.g. a fork PR with no secrets), the + # delta gate below is a neutral pass with a "live drift leg skipped — no + # keys" advisory, and the maintainer-dry-run fallback (documented at the + # bottom of this job) applies. + - name: PR preflight — per-provider key triage + id: preflight + run: | + set -uo pipefail + # Written at the checkout root (cwd) so the relative provider import + # resolves against the repo — ESM relative specifiers key off the + # module's own location, not the process cwd. Cleaned up on exit. + PREFLIGHT_FILE="$GITHUB_WORKSPACE/.drift-pr-preflight.mts" + trap 'rm -f "$PREFLIGHT_FILE"' EXIT + cat > "$PREFLIGHT_FILE" <<'PREFLIGHT' + import { appendFileSync } from "node:fs"; + import { + InfraError, + listOpenAIModels, + listAnthropicModels, + listGeminiModels, + } from "./src/__tests__/drift/providers.js"; + + const PROVIDERS: { + name: string; + keyEnv: string; + list: (key: string) => Promise; + }[] = [ + { name: "OpenAI", keyEnv: "OPENAI_API_KEY", list: listOpenAIModels }, + { name: "Anthropic", keyEnv: "ANTHROPIC_API_KEY", list: listAnthropicModels }, + { name: "Gemini", keyEnv: "GOOGLE_API_KEY", list: listGeminiModels }, + ]; + + const stale: string[] = []; + let liveLegs = 0; + + for (const p of PROVIDERS) { + const key = process.env[p.keyEnv]; + if (!key) { + // NO key configured → SKIP this provider's live leg. Neutral + // advisory, NOT a failure (the drift suite's own describe.skipIf + // already drops the provider's tests when the key is absent). + console.log( + `::notice title=drift-live skipped (no key)::${p.name} live drift leg skipped — ${p.keyEnv} not configured`, + ); + continue; + } + try { + await p.list(key); + liveLegs++; + console.log(`preflight: ${p.name} key OK`); + } catch (err) { + const status = err instanceof InfraError ? err.status : 0; + if (status === 401 || status === 403) { + // stale-key: a CONFIGURED key is dead/expired/revoked. LOUD, + // classified, blocking — this stays a real failure. + console.error( + `::error title=stale-key::${p.name} preflight got HTTP ${status} — rotate ${p.keyEnv}`, + ); + stale.push(p.keyEnv); + } else { + // Transient (429/5xx/network) is NOT a stale key — don't block; + // count it as a live leg and let the collector's retry handle it. + liveLegs++; + const msg = err instanceof Error ? err.message : String(err); + console.warn( + `preflight: ${p.name} non-auth error (status ${status}), continuing: ${msg}`, + ); + } + } + } + + const out = process.env.GITHUB_OUTPUT; + if (out) { + appendFileSync(out, `live_legs_ran=${liveLegs > 0 ? "true" : "false"}\n`); + } + + if (stale.length > 0) { + console.error( + `::error::PR preflight failed (stale-key): rotate ${stale.join(", ")} — a configured key is dead`, + ); + process.exit(1); + } + if (liveLegs === 0) { + console.log( + "::notice title=drift-live skipped::no provider keys configured — live drift leg skipped (neutral); maintainer-dry-run fallback applies", + ); + } + console.log(`preflight: ${liveLegs} live provider leg(s) will run`); + PREFLIGHT + npx tsx "$PREFLIGHT_FILE" + + # ---- BASE leg ------------------------------------------------------- + # Prefer reusing the most recent same-UTC-day, well-formed `main` + # scheduled drift report as the delta base — a same-day main baseline + # already captures the day's environmental drift, so reusing it avoids a + # redundant live run and keeps the PR fast. Only when no reusable report + # is available do we fall back to a fresh live base run. + # + # Reusability is decided by `isBaseReportReusable(report, conclusion, + # sameUtcDay)` from scripts/drift-delta.ts (D6.1): non-empty entries + + # known-good collector conclusion + same UTC day. The producing step + # writes `drift-report-base.json` either way, so downstream steps have a + # single stable base path regardless of which path produced it. + - name: Base drift report — reuse same-UTC-day main run, else run live + id: base + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + REPO: ${{ github.repository }} + LIVE_LEGS_RAN: ${{ steps.preflight.outputs.live_legs_ran }} + run: | + set -euo pipefail + # No provider key had a usable credential (fork PR / all keys absent). + # The live legs are skipped — write an empty, well-formed base report so + # the delta gate has a stable file to read, and stop here (neutral). + if [ "${LIVE_LEGS_RAN:-true}" != "true" ]; then + echo "base: live legs skipped (no keys) — writing empty base report" + printf '%s\n' '{"timestamp":"'"$(date -u +%Y-%m-%dT%H:%M:%SZ)"'","entries":[]}' \ + > drift-report-base.json + echo "reused=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + # Attempt reuse: download the most recent successful scheduled + # `main` run's drift-report artifact, then let isBaseReportReusable + # (D6.1) decide if it is a trustworthy same-UTC-day baseline. On any + # failure to obtain a reusable report, fall through to a live run. + REUSED=false + if BASE_RUN_ID=$(gh run list \ + --workflow="Drift Tests" --repo="$REPO" --branch=main \ + --event=schedule --status=success --limit=1 \ + --json databaseId --jq '.[0].databaseId // empty'); then + if [ -n "$BASE_RUN_ID" ]; then + rm -rf ./.base-artifact + if gh run download "$BASE_RUN_ID" --repo="$REPO" \ + --name drift-report --dir ./.base-artifact; then + BASE_CONCLUSION=$(gh run view "$BASE_RUN_ID" --repo="$REPO" \ + --json conclusion --jq '.conclusion // empty' || echo "") + if node --input-type=module -e ' + import { readFileSync } from "node:fs"; + import { isBaseReportReusable } from "./scripts/drift-delta.ts"; + const path = "./.base-artifact/drift-report.json"; + let report = null; + try { report = JSON.parse(readFileSync(path, "utf-8")); } catch {} + const conclusion = report?.conclusion ?? process.env.BASE_CONCLUSION ?? ""; + const genAt = report?.generatedAt ? new Date(report.generatedAt) : null; + const now = new Date(); + const sameUtcDay = + !!genAt && + genAt.getUTCFullYear() === now.getUTCFullYear() && + genAt.getUTCMonth() === now.getUTCMonth() && + genAt.getUTCDate() === now.getUTCDate(); + process.exit(isBaseReportReusable(report, conclusion, sameUtcDay) ? 0 : 1); + ' BASE_CONCLUSION="$BASE_CONCLUSION"; then + cp ./.base-artifact/drift-report.json drift-report-base.json + REUSED=true + echo "base: reusing same-UTC-day main scheduled report (run $BASE_RUN_ID)" + fi + fi + fi + fi + + if [ "$REUSED" != "true" ]; then + echo "base: no reusable same-UTC-day main report — running fresh live base" + # Check main out into a sibling worktree and run the collector + # there, writing the base report back into the workspace root. + git worktree add ../base-main origin/main + cd ../base-main + pnpm install --frozen-lockfile + # The collector exits 2 when it finds critical drift — that is the + # NORMAL "drift present" signal that feeds the delta gate, NOT a step + # failure. The delta gate (not this leg) decides block-vs-advisory, + # so exit 2 here must be tolerated. `set +e`/`set -e` around the run + # captures the exit code as DATA so an exit 2 does not abort under + # the step's `set -euo pipefail`: + # exit 0 → clean, continue. + # exit 2 → drift present, NON-FATAL (report feeds the delta gate). + # exit 5 → quarantine (unparseable output) — genuine fault, FAIL. + # any other non-{0,2} → genuine collector fault, FAIL. + set +e + npx tsx scripts/drift-report-collector.ts \ + --out "$GITHUB_WORKSPACE/drift-report-base.json" + BASE_EXIT=$? + set -e + cd "$GITHUB_WORKSPACE" + if [ "$BASE_EXIT" -eq 5 ]; then + echo "::error::Base collector quarantined unparseable output (exit 5) — manual triage required" + exit "$BASE_EXIT" + fi + if [ "$BASE_EXIT" -ne 0 ] && [ "$BASE_EXIT" -ne 2 ]; then + echo "::error::Base collector faulted (exit $BASE_EXIT) — not a drift signal" + exit "$BASE_EXIT" + fi + fi + echo "reused=$REUSED" >> "$GITHUB_OUTPUT" + + # ---- HEAD leg ------------------------------------------------------- + # Run the collector live against the PR head (the checked-out ref) to + # produce the head report the delta compares against the base. + # + # CRITICAL (this was the wiring bug that hard-failed the required check): + # the collector exits 2 whenever it finds ANY critical drift — that is the + # EXPECTED "drift present" signal on the head leg, NOT a step failure. The + # whole point of the two-report + delta design is that `computeDelta` + # (the delta gate below) decides block-vs-advisory by KEY PRESENCE — a + # both-present critical is advisory, only a new-in-head key blocks. Letting + # the collector's exit 2 fail THIS step meant the delta gate was skipped + # and the check died for the wrong reason (every drift-code PR would + # red-fail regardless of whether its diff introduced the drift). So exit 2 + # here is tolerated; the report file is the signal. Exit 5 (quarantine) and + # any other non-zero are genuine collector faults and DO fail the leg. + # + # When no provider key had a usable credential (fork PR / all keys absent), + # skip the live head run and write an empty, well-formed head report so the + # delta gate has a stable file — it will then neutral-pass with an advisory. + - name: Head drift report — run live on PR ref + env: + LIVE_LEGS_RAN: ${{ steps.preflight.outputs.live_legs_ran }} + run: | + set -uo pipefail + if [ "${LIVE_LEGS_RAN:-true}" != "true" ]; then + echo "head: live legs skipped (no keys) — writing empty head report" + printf '%s\n' '{"timestamp":"'"$(date -u +%Y-%m-%dT%H:%M:%SZ)"'","entries":[]}' \ + > drift-report-head.json + exit 0 + fi + # The collector exits 2 when it finds critical drift — that is the + # NORMAL "drift present" signal that feeds the delta gate below, NOT a + # step failure. GitHub runs `run:` under `bash -e`, so without an + # explicit `set +e` guard the `-e` would abort THIS step the instant + # the collector returns 2 (never reaching the capture below) — that is + # the #297 wiring bug this leg must avoid. So capture the exit code + # with `set +e`/`set -e` and treat it as DATA: + # exit 0 → clean, continue. + # exit 2 → drift present, NON-FATAL (report feeds the delta gate). + # exit 5 → quarantine (unparseable output) — genuine fault, FAIL. + # any other non-{0,2} → genuine collector fault, FAIL. + set +e + npx tsx scripts/drift-report-collector.ts --out drift-report-head.json + HEAD_EXIT=$? + set -e + if [ "$HEAD_EXIT" -eq 5 ]; then + echo "::error::Head collector quarantined unparseable output (exit 5) — manual triage required" + exit "$HEAD_EXIT" + fi + if [ "$HEAD_EXIT" -ne 0 ] && [ "$HEAD_EXIT" -ne 2 ]; then + echo "::error::Head collector faulted (exit $HEAD_EXIT) — not a drift signal" + exit "$HEAD_EXIT" + fi + echo "head: collector exit $HEAD_EXIT (0 clean / 2 drift-present — both non-fatal here)" + + - name: Upload base drift report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: drift-report-base + path: drift-report-base.json + if-no-files-found: warn + retention-days: 30 + + - name: Upload head drift report + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: drift-report-head + path: drift-report-head.json + if-no-files-found: warn + retention-days: 30 + + # =================================================================== + # D6.3b — DELTA STEP. Load the two reports produced above and run + # `computeDelta(base, head)` (scripts/drift-delta.ts, D6.1). Routing is + # by KEY PRESENCE only (#292 invariant), NEVER by DriftClass: + # - block[] → keys NEW in head (absent from base): diff-attributable. + # ANY block key fails this required check (exit 1) — a + # new-in-head *critical* blocks, and so does a new-in-head + # advisory-class finding. Class is annotation only. + # - advisory[] → keys in BOTH base and head: pre-existing / world drift. + # Surfaced as a NEUTRAL annotation + job-summary line but + # NEVER fails the check (that is the daily job's concern, + # not the PR's). + # - fixed[] → keys in base but gone in head: informational only. + # The block-only-on-new-in-head decision is the incident-3 / M-1 backstop: + # a PR is blamed strictly for drift its diff introduced. + - name: Delta gate - block new-in-head, advisory-annotate base-present + env: + LIVE_LEGS_RAN: ${{ steps.preflight.outputs.live_legs_ran }} + run: | + set -euo pipefail + # When no provider key had a usable credential (fork PR / all keys + # absent), no live leg ran — there is no diff-attributable drift to + # evaluate. Neutral-pass with a "live drift leg skipped — no keys" + # advisory (the maintainer-dry-run fallback documented below applies), + # rather than blocking a drift-code PR on absent keys. + if [ "${LIVE_LEGS_RAN:-true}" != "true" ]; then + echo "::notice title=drift-live skipped (no keys)::live drift leg skipped — no provider keys configured; delta gate is a neutral pass (maintainer-dry-run fallback applies)" + { + echo "## Drift delta" + echo "" + echo "_Live drift leg skipped — no provider keys configured. Neutral pass; the maintainer-dry-run fallback applies._" + } >> "$GITHUB_STEP_SUMMARY" + echo "Drift gate PASSED (neutral): live legs skipped — no keys." + exit 0 + fi + # Written at the checkout root (cwd) so the relative drift-delta + # import resolves against the repo — same rationale as the preflight + # step (ESM relative specifiers key off the module's own location). + # Quoted heredoc delimiter: no shell expansion inside the script. + DELTA_FILE="$GITHUB_WORKSPACE/.drift-delta-gate.mts" + trap 'rm -f "$DELTA_FILE"' EXIT + cat > "$DELTA_FILE" <<'DELTAGATE' + import { readFileSync, appendFileSync } from "node:fs"; + import { computeDelta } from "./scripts/drift-delta.js"; + import type { DeltaKey } from "./scripts/drift-delta.js"; + + const read = (p: string) => JSON.parse(readFileSync(p, "utf-8")); + const base = read("drift-report-base.json"); + const head = read("drift-report-head.json"); + + const { block, advisory, fixed } = computeDelta(base, head); + const fmt = (k: DeltaKey) => + `${k.provider} ${k.id}${k.class ? ` [${k.class}]` : ""}`; + + const summary: string[] = []; + summary.push("## Drift delta"); + summary.push(""); + summary.push(`- block (new-in-head): ${block.length}`); + summary.push(`- advisory (base-present): ${advisory.length}`); + summary.push(`- fixed (gone in head): ${fixed.length}`); + + // Advisory: pre-existing / environmental drift. Surface it as a + // NEUTRAL annotation (::notice::) and in the summary, but do NOT fail + // the check on it — that drift is the daily main job's concern. + for (const k of advisory) { + console.log(`::notice title=drift-advisory (pre-existing)::${fmt(k)}`); + } + if (advisory.length > 0) { + summary.push(""); + summary.push("### Advisory (pre-existing on main - not blamed on this PR)"); + for (const k of advisory) summary.push(`- ${fmt(k)}`); + } + + // Block: drift the diff INTRODUCED. Each is a loud error annotation; + // a non-empty block list fails the required check. + for (const k of block) { + console.log(`::error title=drift-block (new-in-head)::${fmt(k)}`); + } + if (block.length > 0) { + summary.push(""); + summary.push("### Block (introduced by this diff - required check FAILS)"); + for (const k of block) summary.push(`- ${fmt(k)}`); + } + + const summaryFile = process.env.GITHUB_STEP_SUMMARY; + if (summaryFile) { + appendFileSync(summaryFile, summary.join("\n") + "\n"); + } + + if (block.length > 0) { + console.error( + `::error::Drift gate BLOCKED: ${block.length} new-in-head failure(s) introduced by this diff: ${block.map(fmt).join(", ")}`, + ); + process.exit(1); + } + console.log( + advisory.length > 0 + ? `Drift gate PASSED: ${advisory.length} advisory (pre-existing) finding(s) surfaced, none new-in-head.` + : "Drift gate PASSED: no new-in-head drift.", + ); + DELTAGATE + npx tsx "$DELTA_FILE" + + # ---- fork → maintainer-dry-run fallback -------------------------------- + # This whole `drift-live-pr` job is gated on `pull_request` (NOT + # `pull_request_target`, O3), so a run originating from a FORK receives a + # read-only GITHUB_TOKEN and NO repo secrets — the provider API keys above + # are simply empty. Consequently a fork PR cannot run the live base/head + # collectors and this delta gate never executes meaningfully for it. + # + # Fallback protocol (maintainer dry-run): for a fork PR, a maintainer runs + # the delta gate manually from a trusted checkout — + # gh pr checkout # fetch the fork head + # npx tsx scripts/drift-report-collector.ts --out drift-report-head.json + # # (reuse or run a same-UTC-day main base as drift-report-base.json) + # node --input-type=module -e '' + # — and records the block/advisory conclusion as a PR comment. The + # branch-protection required-check registration for this job is a manual + # orchestrator step outside this diff (the check must be marked required in + # branch protection for the block=exit-1 signal to actually gate merges). + # =================================================================== diff --git a/.github/zizmor.yml b/.github/zizmor.yml index 3d691212..d6379923 100644 --- a/.github/zizmor.yml +++ b/.github/zizmor.yml @@ -4,16 +4,6 @@ # Adding a rule here REQUIRES a comment explaining why it is safe. rules: - # ─── artipacked ──────────────────────────────────────────────────────── - artipacked: - ignore: - # Dependabot auto-merge: uses pull_request_target for write token. - # Does NOT checkout PR code. Actor-gated to dependabot[bot]. - - dependabot-auto-merge.yml - # Dependabot major analysis: uses pull_request_target for PR comments. - # Does NOT checkout PR code. Actor-gated to dependabot[bot]. - - dependabot-major-analysis.yml - # ─── dangerous-triggers ──────────────────────────────────────────────── # `notify-pr.yml` uses `pull_request_target` intentionally: it needs # `secrets.SLACK_WEBHOOK` to fire on PRs from forks. It does NOT check out @@ -29,9 +19,3 @@ rules: ignore: - notify-pr.yml - fix-drift.yml - # Dependabot auto-merge: uses pull_request_target for write token. - # Does NOT checkout PR code. Actor-gated to dependabot[bot]. - - dependabot-auto-merge.yml - # Dependabot major analysis: uses pull_request_target for PR comments. - # Does NOT checkout PR code. Actor-gated to dependabot[bot]. - - dependabot-major-analysis.yml diff --git a/.gitignore b/.gitignore index d672c330..0e8cbce0 100644 --- a/.gitignore +++ b/.gitignore @@ -14,3 +14,10 @@ packages/.claude/ docs/superpowers/ docs/plans/ docs/specs/ + +# fix-drift.ts workflow artifacts (regenerated each run; uploaded via fix-drift.yml) +# The workflow writes these to $RUNNER_TEMP (outside the checkout) so the +# drift-success predicate's `git status` scan never sees them; this glob is +# belt-and-suspenders for local runs and covers the post-fix / pinned variants. +claude-code-output.log +drift-report*.json diff --git a/CHANGELOG.md b/CHANGELOG.md index 034359bb..50c86469 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,50 @@ # @copilotkit/aimock -## [Unreleased] +## [1.37.2] - 2026-07-17 + +### Fixed + +- Prevent `Invalid string length` crash on `GET /__aimock/journal` by capping retained request bodies at 64 KB (#308) + +### Changed + +- Log full error stack on unhandled request errors for prod diagnosis (#308) + +## [1.37.1] - 2026-07-16 + +### Fixed + +- Capped the uncapped in-memory record buffers that could grow a string past V8's ~512 MiB max string length and throw `RangeError: Invalid string length` (the ~1/sec production crash, amplified once the real-key fixture-miss passthrough landed). The AG-UI recorder, the generic recorder, and the stream-collapse path now bound the buffer they `Buffer.concat(chunks).toString()` for fixture construction to a configurable `maxRecordBufferBytes` (default 64 MiB) clamped to a 256 MiB hard ceiling. Once the cap is crossed the recorder frees the buffer, marks the recording truncated, and skips fixture construction — the full upstream response is still relayed to the client byte-for-byte in real time, so capping never truncates what the client receives ("don't journal", not "don't answer") (#307) + +## [1.37.0] - 2026-07-15 + +### Added + +- New `match.toolResultContains` string gate: passes when the request's LAST message is a `role: "tool"` result whose text content contains the substring (same last-message rule as `toolCallId`, and composable with it). Discriminates fixtures whose requests differ only inside the tool-result payload — e.g. a human-in-the-loop suspend tool where approve and cancel resume with the same `tool_call_id` but different result JSON. JSON-expressible, so fixture files can finally split those legs without a programmatic `predicate`. Validated at load time (non-empty string — an empty substring would silently act as a catch-all), included in the duplicate-userMessage dedup key and the catch-all discriminator set, and routed through the `aimock-pytest` match-level kwarg keys (aimock-pytest 0.5.0) (#299) + +### Fixed + +- The duplicate-userMessage shadow check no longer warns for fixtures that share a `userMessage` but are legitimately disambiguated by a matcher the dedup key previously omitted (`systemMessage`, `model`, `toolName`, `toolCallId`, `inputText`, `responseFormat`, `endpoint`). The dedup key now covers every discriminator the router actually matches on and serialises RegExp / array matchers kind-aware so they don't collide; a `predicate`-bearing fixture is keyed by identity so it is never a false duplicate. This only suppresses spurious warnings — the check never dropped or deduped fixtures (#299) +- Bumped the `aimock-pytest` default server pin (`_version.py`) to 1.37.0 so the auto-download path (used when `AIMOCK_CLI_PATH` is unset) runs a server that supports the new `toolResultContains` match kwarg the client forwards (#299) +- Guarded `recordedTimings.interChunkDelaysMs` against non-array values: a fixture whose `interChunkDelaysMs` is malformed (null, missing, or not an array) no longer throws during load and silently drops every fixture in that file — the value is coerced to an empty array (#299) + +## [1.36.1] - 2026-07-14 + +### Fixed + +- `aimock.json` now honours `llm.latency`, `llm.chunkSize`, `llm.replaySpeed` and `llm.logLevel`, which the config loader previously accepted and ignored. Each behaves as its `llmock` CLI flag and `createMockSuite({ llm })` equivalent. `llm.replaySpeed` divides every delay source (recorded timings, streaming profiles, `latency`), so a fixture suite can replay recorded inter-chunk timings faster than they were recorded; a fixture's own `replaySpeed` still takes precedence. A non-positive `llm.replaySpeed` is ignored with a warning, matching the existing fixture-level guard (#294) + +## [1.36.0] - 2026-07-13 + +### Added + +- On a fixture-miss passthrough (record/proxy mode), aimock can inject its own configured upstream provider key instead of forwarding a caller's dummy placeholder key. Every static-key provider aimock proxies is wired end-to-end, each with an independent `AIMOCK_PROVIDER__KEY` env var applied with the provider-correct wire scheme: `Authorization: Bearer` for OpenAI (`_OPENAI_KEY`), OpenRouter (`_OPENROUTER_KEY`), Cohere (`_COHERE_KEY`), Grok/xAI (`_GROK_KEY`), and Ollama (`_OLLAMA_KEY`); `x-api-key` for Anthropic (`_ANTHROPIC_KEY`); `x-goog-api-key` for Gemini (`_GEMINI_KEY`, also used for Gemini Interactions) and Veo (`_VEO_KEY`); `api-key` for Azure OpenAI (`_AZURE_KEY`); `xi-api-key` for ElevenLabs (`_ELEVENLABS_KEY`); and `Authorization: Key` for fal.ai (`_FAL_KEY`). Injection fires only when the caller credential is absent or dummy-prefixed (`sk-aimock-`, overridable via `AIMOCK_DUMMY_KEY_MARKER`); a real caller key is always forwarded unchanged, an empty-string env var is treated as unset, and with no built-in key configured the feature is inert. Signed/exchanged credentials — AWS Bedrock (SigV4) and Vertex AI (OAuth) — are never rewritten (#293) + +## [1.35.1] - 2026-07-06 + +### Fixed + +- Record mode no longer silently drops fixtures when an SDK closes its socket immediately after `data: [DONE]` (e.g. the OpenAI Python SDK); the completed upstream response is now persisted. Genuine mid-stream aborts (client disconnect before `[DONE]`) still abort upstream and write no fixture (#288) ## [1.35.0] - 2026-06-27 diff --git a/README.md b/README.md index 0e3a29b7..4203bd44 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ Run them all on one port with `npx @copilotkit/aimock --config aimock.json`, or - **[Record & Replay](https://aimock.copilotkit.dev/record-replay)** — Proxy real APIs, save as fixtures, replay deterministically forever - **Timing-aware recording and replay** — Recorded fixtures capture per-frame arrival timestamps; replay uses recorded timings for approximate timing reproduction based on recorded TTFT and inter-frame cadence (replay chunk count may differ from recording — TTFT and average pace are preserved, not per-token fidelity) with configurable `--replay-speed` multiplier -- **[Multi-turn Conversations](https://aimock.copilotkit.dev/multi-turn)** — Record and replay multi-turn traces with tool rounds; match distinct turns via `turnIndex`, `hasToolResult`, `toolCallId`, `sequenceIndex`, `systemMessage` (gate on host-supplied agent context), or custom predicates +- **[Multi-turn Conversations](https://aimock.copilotkit.dev/multi-turn)** — Record and replay multi-turn traces with tool rounds; match distinct turns via `turnIndex`, `hasToolResult`, `toolCallId`, `toolResultContains` (gate on the tool-result payload), `sequenceIndex`, `systemMessage` (gate on host-supplied agent context), or custom predicates - **[12 providers across 14 API surfaces](https://aimock.copilotkit.dev/docs)** — OpenAI Chat, OpenAI Responses, OpenAI Realtime (GA + Beta shim), Claude, Gemini (REST + embedContent), Gemini Live, Gemini Interactions, Azure, Bedrock, Vertex AI, Ollama (chat + embeddings), Cohere (chat + embed), ElevenLabs TTS — full streaming support - **Multimedia APIs** — [image generation](https://aimock.copilotkit.dev/images) (DALL-E, Imagen), [image editing](https://aimock.copilotkit.dev/images) (/v1/images/edits), [text-to-speech](https://aimock.copilotkit.dev/speech) (OpenAI + ElevenLabs), [audio transcription](https://aimock.copilotkit.dev/transcription), [audio translation](https://aimock.copilotkit.dev/transcription) (/v1/audio/translations), [video generation](https://aimock.copilotkit.dev/video), [OpenRouter video generation](https://aimock.copilotkit.dev/openrouter-video) (/api/v1/videos with async job lifecycle), [Google Veo video generation](https://aimock.copilotkit.dev/veo-video) (:predictLongRunning + /v1beta/operations async lifecycle), [Grok Imagine video generation](https://aimock.copilotkit.dev/grok-video) (/v1/videos/generations with async job lifecycle), [fal.ai](https://aimock.copilotkit.dev/fal-ai) (image / video / audio with queue lifecycle) - **[MCP](https://aimock.copilotkit.dev/mcp-mock) / [A2A](https://aimock.copilotkit.dev/a2a-mock) / [AG-UI](https://aimock.copilotkit.dev/agui-mock) / [Vector](https://aimock.copilotkit.dev/vector-mock)** — Mock every protocol your AI agents use @@ -127,6 +127,28 @@ Private and link-local addresses (loopback, RFC1918, CGNAT, cloud metadata, ULA, On replay, `turnIndex` is a non-fatal disambiguator, not a hard reject gate: a content-matching fixture is served even when its scripted `turnIndex` differs from the request's assistant-message count. This kills false "no fixture matched" misses for multi-bubble agent runs (multi-step agents emit several assistant bubbles per logical turn). When a served fixture diverges from its scripted `turnIndex`, the match diagnostic carries `turnIndexRelaxed: true` and aimock logs a one-shot warning (at the `warn` log level — silent by default). To restore the legacy strict behavior where a defined `turnIndex` must equal the assistant count exactly, set `AIMOCK_STRICT_TURN_INDEX=1`. The record path is always strict regardless of this flag. +### aimock-owned upstream keys — `AIMOCK_PROVIDER_*_KEY` + +In record or `--proxy-only` mode, aimock forwards the caller's auth header to the real provider unchanged. If your tests can only send a dummy placeholder key (e.g. an SDK that refuses to start without a non-empty API key), aimock can inject its own configured upstream key on a fixture-miss passthrough so the proxied call actually authenticates. Each provider has an independent env var, and the key is applied with the provider-correct wire scheme: + +| Env var | Provider | Injected header | +| -------------------------------- | -------------------------------- | ----------------------------- | +| `AIMOCK_PROVIDER_OPENAI_KEY` | OpenAI | `Authorization: Bearer ` | +| `AIMOCK_PROVIDER_OPENROUTER_KEY` | OpenRouter | `Authorization: Bearer ` | +| `AIMOCK_PROVIDER_COHERE_KEY` | Cohere | `Authorization: Bearer ` | +| `AIMOCK_PROVIDER_GROK_KEY` | Grok (xAI) | `Authorization: Bearer ` | +| `AIMOCK_PROVIDER_OLLAMA_KEY` | Ollama (Cloud / bearer-gated) | `Authorization: Bearer ` | +| `AIMOCK_PROVIDER_ANTHROPIC_KEY` | Anthropic | `x-api-key: ` | +| `AIMOCK_PROVIDER_GEMINI_KEY` | Gemini (and Gemini Interactions) | `x-goog-api-key: ` | +| `AIMOCK_PROVIDER_VEO_KEY` | Veo | `x-goog-api-key: ` | +| `AIMOCK_PROVIDER_AZURE_KEY` | Azure OpenAI | `api-key: ` | +| `AIMOCK_PROVIDER_ELEVENLABS_KEY` | ElevenLabs | `xi-api-key: ` | +| `AIMOCK_PROVIDER_FAL_KEY` | fal.ai | `Authorization: Key ` | + +The Gemini interactions provider mode reuses `AIMOCK_PROVIDER_GEMINI_KEY` (same upstream API as Gemini). An empty-string value is treated as unset. + +This is opt-in and backward-compatible: with no key configured the feature is inert and the caller's header passes through as-is. Injection fires only when the caller sent no credential **or** a dummy credential prefixed with `sk-aimock-` (overridable via `AIMOCK_DUMMY_KEY_MARKER`); a real caller key never starting with that marker is always forwarded verbatim, so the caller overrides aimock. Signed and exchanged credentials — AWS Bedrock (SigV4) and Vertex AI (OAuth) — are never rewritten and always forwarded unchanged. (Azure's static `api-key` is injected; a real Microsoft Entra ID `Authorization: Bearer` token from the caller is never dummy-prefixed, so it too passes through verbatim.) + ## Framework Guides Test your AI agents with aimock — no API keys, no network calls: [LangChain](https://aimock.copilotkit.dev/integrate-langchain) · [CrewAI](https://aimock.copilotkit.dev/integrate-crewai) · [PydanticAI](https://aimock.copilotkit.dev/integrate-pydanticai) · [LlamaIndex](https://aimock.copilotkit.dev/integrate-llamaindex) · [Mastra](https://aimock.copilotkit.dev/integrate-mastra) · [Google ADK](https://aimock.copilotkit.dev/integrate-adk) · [Microsoft Agent Framework](https://aimock.copilotkit.dev/integrate-maf) diff --git a/charts/aimock/Chart.yaml b/charts/aimock/Chart.yaml index 6c63c6eb..d972ea60 100644 --- a/charts/aimock/Chart.yaml +++ b/charts/aimock/Chart.yaml @@ -3,4 +3,4 @@ name: aimock description: Mock infrastructure for AI application testing (OpenAI, Anthropic, Gemini, MCP, A2A, vector) type: application version: 0.1.0 -appVersion: "1.35.0" +appVersion: "1.35.1" diff --git a/docs/aimock-cli/index.html b/docs/aimock-cli/index.html index 2ad382de..51f37421 100644 --- a/docs/aimock-cli/index.html +++ b/docs/aimock-cli/index.html @@ -102,6 +102,7 @@

Config File Format

"fixtures": "./fixtures", "latency": 0, "chunkSize": 20, + "replaySpeed": 1, "logLevel": "info", "validateOnLoad": true, "metrics": true, @@ -135,9 +136,9 @@

Config Fields

object LLMock configuration. Accepts fixtures, latency, - chunkSize, logLevel, validateOnLoad, - metrics, strict, chaos, - streamingProfile, record. + chunkSize, replaySpeed, logLevel, + validateOnLoad, metrics, strict, + chaos, streamingProfile, record. diff --git a/docs/fixtures/index.html b/docs/fixtures/index.html index 67a0b2d9..acbfc9e2 100644 --- a/docs/fixtures/index.html +++ b/docs/fixtures/index.html @@ -97,6 +97,16 @@

Match Fields

requestTransform is set) or regex (pattern match) + + systemMessage + string | RegExp | string[] + + Match on the joined text of the request’s system messages — string + (case-sensitive substring, or exact when requestTransform is set), + regex (pattern match), or an array of strings (all substrings must be present). Gate + a fixture on host-supplied context + + inputText string | RegExp @@ -110,6 +120,16 @@

Match Fields

The onToolResult(id, response) helper is sugar over this field + + toolResultContains + string + + Substring match on the text content of the last message when it is a + role: "tool" result. Discriminates fixtures whose requests differ only + inside the tool-result payload (e.g. approve vs cancel resuming with the same + tool_call_id). See Multi-Turn Conversations + + toolName string @@ -200,12 +220,16 @@

1. userMessage matches only the LAST user message

tool-round idiom.

-

2. toolCallId matches the LAST tool message

+

2. toolCallId matches the overall last message

- toolCallId is compared against the tool_call_id of the - last role: "tool" message in the request — regardless of - whether that’s the overall last message. If no tool message is present in the - history, toolCallId never matches. See + toolCallId matches only when the request’s + overall last message is a role: "tool" result whose + tool_call_id equals the given value. This mirrors the API contract: a tool + result is the last message only while the model is answering that tool round. If the last + message is anything else — a newer user turn, or no tool result at all + — toolCallId never matches, so a stale tool round can’t shadow + userMessage matchers. This is the same overall-last-message rule as + toolResultContains. See Multi-Turn Conversations for the tool-round idiom.

@@ -234,10 +258,10 @@

4. Substring by default, exact when a requestTransform is set If the router is configured with a requestTransform (typically used to strip dynamic data like timestamps or UUIDs from the request before matching), string - userMessage and inputText flip to strict equality - (===). The rationale: transforms normalize requests to a canonical form, and - once normalized, the sensible comparison is exact — substring matching on a - normalized string is more likely to hide bugs than catch flexible input. + userMessage, systemMessage, and inputText flip to + strict equality (===). The rationale: transforms normalize requests to a + canonical form, and once normalized, the sensible comparison is exact — substring + matching on a normalized string is more likely to hide bugs than catch flexible input.

5. Validation warnings surface shadowing at load time

@@ -252,15 +276,19 @@

5. Validation warnings surface shadowing at load time

duplicate userMessage 'hello' — shadows fixture 0, where 'hello' is the duplicated message and 0 is the zero-based index of the earlier fixture being shadowed. This is advisory, not a hard error: the - check now factors in turnIndex, hasToolResult, - context, and sequenceIndex when deciding whether two fixtures - truly collide, but it does not consider toolCallId, - model, or predicate, so the warning may still fire when those - discriminators are present. Treat it as advisory: if a runtime differentiator is in - place, the fixtures won't actually shadow each other at match time. Only fixtures with - no differentiator at all will truly shadow on match — that's the case where the - second is never reached because the first wins. Safe to ignore in the former case; - investigate in the latter. + check factors in every match discriminator the router gates on when deciding + whether two fixtures truly collide — userMessage, + systemMessage, inputText, toolCallId, + toolResultContains, toolName, model, + responseFormat, endpoint, context, + sequenceIndex, turnIndex, and + hasToolResult — so two fixtures that differ on any one of them do not + trigger the warning. A predicate is a function that cannot be compared for + equivalence, so any fixture carrying one is keyed by its own identity: it is treated as + unconditionally distinct and never reported as a duplicate (and two fixtures with + distinct predicates are likewise treated as distinct). Only fixtures with no + differentiator at all will truly shadow on match — that's the case where the + second is never reached because the first wins. That is the case worth investigating.
  • Catch-all not last — a fixture with an empty match diff --git a/docs/multi-turn/index.html b/docs/multi-turn/index.html index 3b56a788..0d98c741 100644 --- a/docs/multi-turn/index.html +++ b/docs/multi-turn/index.html @@ -72,6 +72,17 @@

    How matching works across turns

    distinguish the turn that requests a tool from the turn that follows up on a tool result.
    • +
  • + toolResultContains (v1.37.0) is a substring match against the + text content of the last tool message (same last-message rule as + toolCallId). Use it when two resume paths share the same + tool_call_id and differ only inside the tool-result payload — e.g. a + human-in-the-loop tool where approve resumes with + {"chosen_time": …} and cancel resumes with + {"cancelled": true}. Gate the cancel fixture on + "toolResultContains": "\"cancelled\"" (match the quoted JSON key so it + can’t match the word inside prose) and order it before the approve leg. +

    • A request carrying a 20-message history still only matches on its last user message (and, @@ -100,8 +111,10 @@

    How matching works across turns

    Substring by default, exact when transformed. userMessage is a substring match by default ("hello" matches "say hello world"). When you register a requestTransform, - matching flips to exact string equality — but only for - userMessage and inputText; other fields like + matching flips to exact string equality — but only for the + string forms of userMessage, systemMessage, and + inputText. Regex and array-of-substrings forms are unaffected, and + toolResultContains stays a substring match. Fields like toolName and toolCallId are always exact. This trips people up — see Gotchas below.

    @@ -292,10 +305,10 @@

    Turn 2 — client runs the tool, sends the result

    - Choosing between sequenceIndex, toolCallId, turnIndex, hasToolResult, context, and - predicate + Choosing between sequenceIndex, toolCallId, toolResultContains, turnIndex, hasToolResult, + context, and predicate

    -

    Five mechanisms handle different shapes of “the same prompt twice”:

    +

    Seven mechanisms handle different shapes of “the same prompt twice”:

    @@ -318,8 +331,20 @@

    + + + + + @@ -420,19 +445,25 @@

    Gotchas

    Substring vs. exact matching. Default matching is substring. Adding a requestTransform (e.g. to strip timestamps or request ids) flips matching to exact string equality — fixtures that previously matched as substrings will - silently stop matching. Only userMessage and inputText flip; - fields like toolName and toolCallId are always exact. Pin - exact strings in your fixtures when you use a transform. + silently stop matching. Only the string forms of userMessage, + systemMessage, and inputText flip; regex and + array-of-substrings forms are unaffected, and toolResultContains remains a + substring match even under a transform. Fields like toolName and + toolCallId are always exact. Pin exact strings in your fixtures when you + use a transform.
  • Duplicate userMessage warnings. validateFixtures warns when two fixtures share the same - userMessage with identical turnIndex, - hasToolResult, and sequenceIndex values. Fixtures that differ - on any of these discriminators do not trigger the warning. Other fields like - toolCallId, model, and predicate are not factored - in, so the warning may still fire when those discriminators are present. Treat it as - advisory. + userMessage with identical values for every match discriminator + the router gates on — userMessage, systemMessage, + inputText, toolCallId, toolResultContains, + toolName, model, responseFormat, + endpoint, context, sequenceIndex, + turnIndex, and hasToolResult. Fixtures that differ on any one + of these discriminators do not trigger the warning. A predicate cannot be + compared for equivalence, so any fixture carrying one is keyed by its own identity and + is never reported as a duplicate. Treat it as advisory.
  • First-wins ordering. Fixtures are evaluated in registration order (and, diff --git a/docs/record-replay/index.html b/docs/record-replay/index.html index 062ab46c..d5974b30 100644 --- a/docs/record-replay/index.html +++ b/docs/record-replay/index.html @@ -489,6 +489,114 @@

    Header Forwarding

    recordedTimings block (streaming frame timings) — never the request headers.

    +

    aimock-Owned Upstream Keys

    +

    + By default, aimock forwards the caller's auth header to the upstream provider unchanged. + If your tests can only send a dummy placeholder key — for example, an SDK that refuses to + start without a non-empty API key — aimock can inject its own configured upstream + key on a fixture-miss passthrough so the recorded/proxied call actually authenticates. + This is opt-in and backward-compatible: with no key configured the + feature is fully inert and the caller's header is forwarded as-is. +

    +

    + Set one or more of these env vars to aimock's real upstream keys. Each is independent — + configure only the providers you record against: +

    +
    • Different behavior before vs. after tool execution (tool-call round trip) toolCallId - Matches the tool_call_id of the last role: "tool" - message. Turn 1 has no tool message; turn 2 does. + Matches when the request’s overall last message is a + role: "tool" result with that tool_call_id. Turn 1 has no + tool result last; turn 2 does. +
    + Two tool-result outcomes that share a tool_call_id and differ only + inside the result payload (e.g. approve vs. cancel) + toolResultContains + Substring match on the last role: "tool" result’s text. Use when + toolCallId alone can’t tell the outcomes apart. Stateless.
    + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
    Env varProviderInjected header
    AIMOCK_PROVIDER_OPENAI_KEYOpenAIAuthorization: Bearer <key>
    AIMOCK_PROVIDER_OPENROUTER_KEYOpenRouterAuthorization: Bearer <key>
    AIMOCK_PROVIDER_COHERE_KEYCohereAuthorization: Bearer <key>
    AIMOCK_PROVIDER_GROK_KEYGrok (xAI)Authorization: Bearer <key>
    AIMOCK_PROVIDER_OLLAMA_KEYOllama (Cloud / bearer-gated)Authorization: Bearer <key>
    AIMOCK_PROVIDER_ANTHROPIC_KEYAnthropicx-api-key: <key>
    AIMOCK_PROVIDER_GEMINI_KEYGemini (and Gemini Interactions)x-goog-api-key: <key>
    AIMOCK_PROVIDER_VEO_KEYVeox-goog-api-key: <key>
    AIMOCK_PROVIDER_AZURE_KEYAzure OpenAIapi-key: <key>
    AIMOCK_PROVIDER_ELEVENLABS_KEYElevenLabsxi-api-key: <key>
    AIMOCK_PROVIDER_FAL_KEYfal.aiAuthorization: Key <key>
    +

    + gemini-interactions reuses AIMOCK_PROVIDER_GEMINI_KEY (same + upstream API as Gemini). An empty-string value is treated as unset. +

    +

    Injection fires only when both of these hold:

    +
      +
    • + The request is a fixture-miss passthrough (record mode or + --proxy-only). +
      • +
    • + The caller sent no credential for that provider, + or sent a dummy credential prefixed with + sk-aimock-. A caller credential that does not start with the marker is + treated as a real key and forwarded verbatim — the caller always overrides aimock. +
      • +
    +

    + The dummy marker prefix is overridable via AIMOCK_DUMMY_KEY_MARKER for setups + that mint placeholder keys under a different prefix. Gemini is injected as an + x-goog-api-key header only (no query-param ?key= rewrite). +

    +

    + Signed and exchanged credentials are never rewritten. AWS Bedrock + (SigV4), Vertex AI, and Azure AD carry OAuth/signed auth rather than a simple bearer or + api-key header, so aimock always forwards their credentials unchanged regardless of these + env vars. +

    +

    Strict Mode

    When --strict is enabled, an unmatched request returns @@ -899,8 +1007,8 @@

    Request Transform

    - When requestTransform is set, string matching for - userMessage and inputText switches from substring + When requestTransform is set, string matching for userMessage, + systemMessage, and inputText switches from substring (includes) to exact equality (===). This prevents shortened keys from accidentally matching unrelated prompts. Without a transform, the existing includes behavior is preserved for backward compatibility. diff --git a/package.json b/package.json index a32e00eb..20edeede 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@copilotkit/aimock", - "version": "1.35.0", + "version": "1.37.2", "description": "Mock infrastructure for AI application testing — LLM APIs, image generation, image editing, text-to-speech, transcription, audio translation, audio generation, video generation, embeddings, MCP tools, A2A agents, AG-UI event streams, vector databases, search, rerank, and moderation. One package, one port, zero dependencies.", "license": "MIT", "keywords": [ @@ -169,7 +169,7 @@ "test:exports": "publint && attw --pack .", "lint": "eslint .", "format:check": "prettier --check .", - "release": "pnpm build && pnpm test && pnpm lint && npm publish", + "release": "pnpm format:check && pnpm build && pnpm test && pnpm lint && pnpm test:exports && npm publish", "prepare": "husky || true" }, "lint-staged": { diff --git a/packages/aimock-pytest/README.md b/packages/aimock-pytest/README.md index 7f8d4031..ae099024 100644 --- a/packages/aimock-pytest/README.md +++ b/packages/aimock-pytest/README.md @@ -79,7 +79,7 @@ aimock.reset() # alias for reset_fixtures() ``` --aimock-node PATH Path to node binary ---aimock-version VER aimock npm version (default: 1.35.0) +--aimock-version VER aimock npm version (default: 1.35.1) ``` ## Environment Variables diff --git a/packages/aimock-pytest/pyproject.toml b/packages/aimock-pytest/pyproject.toml index 09e0894f..1cbd98c0 100644 --- a/packages/aimock-pytest/pyproject.toml +++ b/packages/aimock-pytest/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "aimock-pytest" -version = "0.4.0" +version = "0.5.0" description = "pytest fixtures for aimock — mock LLM APIs, multimedia, MCP, A2A, AG-UI, vector DBs, and more" readme = "README.md" requires-python = ">=3.10" @@ -12,7 +12,7 @@ license = "MIT" dependencies = ["requests>=2.28"] [project.optional-dependencies] -test = ["pytest>=7.0", "requests>=2.28"] +test = ["pytest>=7.0"] [project.entry-points."pytest11"] aimock = "aimock_pytest.plugin" diff --git a/packages/aimock-pytest/src/aimock_pytest/_server.py b/packages/aimock-pytest/src/aimock_pytest/_server.py index 946902de..07d98a09 100644 --- a/packages/aimock-pytest/src/aimock_pytest/_server.py +++ b/packages/aimock-pytest/src/aimock_pytest/_server.py @@ -182,6 +182,7 @@ def url(self) -> str: "systemMessage", "inputText", "toolCallId", + "toolResultContains", "toolName", "model", "responseFormat", diff --git a/packages/aimock-pytest/src/aimock_pytest/_version.py b/packages/aimock-pytest/src/aimock_pytest/_version.py index d3751b8e..7130d640 100644 --- a/packages/aimock-pytest/src/aimock_pytest/_version.py +++ b/packages/aimock-pytest/src/aimock_pytest/_version.py @@ -8,4 +8,4 @@ control routes ship in the next release). Keep it tracking npm releases. """ -AIMOCK_VERSION = "1.35.0" +AIMOCK_VERSION = "1.37.2" diff --git a/renovate.json b/renovate.json new file mode 100644 index 00000000..62dc1e5a --- /dev/null +++ b/renovate.json @@ -0,0 +1,4 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": ["local>CopilotKit/renovate"] +} diff --git a/scripts/drift-delta.ts b/scripts/drift-delta.ts new file mode 100644 index 00000000..4c435904 --- /dev/null +++ b/scripts/drift-delta.ts @@ -0,0 +1,146 @@ +/** + * Delta-gating core for the drift pipeline. + * + * A PR-scoped drift run produces a HEAD report; a same-UTC-day cached MAIN run + * produces a BASE report. The gate must only BLOCK on drift that is attributable + * to the diff under review — a drift already present on main (environmental / + * world drift) is the daily job's concern, NOT the PR's. This module reduces the + * two reports to a per-key delta: + * + * - block → keys NEW in head (absent from base): diff-attributable, hard fail + * - advisory → keys present in BOTH base and head: pre-existing / environmental + * - fixed → keys present in base but GONE in head: informational + * + * The block/advisory/fixed decision is driven ENTIRELY by key presence. The + * coarse `DriftClass` (critical/advisory/quarantine) is annotation only and NEVER + * enters the routing decision — a new-in-head *critical* still BLOCKS, and a + * both-present *critical* is still only ADVISORY. This is the #292 invariant: a + * real-drift/critical finding that is new in head must block regardless of class. + * + * Pure and side-effect-free — no filesystem, no network, no process state. + */ + +import type { DriftClass, DriftReport } from "./drift-types.js"; + +/** + * A single failure identified by the delta layer. + * + * A failure is keyed by `provider` + per-item `id` (NOT `path`, which is a + * generic bucket like `"knownModels"` that would collapse N distinct model + * drifts into one key). `class` is carried through purely for annotation and + * never participates in routing. + */ +export interface DeltaKey { + /** Provider the failing entry belongs to (e.g. "anthropic"). */ + provider: string; + /** Stable per-item key (e.g. a model id). Falls back to path when absent. */ + id: string; + /** Coarse classification — annotation only, never routes block vs advisory. */ + class?: DriftClass; +} + +/** The delta between a base (main) report and a head (PR) report. */ +export interface DeltaResult { + /** New-in-head failures — diff-attributable, hard fail. */ + block: DeltaKey[]; + /** Failures present in both base and head — pre-existing / environmental. */ + advisory: DeltaKey[]; + /** Failures present in base but gone in head — informational. */ + fixed: DeltaKey[]; +} + +/** + * Build the canonical string key for a failure. Keys by provider + per-item id. + * When a diff has no `id` (legacy diffs), fall back to its `path` so it still + * participates in the delta rather than being dropped — this is a coarse bucket + * but preserves the block/advisory distinction across reports. + */ +function keyOf(provider: string, id: string): string { + return `${provider}::${id}`; +} + +/** + * Flatten a report into a map of `key → DeltaKey`. When multiple diffs collapse + * to the same key, the last one wins for annotation purposes (the routing + * decision does not depend on which representative is retained). + */ +function indexReport(report: DriftReport): Map { + const index = new Map(); + for (const entry of report.entries) { + for (const diff of entry.diffs) { + // `id` is the stable per-item key; `path` is the legacy fallback bucket. + const id = diff.id ?? diff.path; + const key = keyOf(entry.provider, id); + index.set(key, { provider: entry.provider, id, class: diff.class }); + } + } + return index; +} + +/** + * Compute the delta between a base report and a head report. + * + * Routing is by KEY PRESENCE only: + * - key in head but not base → block (new-in-head, diff-attributable) + * - key in both → advisory (pre-existing / environmental) + * - key in base but not head → fixed (informational) + * + * `DriftClass` NEVER enters this decision — a new-in-head critical blocks, a + * both-present critical is advisory. See the module docstring (#292 invariant). + */ +export function computeDelta(baseReport: DriftReport, headReport: DriftReport): DeltaResult { + const base = indexReport(baseReport); + const head = indexReport(headReport); + + const block: DeltaKey[] = []; + const advisory: DeltaKey[] = []; + const fixed: DeltaKey[] = []; + + for (const [key, headKey] of head) { + if (base.has(key)) { + advisory.push(headKey); + } else { + block.push(headKey); + } + } + + for (const [key, baseKey] of base) { + if (!head.has(key)) { + fixed.push(baseKey); + } + } + + return { block, advisory, fixed }; +} + +/** + * Known-good collector conclusions that make a cached base report trustworthy to + * reuse. Anything else (crash, quarantine, unknown) means the base is not a + * reliable baseline and a fresh live base run should be preferred. + */ +const REUSABLE_CONCLUSIONS: ReadonlySet = new Set(["clean", "success"]); + +/** + * Guard (O-2) for reusing a cached same-UTC-day main report as the delta base. + * + * A base report is reusable only when ALL of: + * 1. it has a non-empty `entries[]` (rejects empty / malformed cached JSON), + * 2. its collector `conclusion` is known-good (rejects crash / quarantine), + * 3. it was produced on the same UTC day (`sameUtcDay`) — a stale baseline + * would misattribute a day's worth of environmental drift to the PR. + * + * Anything else → not reusable → caller should run a fresh live base. + */ +export function isBaseReportReusable( + report: DriftReport | null | undefined, + conclusion: string | null | undefined, + sameUtcDay: boolean, +): boolean { + if (!report || !Array.isArray(report.entries) || report.entries.length === 0) { + return false; + } + if (!conclusion || !REUSABLE_CONCLUSIONS.has(conclusion)) { + return false; + } + return sameUtcDay === true; +} diff --git a/scripts/drift-report-collector.ts b/scripts/drift-report-collector.ts index f46626c4..4e440402 100644 --- a/scripts/drift-report-collector.ts +++ b/scripts/drift-report-collector.ts @@ -10,7 +10,8 @@ * Exit codes: * 0 — no critical diffs found (or no drift at all) * 2 — at least one critical diff exists - * 1 — script error (unhandled exception) + * 5 — at least one failure was quarantined (unparseable/untrusted — needs review) + * 1 — AG-UI drift detection was skipped (infra), or an unhandled script error * * Usage: * npx tsx scripts/drift-report-collector.ts [--out drift-report.json] @@ -19,8 +20,18 @@ import { execSync } from "node:child_process"; import { existsSync, statSync, writeFileSync } from "node:fs"; import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; -import type { DriftEntry, DriftReport, DriftSeverity, ParsedDiff } from "./drift-types.js"; +import { SURFACE_REGISTRY, isKnownSurface } from "../src/__tests__/drift/surface-registry.js"; +import type { SurfaceMapping } from "../src/__tests__/drift/surface-registry.js"; + +import type { + DriftEntry, + DriftReport, + DriftSeverity, + ParsedDiff, + QuarantineEntry, +} from "./drift-types.js"; // --------------------------------------------------------------------------- // Vitest JSON reporter types (subset we care about) @@ -42,112 +53,46 @@ interface VitestAssertion { } // --------------------------------------------------------------------------- -// Provider → file mapping +// Surface → file mapping (single source of truth in surface-registry.ts) // --------------------------------------------------------------------------- -interface ProviderMapping { - builderFile: string; - builderFunctions: string[]; - typesFile: string | null; - sdkShapesFile?: string; -} - -const OPENAI_CHAT_MAPPING: ProviderMapping = { - builderFile: "src/helpers.ts", - builderFunctions: [ - "buildTextCompletion", - "buildToolCallCompletion", - "buildTextChunks", - "buildToolCallChunks", - ], - typesFile: "src/types.ts", -}; - -const OPENAI_RESPONSES_MAPPING: ProviderMapping = { - builderFile: "src/responses.ts", - builderFunctions: [ - "buildTextResponse", - "buildToolCallResponse", - "buildTextStreamEvents", - "buildToolCallStreamEvents", - ], - typesFile: null, -}; - -const ANTHROPIC_MAPPING: ProviderMapping = { - builderFile: "src/messages.ts", - builderFunctions: [ - "buildClaudeTextResponse", - "buildClaudeToolCallResponse", - "buildClaudeTextStreamEvents", - "buildClaudeToolCallStreamEvents", - ], - typesFile: null, -}; +/** + * A resolved surface mapping. Structurally identical to the registry's + * `SurfaceMapping` (kept as a local alias so the rest of this module reads + * unchanged from the pre-registry `ProviderMapping`). + */ +type ProviderMapping = SurfaceMapping; -const GEMINI_MAPPING: ProviderMapping = { - builderFile: "src/gemini.ts", - builderFunctions: [ - "buildGeminiTextResponse", - "buildGeminiToolCallResponse", - "buildGeminiTextStreamChunks", - "buildGeminiToolCallStreamChunks", - ], - typesFile: null, -}; +const SDK_SHAPES_FILE = "src/__tests__/drift/sdk-shapes.ts"; -const OPENAI_EMBEDDINGS_MAPPING: ProviderMapping = { - builderFile: "src/helpers.ts", - builderFunctions: ["buildEmbeddingResponse", "generateDeterministicEmbedding"], - typesFile: null, - sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", +/** + * Legacy label aliases → registry slug. The pre-WS-5 `PROVIDER_MAP` keyed some + * surfaces by SHORTER titles than the registry's canonical `provider` label + * (e.g. a bare `"Gemini"`/`"Anthropic"` describe-block title, or `"Bedrock + * Invoke"` prose without a colon suffix). These map onto the canonical slug so + * the LEGACY no-marker fallback keeps recognizing an unmigrated block whose + * prose title uses the shorter form. Newer blocks resolve structurally via the + * `Surface:` marker and never touch this table. + */ +const LEGACY_LABEL_ALIASES: Record = { + Gemini: "gemini", + Anthropic: "anthropic", }; /** - * Maps provider names (from drift test describe blocks) to source files - * and builder function names. The function names are builder functions for - * each provider (internal or exported) — they are included so Claude Code - * can locate them via Read/Grep. + * Reverse index from a human provider label to its mapping, for the LEGACY + * no-marker fallback (`extractProviderName`). Built from the registry's + * canonical `provider` labels PLUS the legacy aliases above. Newer emitters + * declare a `Surface:` slug marker (resolved structurally); this table only + * serves blocks that predate the marker or a path not yet migrated. */ -const PROVIDER_MAP: Record = { - "OpenAI Chat": OPENAI_CHAT_MAPPING, - "OpenAI Responses": OPENAI_RESPONSES_MAPPING, - Anthropic: ANTHROPIC_MAPPING, - "Anthropic Claude": ANTHROPIC_MAPPING, - "Google Gemini": GEMINI_MAPPING, - Gemini: GEMINI_MAPPING, - "OpenAI Realtime": { - builderFile: "src/ws-realtime.ts", - builderFunctions: ["handleWebSocketRealtime", "realtimeItemsToMessages"], - typesFile: null, - }, - "OpenAI Responses WS": { - builderFile: "src/ws-responses.ts", - builderFunctions: ["handleWebSocketResponses"], - typesFile: null, - }, - "Gemini Live": { - builderFile: "src/ws-gemini-live.ts", - builderFunctions: ["handleWebSocketGeminiLive"], - typesFile: null, - }, - "OpenAI Embeddings": OPENAI_EMBEDDINGS_MAPPING, - "Gemini Interactions": { - builderFile: "src/gemini-interactions.ts", - builderFunctions: [ - "buildInteractionsTextResponse", - "buildInteractionsToolCallResponse", - "buildInteractionsContentWithToolCallsResponse", - "buildInteractionsTextSSEEvents", - "buildInteractionsToolCallSSEEvents", - "buildInteractionsContentWithToolCallsSSEEvents", - ], - typesFile: null, - }, +const PROVIDER_LABEL_MAP: Record = { + ...Object.fromEntries(Object.values(SURFACE_REGISTRY).map((m) => [m.provider, m])), + ...Object.fromEntries( + Object.entries(LEGACY_LABEL_ALIASES).map(([label, slug]) => [label, SURFACE_REGISTRY[slug]]), + ), }; -const SDK_SHAPES_FILE = "src/__tests__/drift/sdk-shapes.ts"; - // --------------------------------------------------------------------------- // AG-UI schema drift constants // --------------------------------------------------------------------------- @@ -178,7 +123,7 @@ const AGUI_DRIFT_TEST = "src/__tests__/drift/agui-schema.drift.ts"; */ const VALID_SEVERITIES = new Set(["critical", "warning", "info"]); -function parseDriftBlock(text: string): { context: string; diffs: ParsedDiff[] } | null { +export function parseDriftBlock(text: string): { context: string; diffs: ParsedDiff[] } | null { const headerMatch = text.match(/API DRIFT DETECTED:\s*(.+)/); if (!headerMatch) return null; @@ -199,13 +144,17 @@ function parseDriftBlock(text: string): { context: string; diffs: ParsedDiff[] } ); continue; } + const path = match[3].trim(); diffs.push({ severity: severity as DriftSeverity, issue: match[2].trim(), - path: match[3].trim(), + path, expected: match[4].trim(), real: match[5].trim(), mock: match[6].trim(), + // Stable per-item key derived from the offending path so different paths + // yield different delta keys (provider+id) in the D6.1 delta layer. + id: path, }); } @@ -218,16 +167,35 @@ function parseDriftBlock(text: string): { context: string; diffs: ParsedDiff[] } } /** - * Extract provider name from the describe block title or the drift report context. + * Extract the machine-readable surface slug from a drift block's `Surface:` + * marker line (emitted by `formatDriftReport(context, diffs, surface)`), if + * present. Returns null when no marker line exists (a legacy/unmigrated block). + * + * The marker sits between the `API DRIFT DETECTED:` header and the numbered + * entries, e.g.: + * + * API DRIFT DETECTED: Cohere /v2/chat (non-streaming) + * Surface: cohere-chat + * 1. [critical] … + */ +export function extractSurfaceKey(text: string): string | null { + const match = text.match(/^\s*Surface:\s*(\S+)\s*$/m); + return match ? match[1] : null; +} + +/** + * LEGACY provider-name fallback for a drift block that carries NO `Surface:` + * marker (predates the slug marker, or a path not yet migrated). Matches the + * text against the registry's human `provider` labels (longest first to avoid + * partial matches). Newer blocks resolve structurally via `extractSurfaceKey`; + * this exists only as a defensive back-compat net. * * Examples: * "OpenAI Chat Completions drift" → "OpenAI Chat" - * "OpenAI Chat (non-streaming text)" → "OpenAI Chat" * "Anthropic Claude drift" → "Anthropic Claude" */ -function extractProviderName(text: string): string | null { - // Try matching against known provider keys (longest first to avoid partial matches) - const sorted = Object.keys(PROVIDER_MAP).sort((a, b) => b.length - a.length); +export function extractProviderName(text: string): string | null { + const sorted = Object.keys(PROVIDER_LABEL_MAP).sort((a, b) => b.length - a.length); for (const key of sorted) { if (text.includes(key)) return key; } @@ -240,11 +208,148 @@ function extractProviderName(text: string): string | null { * "OpenAI Chat (non-streaming text)" → "non-streaming text" * "Anthropic Claude (streaming tool call)" → "streaming tool call" */ -function extractScenario(context: string): string { +export function extractScenario(context: string): string { const parenMatch = context.match(/\(([^)]+)\)/); return parenMatch ? parenMatch[1] : context; } +// --------------------------------------------------------------------------- +// Known-models canary recognizer +// --------------------------------------------------------------------------- + +/** + * Discriminated result of parsing the ws-realtime canary failure. + * + * CLASS 3: `ids` holds ONLY genuine model ids (never a prose sentinel). The + * "some ids were truncated in CI output" fact is a boolean flag, not a fake id, + * so a non-model annotation can never flow into `DriftEntry.real` downstream. + * + * CLASS 2: `noGA` marks the hasGA-false mode — the GA realtime family was + * renamed/removed or the credential could not see any realtime models. `ids` + * then carries the OBSERVED realtime models (possibly empty), not "unknown" + * ones. Either mode is a critical, exit-2 drift signal. + */ +export interface CanaryParseResult { + ids: string[]; + truncated?: boolean; + noGA?: boolean; + /** + * NO_GA mode only: the unknown-model list observed in the SAME run. Because + * the hasGA assertion short-circuits the later unknown-models assertion, the + * NO_GA marker carries both lists (`… | UNKNOWN_REALTIME_MODELS=…`); this + * field holds the unknown segment so the combined case loses no information. + */ + unknownIds?: string[]; +} + +/** + * Parse the ws-realtime canary assertion failure. Two canary modes exist: + * + * 1. UNKNOWN models — `expect(unknown, \`UNKNOWN_REALTIME_MODELS=…\`).toEqual([])` + * shape: `AssertionError: UNKNOWN_REALTIME_MODELS=a,b: expected [ 'a', …(1) ] + * to deeply equal []`. + * 2. NO GA models (CLASS 2) — `expect(hasGA, \`NO_GA_REALTIME_MODELS=…\`).toBe(true)` + * shape: `AssertionError: NO_GA_REALTIME_MODELS=x,y: expected false to be true`. + * + * PRIMARY SOURCE: the `*_REALTIME_MODELS=` marker carries the FULL, non-truncated + * comma-joined list verbatim (vitest truncates the printed array with `…(N)` / + * `... (N)`, so ids beyond the first are unrecoverable from the array alone). + * + * FALLBACK (unknown-mode only, marker missing/mangled): parse the printed array + * and set `truncated` when a CI ellipsis is present. + * + * Returns a CanaryParseResult, or null if the message is not a canary shape. + */ +export function parseKnownModelsCanary(text: string): CanaryParseResult | null { + // CLASS 2 PRIMARY: hasGA-false marker. The GA family is gone/unreachable — a + // critical signal even when the observed list is empty. Recognize it FIRST so + // the "expected false to be true" shape is a structured entry, not a crash. + const noGaMatch = text.match(/NO_GA_REALTIME_MODELS=(.*?)(?::\s*expected\b|\n|$)/); + if (noGaMatch) { + // The NO_GA marker carries BOTH the observed realtime models AND the + // unknown-model list observed in the same run, joined as + // ` | UNKNOWN_REALTIME_MODELS=`. Split the two apart so + // neither list pollutes the other (the combined no-GA + unknown case must + // not lose the unknown list, since the hasGA assertion short-circuits the + // later unknown-models assertion). A legacy message without the unknown + // segment still parses — `split` yields a single element. + const [observedRaw, unknownRaw] = noGaMatch[1].split(/\s*\|\s*UNKNOWN_REALTIME_MODELS=/); + const toList = (s: string | undefined): string[] => + (s ?? "") + .split(",") + .map((v) => v.trim()) + .filter((v) => v.length > 0); + const ids = toList(observedRaw); + const unknownIds = toList(unknownRaw); + return unknownIds.length > 0 ? { ids, noGA: true, unknownIds } : { ids, noGA: true }; + } + + // UNKNOWN-mode PRIMARY: stable marker carrying the full, non-truncated list. + // Scan the whole message (not just line 0) so a leading blank line cannot hide + // it. Capture the ENTIRE value up to the vitest boilerplate (`: expected …`) + // or end-of-line. On a malformed/empty marker we fall THROUGH to the + // printed-array fallback so a recoverable id in the array is still surfaced. + const markerMatch = text.match(/UNKNOWN_REALTIME_MODELS=(.*?)(?::\s*expected\b|\n|$)/); + if (markerMatch) { + const ids = markerMatch[1] + .split(",") + .map((s) => s.trim()) + .filter((s) => s.length > 0); + if (ids.length > 0) return { ids }; + } + + // FALLBACK GATE: the printed-array fallback matches the GENERIC vitest shape + // `expected [ ... ] to deeply equal []`, which ANY `toEqual([])` assertion in + // ANY provider's test emits. On its own that shape is NOT a canary signal — a + // non-canary failure (e.g. a different provider asserting an empty array whose + // observed value happens to be a secret or an object shape) would otherwise be + // misclassified as OpenAI-Realtime known-models drift and have its arbitrary + // array contents relabeled as "unknown model ids". So the fallback fires ONLY + // when the message is clearly the ws-realtime known-models canary. Two + // recognizers (either suffices): + // 1. the realtime-canary marker family (`UNKNOWN_REALTIME_MODELS=` / + // `NO_GA_REALTIME_MODELS=`), even mangled/partial — the marker paths + // above only fall through here when the marker VALUE was empty/unusable, + // but the TOKEN itself still identifies the canary; and + // 2. the canary's origin file in the stack trace — a real canary failure + // ALWAYS carries an `at …/ws-realtime.drift.ts` frame. + // A generic non-canary `toEqual([])` failure has neither, so it returns null + // and falls through to the collector's normal unparseable/fail-loud handling. + const isRealtimeCanaryContext = + /_REALTIME_MODELS=/.test(text) || /ws-realtime\.drift\.ts/.test(text); + if (!isRealtimeCanaryContext) return null; + + // FALLBACK: no (usable) marker but a confirmed canary context. Best-effort + // parse of the printed array on the first line. The canary assertion is always + // on line 1. + const firstLine = text.split("\n")[0]; + + // Shape: ...: expected [ ... ] to deeply equal [] + const canaryMatch = firstLine.match(/expected\s*\[([^\]]*)\]\s*to deeply equal \[\]/); + if (!canaryMatch) return null; + + const inner = canaryMatch[1].trim(); + + // Extract quoted model ids from the bracket list + const ids: string[] = []; + const idPattern = /'([^']+)'/g; + let m: RegExpExecArray | null; + while ((m = idPattern.exec(inner)) !== null) { + ids.push(m[1]); + } + + // CI truncation marker in BOTH forms: single-glyph `…(N)` and three-dot ASCII + // `... (N)`. CLASS 3: this is a BOOLEAN flag — never a synthetic id. + const truncated = /(?:…|\.{3})\s*\(\d+\)/.test(inner); + + // A genuinely-empty inner list with no truncation (`expected [] to deeply + // equal []`) means the canary's `unknown` array was empty — no drift to + // surface. Return null so it is not a spurious entry. + if (ids.length === 0 && !truncated) return null; + + return truncated ? { ids, truncated: true } : { ids }; +} + // --------------------------------------------------------------------------- // Run drift tests and collect results // --------------------------------------------------------------------------- @@ -324,9 +429,205 @@ function runDriftTests(): VitestJsonResult { } } -function collectDriftEntries(results: VitestJsonResult): DriftEntry[] { +/** + * Distinguish genuine infrastructure errors from genuine-but-unparseable drift + * reports. Returns true when the whole batch of unparseable messages should be + * SWALLOWED as benign infra (collector exits 0); false means genuine drift is + * present and the collector must throw. + * + * A false "infra" classification here is dangerous: it makes the collector exit + * 0 and silently drop real drift. + * + * F2: `/AssertionError/i` is DELIBERATELY NOT an infra indicator. Every vitest + * drift failure is an AssertionError, so treating it as benign infra masks real + * drift. Infra failures announce themselves with network/HTTP/parse markers + * below; a bare AssertionError is drift until proven infra. + * + * A3: BOTH the infra-indicator scan and the drift-like scan run against the + * SAME normalized text (stack-trace ` at …` frames stripped). An earlier + * version scanned the RAW message for infra indicators but the stack-stripped + * message for drift indicators; that asymmetry could tip the benign-infra gate + * true and silently drop genuine drift. Normalizing both identically keeps the + * two scans from ever disagreeing on the same input. + */ +// Infra indicators. IMPORTANT (CLASS 1 / uniform anchoring): every infra +// indicator is defined here as a BARE phrase (the source string only, no `^`, +// no flags, no anchoring-defeating `(?:.*:\s*)?` prefix). The line-anchor is +// applied UNIFORMLY by `anchorInfraIndicator` below, so no single indicator can +// be individually mis-anchored and a phrase added to this list later is +// automatically anchored the same way as its siblings. +// +// The anchor requires the phrase to BE the failure reason at the START of a +// line (optional leading whitespace, an optional `HTTP ` prefix for the numeric +// ones), followed by a word boundary. This means a genuine infra REASON like +// `empty response`, `fetch failed`, or `API returned 503` at line start still +// classifies as infra, while a labelled drift-body VALUE like +// ` Real: API returned 503` / ` Real: empty response` can NEVER trip +// the gate (the phrase is not at the line start, it follows a `Field:` label). +// +// Each entry is a bare `source` phrase plus a `boundary` flag: `true` appends a +// trailing `\b` (word-like phrases such as `empty response` / `ECONNREFUSED`), +// `false` omits it for phrases whose next char is punctuation (`INFRA_ERROR:`) +// or a tag (`[\b] + * + * so the phrase must be the failure reason at the start of a line. A labelled + * drift-body value (`Real: `) is preceded by a `Field:` label and thus + * never matches. This is the single choke point that makes anchoring uniform — + * there is no per-indicator anchoring to get wrong. + */ +function anchorInfraIndicator(spec: InfraIndicatorSpec): RegExp { + const boundary = spec.boundary === false ? "" : "\\b"; + return new RegExp(`^\\s*(?:HTTP\\s+)?${spec.source}${boundary}`, "im"); +} + +/** + * The bare infra phrase sources, exported so tests can iterate the REAL list + * (property-based test in drift-collector.test.ts) rather than a hand-copied + * subset. A future indicator added to INFRA_INDICATOR_SPECS is automatically + * covered — if it were somehow mis-anchored the property test would fail. + */ +export const INFRA_INDICATOR_SOURCES: readonly string[] = INFRA_INDICATOR_SPECS.map( + (s) => s.source, +); + +/** + * A concrete failure-reason sample for a given bare infra phrase source, used + * by the property-based test to synthesize both a bare (line-start, infra) and + * a labelled (`Real: …`, drift) line. Regex metacharacters in the source + * (`\d{3}`, `\s*:?`, tag `<`) are replaced with literal, matching sample text. + */ +export function infraIndicatorSample(source: string): string { + return source + .replace(/\\s\*:\?\\s\*/g, ": ") // status\s*:?\s* → "status: " + .replace(/\\s\*/g, " ") + .replace(/\\d\{3\}/g, "503") + .replace(/\\d\+/g, "503") + .replace(/\\b/g, ""); +} + +const INFRA_INDICATORS: RegExp[] = INFRA_INDICATOR_SPECS.map(anchorInfraIndicator); + +// Drift-like indicators. A genuine drift failure is drift until proven infra. +// The `expected … to be/equal/deeply equal …` shape is the canonical vitest +// assertion body for our drift/canary assertions — the OLD `/expected.*but/i` +// guard was DEAD (vitest never emits "expected … but …"), so it never fired and +// left bare assertion failures indistinguishable from infra. This repaired +// guard fires on the shapes vitest actually emits, so an unrecognized assertion +// failure counts as drift-like and forces a fail-loud (throw → exit 1). +const DRIFT_LIKE_INDICATORS = [ + /drift/i, + /mismatch/i, + /\bexpected\b.*\bto\s+(?:be|equal|deeply equal|contain|match|have)\b/i, + /LLMOCK DRIFT/i, + /API DRIFT/i, +]; + +/** Strip vitest stack-trace frames (` at …`) so filenames like + * "ws-realtime.drift.ts" cannot influence content-based classification. */ +function stripStackFrames(msg: string): string { + return msg + .split("\n") + .filter((line) => !/^\s*at\s/.test(line)) + .join("\n"); +} + +/** + * O-1: capture the raw `file:line` (or `file:line:col`) from the FIRST usable + * stack frame BEFORE `stripStackFrames` removes it, so a quarantined failure + * carries a pointer the human reviewer can jump to. Prefers a project-source + * frame (`src/…`) over node_modules/internal frames; falls back to the first + * frame with any `path:line` shape. Returns "" when no frame is present. + */ +export function extractRawLocation(msg: string): string { + const frames = msg.split("\n").filter((line) => /^\s*at\s/.test(line)); + // Match `path:line` or `path:line:col`, with an optional trailing `)`. + const locRe = /((?:\/|\.\/|[A-Za-z]:\\|file:\/\/)?[^\s()]+?:\d+(?::\d+)?)\)?\s*$/; + const pick = (predicate: (f: string) => boolean): string | null => { + for (const frame of frames) { + if (!predicate(frame)) continue; + const m = frame.match(locRe); + if (m) return m[1]; + } + return null; + }; + // Prefer a project-source frame, skipping node internals / node_modules. + return ( + pick((f) => /src\//.test(f) && !/node_modules/.test(f) && !/node:internal/.test(f)) ?? + pick((f) => !/node_modules/.test(f) && !/node:internal/.test(f)) ?? + pick(() => true) ?? + "" + ); +} + +export function classifyUnparseableAsInfra(unparseableMessages: string[]): boolean { + // CLASS 1 — fail-loud on absent evidence. No messages means NO positive infra + // evidence, so this is NOT a benign "all clear". `[].every(...)` is vacuously + // true, which would have wrongly classified an empty batch as infra and made + // the collector exit 0. Root invariant: unrecognized ⇒ fail loud, never a + // false all-clear. + if (unparseableMessages.length === 0) return false; + + // Normalize ONCE per message; both scans consume the identical normalized + // text (A3 — the two scans must never disagree due to differing inputs). + const normalized = unparseableMessages.map(stripStackFrames); + + // Infra requires POSITIVE evidence on EVERY message AND zero drift signal on + // ALL of them. If any message lacks an infra indicator, or any message looks + // drift-like, we do NOT swallow — the caller throws (exit 1, investigate). + const allInfraErrors = normalized.every((msg) => INFRA_INDICATORS.some((re) => re.test(msg))); + const anyDriftLike = normalized.some((msg) => DRIFT_LIKE_INDICATORS.some((re) => re.test(msg))); + + return allInfraErrors && !anyDriftLike; +} + +/** + * The result of collecting drift entries. In addition to the trustworthy drift + * `entries`, `quarantine` holds failures that could not be parsed/mapped into a + * trustworthy finding AND were not positively classified as benign infra (A1.3). + * These no longer crash the collector (exit 1) nor get silently dropped (exit + * 0): the caller routes a non-empty `quarantine` to exit 5. Exit 1 is now + * reserved for genuine collector bugs (unhandled exceptions). + */ +export interface CollectResult { + entries: DriftEntry[]; + quarantine: QuarantineEntry[]; +} + +export function collectDriftEntries(results: VitestJsonResult): CollectResult { const entries: DriftEntry[] = []; - const unmapped: string[] = []; + const quarantine: QuarantineEntry[] = []; let unparseable = 0; for (const file of results.testResults) { @@ -337,22 +638,169 @@ function collectDriftEntries(results: VitestJsonResult): DriftEntry[] { const fullMessage = assertion.failureMessages.join("\n"); const parsed = parseDriftBlock(fullMessage); if (!parsed || parsed.diffs.length === 0) { + // Check for the ws-realtime canary assertion shapes BEFORE classifying + // as unparseable — both the unknown-models mode and the hasGA-false mode + // (CLASS 2) are genuine, critical drift and must be surfaced (exit 2), + // never crashed as unparseable (exit 1). + const canary = parseKnownModelsCanary(fullMessage); + if (canary !== null) { + const mapping = SURFACE_REGISTRY["openai-realtime"]; + + // Build the critical diffs. F4: severity MUST be "critical" — the Fix + // Drift workflow (.github/workflows/fix-drift.yml) gates remediation + // entirely on exit code 2, which main() emits only when + // criticalCount > 0. F5: this canary is real-API-only — there is no + // mock leg, so we never mislabel a model id as a mock value. + const NO_MOCK_LEG = ""; + const diffs: ParsedDiff[] = canary.noGA + ? [ + // CLASS 2: the GA realtime family is renamed/removed or the + // credential cannot see it. Surface the observed models (may be + // empty) so the auto-fix prompt knows what IS present. + { + severity: "critical" as const, + issue: + "GA realtime family unavailable — no known GA model in the OpenAI realtime list. " + + "OpenAI may have renamed/removed the GA family, or the realtime credential cannot see it. " + + "Update the gaModels list in ws-realtime.drift.ts.", + path: "gaModels", + expected: "(at least one GA realtime model present)", + real: + canary.ids.length > 0 + ? `observed realtime models: ${canary.ids.join(", ")}` + : "no realtime models observed", + mock: NO_MOCK_LEG, + }, + // The hasGA assertion short-circuits the later unknown-models + // assertion, so surface any unknown models observed in the SAME + // run here (carried by the NO_GA marker). Without this, a run + // that is BOTH no-GA AND has new unknown models would lose the + // unknown list from the auto-fix prompt. One diff per id — each + // is a genuine model id (never a prose sentinel). + ...(canary.unknownIds ?? []).map((id) => ({ + severity: "critical" as const, + issue: + "Unknown realtime model detected (observed in the same run as the missing GA " + + "family) — add to knownModels in ws-realtime.drift.ts", + path: "knownModels", + expected: "(not in knownModels set)", + real: id, + mock: NO_MOCK_LEG, + // D6.2: set `id` to the model id so the delta layer (D6.1) + // can key by provider+id, yielding one distinct key per model + // rather than collapsing all entries under path:"knownModels". + id, + })), + ] + : // CLASS 3: only genuine model ids become `real` diffs. The + // truncation fact is carried as a SEPARATE diff whose `real` is a + // count note, never a fake model id in a per-model slot. + [ + ...canary.ids.map((id) => ({ + severity: "critical" as const, + issue: + "Unknown realtime model detected — add to knownModels in ws-realtime.drift.ts", + path: "knownModels", + expected: "(not in knownModels set)", + real: id, + mock: NO_MOCK_LEG, + // D6.2: set `id` to the model id so the delta layer (D6.1) + // can key by provider+id, yielding one distinct key per model + // rather than collapsing all entries under path:"knownModels". + id, + })), + ...(canary.truncated + ? [ + { + severity: "critical" as const, + issue: + "Additional unknown realtime models were truncated in CI output — " + + "the full list is unrecoverable without the UNKNOWN_REALTIME_MODELS= marker. " + + "Re-run with the marker to enumerate them.", + path: "knownModels[truncated]", + // CLASS 3: `real`/`expected` NEVER carry a prose sentinel. + // The truncation fact lives entirely in `issue`/`path`; + // there is no observed model value to report here. + expected: "", + real: "", + mock: NO_MOCK_LEG, + }, + ] + : []), + ]; + + entries.push({ + provider: "OpenAI Realtime", + scenario: "known-models canary", + builderFile: mapping.builderFile, + builderFunctions: mapping.builderFunctions, + typesFile: mapping.typesFile ?? null, + sdkShapesFile: SDK_SHAPES_FILE, + diffs, + }); + continue; + } unparseable++; continue; } - // Determine provider from ancestor titles (describe block) or context const ancestorText = assertion.ancestorTitles.join(" "); - const provider = extractProviderName(ancestorText) ?? extractProviderName(parsed.context); - if (!provider) { - unmapped.push(`${ancestorText} > ${assertion.title}`); - continue; - } - - const mapping = PROVIDER_MAP[provider]; - if (!mapping) { - unmapped.push(`${ancestorText} > ${assertion.title} (provider: ${provider})`); - continue; + const testName = `${ancestorText} > ${assertion.title}`; + + // Resolution order (see WS-5 spec §3c): + // + // 1. `Surface:` marker present → structural lookup in SURFACE_REGISTRY. + // - hit → auto-fixable DriftEntry (exit-2 lane). THIS IS THE FIX. + // - miss → a NEW surface whose author forgot the registry entry. + // Do NOT silently quarantine — THROW (collector-fault, exit + // 1) so it is loud, distinct and actionable. fix-drift.yml + // treats a non-{2,5} collector exit as a "collector crashed" + // alert, so this cannot be ignored. + // 2. No marker (legacy/unmigrated block) → fall back to the human + // provider-label match. Hit → entry; miss → quarantine (exit 5), + // exactly as before this change. + const surfaceKey = extractSurfaceKey(fullMessage) ?? extractSurfaceKey(parsed.context); + let provider: string; + let mapping: ProviderMapping; + if (surfaceKey !== null) { + // `surfaceKey` is untrusted text (extractSurfaceKey → `\S+`). Guard with + // own-property check BEFORE indexing: a plain-object bracket lookup walks + // the prototype chain, so a slug like `constructor`/`toString`/`__proto__` + // would otherwise resolve to a truthy inherited member and skip the throw, + // emitting a garbage entry (`builderFile: undefined`). isKnownSurface uses + // Object.prototype.hasOwnProperty.call — the same guard the emit side uses. + if (!isKnownSurface(surfaceKey)) { + throw new Error( + `Unknown drift surface "${surfaceKey}" — add it to SURFACE_REGISTRY ` + + `in src/__tests__/drift/surface-registry.ts (test: ${testName})`, + ); + } + const registered = SURFACE_REGISTRY[surfaceKey]; + provider = registered.provider; + mapping = registered; + } else { + // Legacy no-marker fallback: match the human provider label. + const label = extractProviderName(ancestorText) ?? extractProviderName(parsed.context); + // Own-property guard for parity with the marker-path lookup above: even + // though extractProviderName only returns real PROVIDER_LABEL_MAP keys + // today, indexing without hasOwn would resolve prototype members + // (`constructor`, etc.) to a truthy value if such a label were ever added. + const legacyMapping = + label && Object.hasOwn(PROVIDER_LABEL_MAP, label) ? PROVIDER_LABEL_MAP[label] : undefined; + if (!label || !legacyMapping) { + // Parseable drift block we cannot route to a source file. Held for + // review (exit 5) rather than crashing the whole run. O-1: capture the + // raw frame location BEFORE any stack stripping. + quarantine.push({ + provider: label || parsed.context || ancestorText || "unknown", + testName, + rawLocation: extractRawLocation(fullMessage), + message: fullMessage, + }); + continue; + } + provider = label; + mapping = legacyMapping; } entries.push({ @@ -361,81 +809,70 @@ function collectDriftEntries(results: VitestJsonResult): DriftEntry[] { builderFile: mapping.builderFile, builderFunctions: mapping.builderFunctions, typesFile: mapping.typesFile, - sdkShapesFile: SDK_SHAPES_FILE, + sdkShapesFile: mapping.sdkShapesFile ?? SDK_SHAPES_FILE, diffs: parsed.diffs, }); } } - if (unmapped.length > 0) { - console.error(`ERROR: ${unmapped.length} drift failure(s) could not be mapped to a provider:`); - for (const u of unmapped) console.error(` - ${u}`); - throw new Error(`${unmapped.length} unmapped drift entries — update PROVIDER_MAP`); + if (quarantine.length > 0) { + console.warn( + `WARNING: ${quarantine.length} drift failure(s) could not be mapped to a provider — ` + + `quarantined for review (exit 5), not crashed:`, + ); + for (const q of quarantine) + console.warn(` - ${q.testName} @ ${q.rawLocation || ""}`); } if (unparseable > 0 && entries.length === 0) { - // Collect the unparseable failure messages to classify them - const unparseableMessages: string[] = []; + // Collect the unparseable failure messages (with their raw pre-strip + // locations) to classify them. O-1: capture file:line BEFORE stripping. + const unparseableFailures: { message: string; testName: string; rawLocation: string }[] = []; for (const file of results.testResults) { for (const assertion of file.assertionResults) { if (assertion.status !== "failed" || assertion.failureMessages.length === 0) continue; const fullMessage = assertion.failureMessages.join("\n"); const parsed = parseDriftBlock(fullMessage); if (!parsed || parsed.diffs.length === 0) { - unparseableMessages.push(fullMessage); + // Canary shapes are handled above (they became entries) — only truly + // unparseable messages reach here. + if (parseKnownModelsCanary(fullMessage) !== null) continue; + unparseableFailures.push({ + message: fullMessage, + testName: `${assertion.ancestorTitles.join(" ")} > ${assertion.title}`, + rawLocation: extractRawLocation(fullMessage), + }); } } } + const unparseableMessages = unparseableFailures.map((f) => f.message); for (const msg of unparseableMessages) { console.warn(` Unparseable failure message (first 300 chars): ${msg.slice(0, 300)}`); } - // Distinguish infrastructure errors from broken drift report formats - const infraIndicators = [ - /^INFRA_ERROR:/m, - /API returned \d{3}/i, - /status \d{3}/i, - / - infraIndicators.some((re) => re.test(msg)), - ); - const anyDriftLike = unparseableMessages.some((msg) => - driftLikeIndicators.some((re) => re.test(msg)), - ); - - if (allInfraErrors && !anyDriftLike) { + if (classifyUnparseableAsInfra(unparseableMessages)) { console.warn( `WARNING: ${unparseable} test failure(s) appear to be API/infrastructure errors ` + `(not drift reports). Continuing with 0 drift entries.`, ); } else { - console.error( - `ERROR: ${unparseable} test failure(s) could not be parsed as drift reports.`, - "This may indicate broken test infrastructure or a changed report format.", - ); - throw new Error( - `${unparseable} unparseable test failures with 0 drift entries — investigate`, + // A1.3: genuine-but-unparseable drift is no longer a fail-loud crash (exit + // 1). Each such failure is quarantined (exit 5) so it surfaces for human + // review without being silently swallowed as a green. Exit 1 is now + // reserved for genuine collector bugs (unhandled exceptions). + console.warn( + `WARNING: ${unparseable} test failure(s) could not be parsed as drift reports — ` + + `quarantined for review (exit 5).`, ); + for (const f of unparseableFailures) { + quarantine.push({ + provider: "unknown", + testName: f.testName, + rawLocation: f.rawLocation, + message: f.message, + }); + } } } else if (unparseable > 0) { console.warn( @@ -443,7 +880,7 @@ function collectDriftEntries(results: VitestJsonResult): DriftEntry[] { ); } - return entries; + return { entries, quarantine }; } // --------------------------------------------------------------------------- @@ -629,6 +1066,58 @@ function collectAgUiDriftEntries(results: VitestJsonResult): DriftEntry[] { return entries; } +// --------------------------------------------------------------------------- +// Exit-code policy +// --------------------------------------------------------------------------- + +/** + * Map the three terminal signals a drift run can produce onto the collector's + * process exit code. Pure and side-effect-free so the mapping is unit-testable + * without spawning the drift suite. + * + * Precedence (highest first): + * - `criticalCount > 0` → 2 — at least one trustworthy critical drift. + * - `quarantineCount > 0`→ 5 — a failure could not be parsed/mapped into a + * trustworthy finding and was held for review; + * distinct from a critical (2) and from a clean + * pass (0) so it is never silently swallowed. + * - `agUiSkipped` → 1 — AG-UI drift detection could not run (infra). + * O2: AG-UI-skipped stays exit 1 (unchanged). + * - otherwise → 0 — no drift. + * + * Critical wins over quarantine: if the run found a genuine critical drift, that + * is the actionable signal even if some other failure was also quarantined. + */ +export function computeExitCode( + criticalCount: number, + quarantineCount: number, + agUiSkipped: boolean, +): 0 | 1 | 2 | 5 { + if (criticalCount > 0) return 2; + if (quarantineCount > 0) return 5; + if (agUiSkipped) return 1; + return 0; +} + +/** + * Map a collector exit code to the coarse `conclusion` written into the drift + * report, so the base-report reuse guard (`isBaseReportReusable`) can read + * `report.conclusion` directly. Only exit 0 ("clean") is a reusable baseline; + * "critical"/"quarantine" (and the exit-1 "skipped" case) are not. + */ +export function conclusionForExitCode(exitCode: 0 | 1 | 2 | 5): string { + switch (exitCode) { + case 0: + return "clean"; + case 2: + return "critical"; + case 5: + return "quarantine"; + default: + return "skipped"; + } +} + // --------------------------------------------------------------------------- // Main // --------------------------------------------------------------------------- @@ -644,7 +1133,8 @@ function main(): void { console.log("Running HTTP API drift tests..."); const httpResults = runDriftTests(); console.log("Collecting HTTP API drift entries..."); - const httpEntries = collectDriftEntries(httpResults); + const httpResult = collectDriftEntries(httpResults); + const httpEntries = httpResult.entries; // Collect AG-UI schema drift entries console.log("Running AG-UI schema drift tests..."); @@ -659,10 +1149,27 @@ function main(): void { } const entries = [...httpEntries, ...agUiEntries]; + const quarantine = httpResult.quarantine; + + const criticalCount = entries.reduce( + (sum, e) => sum + e.diffs.filter((d) => d.severity === "critical").length, + 0, + ); + const quarantineCount = quarantine.length; + + // Compute the exit code BEFORE writing so the report can carry the coarse + // `conclusion` derived from it (base-report reuse contract). + const exitCode = computeExitCode(criticalCount, quarantineCount, agUiSkipped); + const timestamp = new Date().toISOString(); const report: DriftReport = { - timestamp: new Date().toISOString(), + timestamp, + // Alias of `timestamp` read by the reuse guard; `timestamp` kept for + // back-compat with existing consumers. + generatedAt: timestamp, + conclusion: conclusionForExitCode(exitCode), entries, + ...(quarantine.length > 0 ? { quarantine } : {}), }; try { @@ -680,29 +1187,49 @@ function main(): void { console.log(` AG-UI schema entries: ${agUiEntries.length}`); } console.log(` Total entries: ${entries.length}`); - - const criticalCount = entries.reduce( - (sum, e) => sum + e.diffs.filter((d) => d.severity === "critical").length, - 0, - ); console.log(` Critical diffs: ${criticalCount}`); - - if (criticalCount > 0) { - console.log("Exiting with code 2 (critical diffs found)."); - process.exit(2); + console.log(` Quarantined failures: ${quarantineCount}`); + + switch (exitCode) { + case 2: + console.log("Exiting with code 2 (critical diffs found)."); + process.exit(2); + // eslint-disable-next-line no-fallthrough + case 5: + console.warn(`Exiting with code 5 (${quarantineCount} failure(s) quarantined for review).`); + process.exit(5); + // eslint-disable-next-line no-fallthrough + case 1: + console.warn("Exiting with code 1 (AG-UI drift detection was skipped — infra failure)."); + process.exit(1); + // eslint-disable-next-line no-fallthrough + default: + console.log("No critical diffs found."); } +} - if (agUiSkipped) { - console.warn("Exiting with code 1 (AG-UI drift detection was skipped — infra failure)."); - process.exit(1); +/** + * Entry-point guard: only run main() when this module is executed directly + * (e.g. `npx tsx scripts/drift-report-collector.ts` from the Fix Drift + * workflow), NOT when it is imported (e.g. by the vitest suite that exercises + * the exported pure functions). Without this guard, importing the module for + * tests would spawn the whole drift suite via execSync and call process.exit. + */ +function isDirectRun(): boolean { + const entry = process.argv[1]; + if (!entry) return false; + try { + return fileURLToPath(import.meta.url) === resolve(entry); + } catch { + return false; } - - console.log("No critical diffs found."); } -try { - main(); -} catch (err: unknown) { - console.error("Fatal error:", err); - process.exit(1); +if (isDirectRun()) { + try { + main(); + } catch (err: unknown) { + console.error("Fatal error:", err); + process.exit(1); + } } diff --git a/scripts/drift-retry.ts b/scripts/drift-retry.ts index 6ba8e3a8..945bad5d 100644 --- a/scripts/drift-retry.ts +++ b/scripts/drift-retry.ts @@ -50,6 +50,7 @@ import { fileURLToPath } from "node:url"; // Collector exit-code contract (see drift-report-collector.ts header). export const EXIT_CLEAN = 0; export const EXIT_CRITICAL_DRIFT = 2; +export const EXIT_QUARANTINE = 5; // Defaults: keep the fleet of real-API calls small. 3 total attempts with a // ~45s backoff mirrors the observed transient window (the Fix Drift workflow @@ -78,7 +79,7 @@ export interface RetryOptions { } export interface RetryResult { - /** Final exit code to propagate (0 = clean/transient, 2 = persistent, other = crash). */ + /** Final exit code to propagate (0 = clean/transient, 2 = persistent, 5 = quarantine, other = crash). */ exitCode: number; /** True when at least one critical run was seen but a later run cleared it. */ transient: boolean; @@ -86,6 +87,8 @@ export interface RetryResult { criticalRuns: number; /** Per-attempt record, in order. */ attempts: RetryAttempt[]; + /** True when the collector exited with EXIT_QUARANTINE (5): unparseable output quarantined. */ + quarantine?: boolean; } /** @@ -125,6 +128,19 @@ export function retryUntilStable(opts: RetryOptions): RetryResult { continue; } + if (exitCode === EXIT_QUARANTINE) { + // Quarantine is a distinct terminal outcome: the collector encountered + // output it could not parse/classify. No retry — propagate immediately. + opts.log(`Collector exited ${exitCode} (quarantine) — propagating without retry.`); + return { + exitCode: EXIT_QUARANTINE, + transient: false, + criticalRuns, + attempts, + quarantine: true, + }; + } + // Any other code = collector crash / infra error. Do not retry — surface it. opts.log(`Collector exited ${exitCode} (not drift) — propagating without retry.`); return { exitCode, transient: false, criticalRuns, attempts }; diff --git a/scripts/drift-slack-summary.ts b/scripts/drift-slack-summary.ts index 6fb5412a..6b705477 100644 --- a/scripts/drift-slack-summary.ts +++ b/scripts/drift-slack-summary.ts @@ -29,6 +29,54 @@ import { fileURLToPath } from "node:url"; import { readDriftReport } from "./fix-drift.js"; import type { DriftEntry, DriftReport, DriftSeverity } from "./drift-types.js"; +// --------------------------------------------------------------------------- +// Headline classification +// --------------------------------------------------------------------------- + +/** + * Closed headline-class enum for a drift run. + * + * Derivation priority (highest wins): + * stale-key — InfraError status 401 or 403 + * infra-transient — InfraError status 429 or ≥500 + * quarantine — report.quarantine[] is non-empty (exit 5) + * real-drift — at least one entry has a critical diff (exit 2) + * test-infra-false-positive — everything else (exit 0 / advisory only) + */ +export type HeadlineClass = + | "real-drift" + | "test-infra-false-positive" + | "infra-transient" + | "stale-key" + | "quarantine"; + +/** Optional context the caller may supply to sharpen classification. */ +export interface SummarizeOptions { + /** HTTP status from an InfraError that aborted the run, if any. */ + infraErrorStatus?: number; + /** Process exit code produced by the collector/retry wrapper, if known. */ + exitCode?: number; +} + +function computeHeadlineClass(report: DriftReport, opts: SummarizeOptions): HeadlineClass { + const { infraErrorStatus } = opts; + + if (infraErrorStatus === 401 || infraErrorStatus === 403) return "stale-key"; + if (infraErrorStatus === 429 || (infraErrorStatus !== undefined && infraErrorStatus >= 500)) + return "infra-transient"; + + // Critical drift wins over quarantine — a confirmed critical finding is the + // actionable signal even when some failures were also quarantined (mirrors the + // computeExitCode contract: crit→2 wins over quarantine→5). + const hasCritical = report.entries.some((e) => e.diffs.some((d) => d.severity === "critical")); + if (hasCritical) return "real-drift"; + + const hasQuarantine = (report.quarantine?.length ?? 0) > 0; + if (hasQuarantine) return "quarantine"; + + return "test-infra-false-positive"; +} + // Keep the message scannable: cap how many example paths we list per provider // and how many total providers we enumerate before collapsing to a count. const MAX_PATHS_PER_PROVIDER = 3; @@ -39,12 +87,19 @@ const SEVERITY_ORDER: DriftSeverity[] = ["critical", "warning", "info"]; interface ProviderSummary { provider: string; counts: Record; + /** Ordered list of changed path strings (de-duplicated). */ paths: string[]; + /** Ordered list of per-diff ids (de-duplicated), when present. */ + ids: string[]; + /** The builderFile from the first matching entry, for the file reference. */ + builderFile: string; + /** The issue text from the first diff, for one-line context. */ + firstIssue: string; } /** * Group drift entries by provider, tallying severities and collecting a small, - * de-duplicated set of representative changed paths. + * de-duplicated set of representative changed paths and ids. */ function summarizeByProvider(entries: DriftEntry[]): ProviderSummary[] { const byProvider = new Map(); @@ -56,14 +111,23 @@ function summarizeByProvider(entries: DriftEntry[]): ProviderSummary[] { provider: entry.provider, counts: { critical: 0, warning: 0, info: 0 }, paths: [], + ids: [], + builderFile: entry.builderFile, + firstIssue: "", }; byProvider.set(entry.provider, summary); } for (const diff of entry.diffs) { summary.counts[diff.severity]++; + if (diff.id && !summary.ids.includes(diff.id)) { + summary.ids.push(diff.id); + } if (diff.path && !summary.paths.includes(diff.path)) { summary.paths.push(diff.path); } + if (!summary.firstIssue && diff.issue) { + summary.firstIssue = diff.issue; + } } } @@ -75,6 +139,18 @@ function summarizeByProvider(entries: DriftEntry[]): ProviderSummary[] { }); } +/** + * For a provider summary, return the ordered list of identifiers to use as + * per-item references in the Slack bullet. + * + * When any diff carries a stable `id` (e.g. a model id), prefer those over + * raw paths — they're more human-readable in a Slack alert. Fall back to paths + * when no ids are present. + */ +function buildIdOrPaths(s: ProviderSummary): string[] { + return s.ids.length > 0 ? s.ids : s.paths; +} + /** Format a severity tally like "2 critical, 1 warning" (omitting zeroes). */ function formatCounts(counts: Record): string { const parts: string[] = []; @@ -87,32 +163,90 @@ function formatCounts(counts: Record): string { /** * Build the Slack mrkdwn summary lines for the drifted providers. * - * Returns a single string with real `\n` line breaks. Each provider gets a - * bullet line: `• *Provider* — 2 critical: \`path.a\`, \`path.b\``. - * Returns an empty string when there are no entries (caller decides fallback). + * Returns a single string with real `\n` line breaks. The first line is a + * headline classification (`*Classification:* `), followed by per-item + * bullets in one of two forms depending on the class: + * + * Real-drift/advisory: + * `• *Provider* — 2 critical: \`path.a\`, \`path.b\` (src/helpers.ts)` + * + * Quarantine: + * `• [quarantine] Provider — testName (file:line): truncated message` + * + * Returns an empty string only when no entries, no quarantine, no InfraError + * context, and no exitCode hint are present (i.e. caller has nothing to say). + * + * The `drift_summary` GITHUB_OUTPUT key is the full return value of this + * function — callers write it verbatim to `GITHUB_OUTPUT` via writeGithubOutput. + * Existing consumers that read `steps.summary.outputs.drift_summary` receive + * the augmented text; they pattern-match on its content (not on internal + * structure), so the addition of the classification line is backward-compatible. */ -export function summarizeDriftReport(report: DriftReport): string { +export function summarizeDriftReport(report: DriftReport, opts: SummarizeOptions = {}): string { + const headlineClass = computeHeadlineClass(report, opts); + const summaries = summarizeByProvider(report.entries); - if (summaries.length === 0) return ""; + const quarantine = report.quarantine ?? []; + + // Nothing to say at all — preserve historical empty-string behaviour for a + // clean run with no context supplied. + if ( + summaries.length === 0 && + quarantine.length === 0 && + headlineClass === "test-infra-false-positive" && + opts.infraErrorStatus === undefined && + opts.exitCode === undefined + ) { + return ""; + } - const shown = summaries.slice(0, MAX_PROVIDERS_LISTED); const lines: string[] = []; - for (const s of shown) { - const examplePaths = s.paths.slice(0, MAX_PATHS_PER_PROVIDER).map((p) => `\`${p}\``); - const extraPaths = s.paths.length - examplePaths.length; - let pathStr = examplePaths.join(", "); - if (extraPaths > 0) pathStr += `, +${extraPaths} more`; + // ── Headline classification ──────────────────────────────────────────────── + lines.push(`*Classification:* ${headlineClass}`); - const counts = formatCounts(s.counts); - lines.push( - pathStr ? `• *${s.provider}* — ${counts}: ${pathStr}` : `• *${s.provider}* — ${counts}`, - ); + // ── Per-entry drift bullets (real-drift / advisory) ─────────────────────── + if (summaries.length > 0) { + const shown = summaries.slice(0, MAX_PROVIDERS_LISTED); + + for (const s of shown) { + // Per-item: prefer id over path when present on the first diff that has one. + const idOrPaths = buildIdOrPaths(s); + const exampleRefs = idOrPaths.slice(0, MAX_PATHS_PER_PROVIDER).map((r) => `\`${r}\``); + const extraRefs = idOrPaths.length - exampleRefs.length; + let refStr = exampleRefs.join(", "); + if (extraRefs > 0) refStr += `, +${extraRefs} more`; + + // Per-item: file reference from the entry's builderFile. + const fileRef = s.builderFile ? ` (${s.builderFile})` : ""; + + // Per-item: first one-line issue text. + const issueStr = s.firstIssue ? ` — _${s.firstIssue}_` : ""; + + const counts = formatCounts(s.counts); + lines.push( + refStr + ? `• *${s.provider}* — ${counts}: ${refStr}${issueStr}${fileRef}` + : `• *${s.provider}* — ${counts}${issueStr}${fileRef}`, + ); + } + + const hiddenProviders = summaries.length - shown.length; + if (hiddenProviders > 0) { + lines.push(`• …and ${hiddenProviders} more provider${hiddenProviders === 1 ? "" : "s"}`); + } } - const hiddenProviders = summaries.length - shown.length; - if (hiddenProviders > 0) { - lines.push(`• …and ${hiddenProviders} more provider${hiddenProviders === 1 ? "" : "s"}`); + // ── Quarantine entries ───────────────────────────────────────────────────── + if (quarantine.length > 0) { + lines.push( + `*Quarantined* (${quarantine.length} failure${quarantine.length === 1 ? "" : "s"} need human review):`, + ); + for (const q of quarantine) { + const loc = q.rawLocation ? ` (${q.rawLocation})` : ""; + const msg = q.message.slice(0, 120); + lines.push(`• [quarantine] *${q.provider}* — ${q.testName}${loc}: ${msg}`); + } } return lines.join("\n"); diff --git a/scripts/drift-success-predicate.ts b/scripts/drift-success-predicate.ts new file mode 100644 index 00000000..02a30672 --- /dev/null +++ b/scripts/drift-success-predicate.ts @@ -0,0 +1,916 @@ +/// + +/** + * Drift-Success Predicate (WS-2) + * + * A pure predicate — plus a thin CLI wrapper — that decides whether an + * auto-fix run ACTUALLY resolved API drift, versus merely GAMING the drift + * detector by relaxing one of the comparison legs (the SDK-shape fixture, the + * triangulation schema/allowlist, the real-API harness, or a `*.drift.ts` + * assertion). + * + * ALLOWLIST MODEL (round-2 CR — replaces the earlier denylist). A denylist of + * "known gameable legs" leaks: any editable collector input NOT on the list + * (package.json/lockfile pinning a vendored SDK, a tsconfig, an imported + * sub-fixture, an unknown path, a drift-dir `*.test.ts`) could accompany a token + * on-target production edit and reach `resolved:true`. So the predicate INVERTS + * to an allowlist: a fix is RESOLVED only when EVERY changed file is on the + * allowlist, which is (a) PRODUCTION SOURCE — `src/**` that is NOT under + * `src/__tests__/` and is not a config/manifest — PLUS (b) a fixture file + * EXPLICITLY NAMED for a drift entry in the report (`entry.builderFile` / + * `entry.typesFile`, e.g. a canary model-registry.ts the collector sanctioned). + * ANYTHING else blocks: any other `src/__tests__/**` file (legs, `*.drift.ts`, + * `*.test.ts`, providers/ws-providers/schema/sdk-shapes), package.json / + * lockfiles / manifests / config, and any unrecognized path. All paths are + * CANONICALIZED (strip `./`, collapse `//`, resolve `.`/`..`, reject repo-root + * escapes) before classification so a spelling variant cannot sneak a leg past + * the matcher. + * + * The hole this closes: the drift tests are three-way triangulations + * (SDK vs real API vs mock). The SDK leg is literally the repo fixture + * `src/__tests__/drift/sdk-shapes.ts`. Deleting a field from that fixture makes + * the SDK leg null for that path, so the "critical" branch cannot fire and the + * collector reports clean (exit 0) WITHOUT any change to the mock builder. + * Re-running the collector alone therefore cannot detect the cheat — its own + * SDK leg reads the relaxed fixture. The old guard in fix-drift.ts + * (`builderFiles.length === 0 && testFiles.length === 0`) ACCEPTS such a run + * because `testFiles` is non-empty. + * + * The predicate requires THREE independent signals for `resolved:true`: + * 1. AUTHORITATIVE — the post-fix collector re-run is clean (exit 0 AND + * criticalCount 0). + * 2. PRODUCTION CHANGE — at least one PRODUCTION mock-builder file changed + * (`src/**` excluding `src/__tests__/`). A relaxation NEVER changes one. + * 3. NO GAMEABLE-LEG EDIT AT ALL — the changed set touches NO gameable leg. + * HARDENED (fix #1): a gameable-leg edit ALWAYS blocks, INDEPENDENT of how + * many production files also changed. A legitimate auto-remediation updates + * the mock BUILDER to match the SDK; it never edits a comparison/SDK/harness + * leg (those change only on a deliberate human vendored-SDK bump). This + * closes the WS-2b hybrid cheat (relax a leg + one trivial on-target + * production edit) that the old "only check legs when productionFiles===0" + * logic passed straight to auto-merge. Schema/allowlist and `*.drift.ts` + * assertion edits always map to SUPPRESSION_SUSPECTED (actively silencing + * the detector); other gameable legs (sdk-shapes, harness incl the + * dual-classified voice-models.ts) map to SUPPRESSION_SUSPECTED when paired + * with a production change and COMPARISON_LEG_ONLY when standalone. + * + * Additionally, the production change should intersect the report's SANCTIONED + * target set (`union(entry.builderFile, entry.typesFile≠null)`); an off-target + * production change is a WARNING that still blocks (a shared helper MAY be the + * real fix, so it is distinct from an outright cheat). An EMPTY sanctioned set is + * fail-closed (fix #3): with no named target we cannot verify the change landed + * where the drift lives, so it routes to human rather than rubber-stamps. + * + * CLI exit codes (mirrors drift-report-collector.ts's distinct-code discipline): + * 0 — RESOLVED + * 10 — NO_PRODUCTION_CHANGE + * 11 — COMPARISON_LEG_ONLY (leg edit with NO production change) + * 12 — SUPPRESSION_SUSPECTED (schema/*.drift.ts edit, OR any gameable + * leg edited ALONGSIDE a production change) + * 13 — STILL_DIRTY (post-fix collector exit 2, or exit 0 with + * criticalCount>0) + * 14 — QUARANTINE_AFTER_FIX (post-fix collector exit 5 — checked BEFORE + * criticalCount, so exit 5 wins) + * 15 — COLLECTOR_INFRA (post-fix collector exit 1 — likewise wins + * over criticalCount) + * 16 — PRODUCTION_CHANGE_OFF_TARGET (off-target OR zero sanctioned targets; + * WARNING, still blocks) + * 17 — UNSANCTIONED_CHANGE (a changed file is NOT on the allowlist — + * package.json/lockfile/config/unknown path/ + * non-drift or drift `*.test.ts`; a real fix + * touches only production source + report- + * named fixture targets) + * 18 — VERSION_BUMP_FAILED (NOT scored by the predicate — surfaced by + * fix-drift.ts's createPr when the mandatory + * version bump / CHANGELOG step fails, so an + * unversioned fix that never publishes is + * never opened as a PR; fail-closed to human) + * 2 — CONFIG_ERROR (missing/unreadable report, bad args, a + * --changed-file list that disagrees w/ git, + * a path escaping the repo root, or a + * malformed post-fix report that cannot be + * scored) + * + * Usage: + * npx tsx scripts/drift-success-predicate.ts \ + * --report drift-report.json \ + * --post-fix-report drift-report.post-fix.json \ + * --post-fix-exit \ + * [--changed-file src/helpers.ts ...] + * + * When no --changed-file args are supplied the CLI derives the changed set from + * `git status --porcelain` (mirrors fix-drift.ts:getChangedFiles()). When a + * --changed-file list IS supplied it is cross-checked against git and rejected + * (CONFIG_ERROR) on any mismatch (fix #4 — a leg-omitting list must not blind + * the predicate). + */ + +import { execSync } from "node:child_process"; +import { existsSync, readFileSync } from "node:fs"; +import { resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +import type { DriftReport } from "./drift-types.js"; + +// --------------------------------------------------------------------------- +// Public types +// --------------------------------------------------------------------------- + +export enum PredicateReason { + RESOLVED = "resolved", + NO_PRODUCTION_CHANGE = "no-production-change", + COMPARISON_LEG_ONLY = "comparison-leg-only", + SUPPRESSION_SUSPECTED = "suppression-suspected", + UNSANCTIONED_CHANGE = "unsanctioned-change", + STILL_DIRTY = "still-dirty", + QUARANTINE_AFTER_FIX = "quarantine-after-fix", + COLLECTOR_INFRA = "collector-infra", + PRODUCTION_CHANGE_OFF_TARGET = "production-change-off-target", + CONFIG_ERROR = "config-error", + /** + * The version bump / CHANGELOG step failed while opening a drift-fix PR. A + * release ALWAYS accompanies an auto-remediation; without it the PR would + * merge a fix that never publishes (silent value loss). This is a hard, + * fail-closed reason surfaced by fix-drift.ts's createPr — never produced by + * the predicate's own scoring — routed to human review like any other + * needs-human reason. + */ + VERSION_BUMP_FAILED = "version-bump-failed", + /** + * The `--post-fix-*` arguments were present but could not be parsed/read + * while opening a drift-fix PR (e.g. an empty/non-integer `--post-fix-exit` + * from a skipped recollect, or an unreadable post-fix report). fix-drift.ts + * already fails CLOSED on this (no PR), but the throw historically reached the + * top-level catch with a BLANK `reason=`; this names the cause so the Slack + * alert is not blank. Surfaced by fix-drift.ts's main(), never the predicate. + */ + POST_FIX_PARSE_ERROR = "post-fix-parse-error", + /** + * A git operation (checkout / add / commit / push) failed while opening a + * drift-fix PR. This fails CLOSED (no PR is opened — the push never completed, + * so no partial/unversioned PR ships) but historically alerted with a BLANK + * `reason=`; this names the cause. Surfaced by fix-drift.ts's createPr, never + * the predicate. + */ + GIT_PUSH_FAILED = "git-push-failed", +} + +/** Stable exit code for each reason (see module header). */ +export const REASON_EXIT_CODE: Record = { + [PredicateReason.RESOLVED]: 0, + [PredicateReason.NO_PRODUCTION_CHANGE]: 10, + [PredicateReason.COMPARISON_LEG_ONLY]: 11, + [PredicateReason.SUPPRESSION_SUSPECTED]: 12, + [PredicateReason.UNSANCTIONED_CHANGE]: 17, + [PredicateReason.STILL_DIRTY]: 13, + [PredicateReason.QUARANTINE_AFTER_FIX]: 14, + [PredicateReason.COLLECTOR_INFRA]: 15, + [PredicateReason.PRODUCTION_CHANGE_OFF_TARGET]: 16, + [PredicateReason.CONFIG_ERROR]: 2, + [PredicateReason.VERSION_BUMP_FAILED]: 18, + [PredicateReason.POST_FIX_PARSE_ERROR]: 19, + [PredicateReason.GIT_PUSH_FAILED]: 20, +}; + +export interface PredicateInputs { + /** Changed-file paths from getChangedFiles() (git porcelain). */ + changedFiles: string[]; + /** The ORIGINAL pre-fix drift report (source of sanctioned fix targets). */ + report: DriftReport; + /** Exit code of the re-run collector (0 clean / 2 dirty / 5 quarantine / 1 infra). */ + postFixCollectorExit: number; + /** + * criticalCount parsed from the re-run report (belt-and-suspenders vs exit 0). + * + * FIX #7 — INDEPENDENCE CAVEAT: this signal (and the collector exit code) is + * derived from the SAME fixtures the fixer was told to make pass, so it is + * NOT independent of a fixture-relaxation cheat — a run that relaxed the SDK + * leg reports clean here too. It is therefore only trustworthy BECAUSE the + * ALLOWLIST gate now requires EVERY changed file to be production source or a + * report-named fixture target (see isAllowlisted / the UNSANCTIONED_CHANGE + + * SUPPRESSION_SUSPECTED + COMPARISON_LEG_ONLY branches). Any leg/fixture/config + * edit that could relax the collector's own inputs is NOT allowlisted and + * blocks, regardless of production files. That default-deny rule is + * load-bearing: it is what makes a clean post-fix signal mean "the mock was + * really fixed" rather than "an input the collector reads was relaxed". + * + * FIX #4 (assess) — the criticalCount and the post-fix exit code are both + * scored from the AUTHORITATIVE in-workflow re-collect output + * (drift-report.post-fix.json, written by the "Re-collect drift" step), NOT a + * repo-committed file the fixer could forge: neither the pre-fix report nor the + * post-fix report is committed to the repo (see .github/workflows/fix-drift.yml + * — both are produced by collector invocations in-workflow). The re-collect + * runs AFTER autofix and OVERWRITES anything written to that path during the + * fix, so the predicate scores a freshly-generated report, not attacker + * content. + */ + postFixCriticalCount: number; +} + +export interface PredicateResult { + resolved: boolean; + reason: PredicateReason; + /** Human-readable one-liner for Slack / PR body. */ + detail: string; + /** The subset of changedFiles that triggered a block (for LOUD alerts). */ + offendingFiles: string[]; +} + +/** Thrown for malformed CLI args / unreadable inputs / bad paths — maps to exit 2. */ +export class PredicateConfigError extends Error {} + +// --------------------------------------------------------------------------- +// File classification (see spec §2) +// --------------------------------------------------------------------------- + +/** + * Canonicalize a git-reported path to a stable repo-relative POSIX form BEFORE + * classification, so an equivalent-but-non-identical spelling of a leg cannot + * sneak past the exact-string matchers (round-2 CR F1 / slot-1 / slot-2): + * - strip a leading `./` + * - collapse doubled slashes (`//` → `/`) + * - resolve `.` segments and interior `..` segments + * - FAIL CLOSED (PredicateConfigError → exit 2) on any path that escapes the + * repo root (a leading `..` after resolution) or is absolute — such a path + * was never a legitimate in-repo change and must not be silently reclassified. + */ +export function canonicalizePath(file: string): string { + if (file.startsWith("/")) { + throw new PredicateConfigError(`Refusing to classify an absolute path: ${file}`); + } + const rawSegments = file.split("/"); + const out: string[] = []; + for (const seg of rawSegments) { + if (seg === "" || seg === ".") continue; // drop empty (//, leading ./) and `.` + if (seg === "..") { + if (out.length === 0) { + throw new PredicateConfigError(`Path escapes the repo root (fail-closed): ${file}`); + } + out.pop(); + continue; + } + out.push(seg); + } + return out.join("/"); +} + +/** + * The triangulation SCHEMA file. Editing it (esp. its ALLOWLISTED_PATHS set) + * silences diffs globally — a human-reviewed artifact, never a valid fix. + */ +const SCHEMA_FILE = "src/__tests__/drift/schema.ts"; + +/** + * The SDK-shape fixture — the SDK leg of the three-way compare and the primary + * cheat surface (relaxing it makes the critical branch unreachable). + */ +const SDK_SHAPES_FILE = "src/__tests__/drift/sdk-shapes.ts"; + +/** + * The real-API call harness files. Weakening these could elicit a smaller real + * shape (shrinking the real leg), making a diff disappear without a mock change. + */ +const HARNESS_FILES: ReadonlySet = new Set([ + "src/__tests__/drift/providers.ts", + "src/__tests__/drift/ws-providers.ts", + "src/__tests__/drift/helpers.ts", + "src/__tests__/drift/voice-models.ts", +]); + +/** + * LEGITIMATE-FIXTURE-THAT-IS-THE-FIX-TARGET: drift fixtures under + * `src/__tests__/drift/` that ARE the correct fix target for certain drifts + * (the known-models canary routes fixes to these model-list files). These are + * NOT gameable comparison legs — adding a newly-shipped model id here is a + * legit fix, not a relaxation. They are allowed as accompanying changes and are + * never counted as a comparison-leg cheat. + * + * FIX #2 — `voice-models.ts` is deliberately NOT listed here: it is also a + * real-API HARNESS file, and block-classification wins over legit-accept + * (fail-closed precedence). It is classified as a gameable leg in isGameableLeg. + */ +const LEGIT_FIXTURE_TARGETS: ReadonlySet = new Set([ + "src/__tests__/drift/model-registry.ts", + "src/__tests__/drift/model-family.ts", +]); + +/** + * True when `file` is a PRODUCTION mock-builder source file: under `src/` but + * NOT under `src/__tests__/`. This matches fix-drift.ts's existing + * `builderFiles` predicate exactly. + */ +export function isProductionFile(file: string): boolean { + return file.startsWith("src/") && !file.startsWith("src/__tests__/"); +} + +/** + * True when `file` is a `*.drift.ts` test file whose assertions could be + * loosened to make a diff disappear (e.g. `expect(...).toEqual([])`). + */ +export function isDriftTestFile(file: string): boolean { + return file.startsWith("src/__tests__/drift/") && file.endsWith(".drift.ts"); +} + +/** + * GAMEABLE-LEG (the block set). Editing ANY of these can erase a drift diff + * WITHOUT changing the mock output — either by relaxing a comparison leg (the + * SDK-shape fixture, the schema/allowlist, the real-API harness) or by loosening + * a `*.drift.ts` assertion. Every one of these is a HUMAN-REVIEWED artifact: a + * legitimate auto-remediation updates the mock BUILDER to match the SDK, and the + * leg only changes on a deliberate human vendored-SDK bump. So any leg edit — + * ALONE OR ACCOMPANIED BY A PRODUCTION CHANGE — is fail-closed to needs-human + * (SUPPRESSION_SUSPECTED). This closes the WS-2b hybrid cheat (relax a leg + + * one trivial on-target production edit), which the old "only check legs when + * productionFiles===0" logic let through to auto-merge. + * + * FIX #2 — CLASSIFICATION PRECEDENCE: a file that is BOTH a harness leg AND a + * legit fixture target (e.g. `voice-models.ts`) is treated as gameable (block). + * Block-classification wins over legit-accept; fail-closed. Only PURE legit + * fixture targets (model-registry.ts / model-family.ts), which are NOT harness + * legs, are excluded — a canary model-list fix routes to those plus its + * production builder and is not blocked. + */ +export function isGameableLeg(file: string): boolean { + // Block-set membership is checked FIRST (fix #2 precedence): a file that is + // BOTH a harness leg AND a legit fixture target (voice-models.ts) matches + // HARNESS_FILES here and blocks, before the legit-target set is ever consulted. + if (file === SDK_SHAPES_FILE) return true; + if (file === SCHEMA_FILE) return true; + if (HARNESS_FILES.has(file)) return true; + if (isDriftTestFile(file)) return true; + // PURE legit fixture targets (not also a harness leg) are explicitly + // non-gameable — a canary model-list fix routes here plus its production + // builder and must not be blocked. + if (LEGIT_FIXTURE_TARGETS.has(file)) return false; + // Everything else (production files, unrelated paths) is non-gameable. + return false; +} + +/** + * SUPPRESSION surface (the NARROW always-SUPPRESSION_SUSPECTED subset): the + * triangulation schema/allowlist and `*.drift.ts` assertion files. Editing these + * ACTIVELY SILENCES the detector (allowlist growth, loosened assertion) and is + * never a valid fix — so it ALWAYS maps to SUPPRESSION_SUSPECTED, standalone or + * alongside a production change. (The broader gameable-leg set below — sdk-shapes, + * harness — also always blocks per fix #1, but maps to COMPARISON_LEG_ONLY when + * standalone and SUPPRESSION_SUSPECTED only when paired with a production change.) + */ +export function isSuppressionSurface(file: string): boolean { + return file === SCHEMA_FILE || isDriftTestFile(file); +} + +/** + * GAMEABLE-COMPARISON-LEG (retained for API compatibility / classification + * unit coverage). Same full set as isGameableLeg — every leg edit blocks. + */ +export function isComparisonLeg(file: string): boolean { + return isGameableLeg(file); +} + +/** + * Derive the SANCTIONED fix-target set from the pre-fix report: + * `union(entry.builderFile, entry.typesFile≠null)`. These are the files the + * collector itself named as the correct place to fix each drift. + */ +export function sanctionedTargets(report: DriftReport): Set { + const targets = new Set(); + for (const entry of report.entries) { + if (entry.builderFile) targets.add(canonicalizePath(entry.builderFile)); + if (entry.typesFile) targets.add(canonicalizePath(entry.typesFile)); + } + return targets; +} + +/** + * ALLOWLIST membership (round-2 CR F1/F2/F3 — the inversion). A changed file is + * SANCTIONED for an auto-fix run when it is either: + * (a) PRODUCTION SOURCE — `src/**` that is NOT under `src/__tests__/` (a mock + * builder / type file, the legitimate fix target); OR + * (b) a fixture file EXPLICITLY NAMED for a drift entry in the report + * (`entry.builderFile` / `entry.typesFile`, e.g. a canary model-registry.ts + * the collector itself sanctioned as the fix target for this run). + * + * EVERYTHING else is NOT allowlisted and blocks: any other `src/__tests__/**` + * file (comparison legs, `*.drift.ts`, `*.test.ts`, providers/schema/sdk-shapes), + * `package.json` / lockfiles / manifests / config, and any unrecognized path. + * `file` MUST already be canonicalized; `sanctioned` is the canonicalized + * sanctioned-target set (see sanctionedTargets). + */ +export function isAllowlisted(file: string, sanctioned: ReadonlySet): boolean { + if (isProductionFile(file)) return true; + if (sanctioned.has(file)) return true; + return false; +} + +// --------------------------------------------------------------------------- +// The predicate (see spec §3) +// --------------------------------------------------------------------------- + +export function evaluateDriftResolved(i: PredicateInputs): PredicateResult { + const { report, postFixCollectorExit, postFixCriticalCount } = i; + // Canonicalize every changed path BEFORE classification so a spelling variant + // (`./src/...`, `src//...`, `.`/`..` segments) of a leg cannot slip past the + // exact-string matchers (round-2 CR F1). An absolute path or a repo-root escape + // throws PredicateConfigError from canonicalizePath → CONFIG_ERROR at the CLI. + const changedFiles = i.changedFiles.map(canonicalizePath); + + // ---- Signal 1: AUTHORITATIVE — collector clean on re-run. ------------- + // Checked FIRST: a dirty/quarantine/infra collector makes any file-set moot. + // + // FIX #6 — the collector-STATE classification (exit 2/5/1) is checked BEFORE + // the belt-and-suspenders criticalCount>0 branch. A quarantine (exit 5) or an + // infra failure (exit 1) that ALSO happens to carry a parseable criticalCount>0 + // is a quarantine/infra event, NOT a plain STILL_DIRTY — it gets its own + // distinct reason so the Slack alert names the real cause. STILL_DIRTY is + // reserved for exit 2, or a clean-looking exit 0 that nonetheless reports + // criticalCount>0 (report/exit disagreement). + if (postFixCollectorExit === 5) { + return { + resolved: false, + reason: PredicateReason.QUARANTINE_AFTER_FIX, + detail: + "Post-fix drift collector returned quarantine (exit 5) — unparseable/untrusted output after the fix. Needs human review.", + offendingFiles: [], + }; + } + if (postFixCollectorExit === 1) { + return { + resolved: false, + reason: PredicateReason.COLLECTOR_INFRA, + detail: + "Post-fix drift collector returned infra failure (exit 1) — AG-UI skipped or the collector crashed. Cannot trust a clean signal.", + offendingFiles: [], + }; + } + if (postFixCollectorExit === 2 || postFixCriticalCount > 0) { + return { + resolved: false, + reason: PredicateReason.STILL_DIRTY, + detail: + "Post-fix drift collector still reports critical drift " + + `(exit ${postFixCollectorExit}, criticalCount ${postFixCriticalCount}). The fix did not resolve the drift.`, + offendingFiles: [], + }; + } + if (postFixCollectorExit !== 0) { + // Any other non-zero exit is an unrecognized collector state — fail closed. + return { + resolved: false, + reason: PredicateReason.COLLECTOR_INFRA, + detail: `Post-fix drift collector returned an unexpected exit code (${postFixCollectorExit}). Failing closed.`, + offendingFiles: [], + }; + } + + // ---- Classify the changed-file set. ----------------------------------- + const targets = sanctionedTargets(report); + const productionFiles = changedFiles.filter(isProductionFile); + const gameableLegFiles = changedFiles.filter(isGameableLeg); + const suppressionFiles = changedFiles.filter(isSuppressionSurface); + + // ---- Signal 3a (suppression surface): ALWAYS block, standalone or paired. - + // Editing the schema/allowlist or a *.drift.ts assertion actively SILENCES the + // detector — never a valid fix, even alongside a production change. Distinct + // reason so the workflow's Slack alert names "silenced the detector". + if (suppressionFiles.length > 0) { + return { + resolved: false, + reason: PredicateReason.SUPPRESSION_SUSPECTED, + detail: + "Fix edited the triangulation schema/allowlist or a *.drift.ts assertion " + + `(${suppressionFiles.join(", ")}) — silencing the drift detector is never a valid fix. Needs human review.`, + offendingFiles: suppressionFiles, + }; + } + + // ---- Signal 3b (gameable leg): ALWAYS block, independent of production. - + // FIX #1 (HEADLINE) + FIX #2 — a legitimate auto-remediation updates the mock + // BUILDER to match the SDK; it NEVER edits a comparison/SDK/harness leg (those + // change only on a deliberate human vendored-SDK bump). So the presence of ANY + // gameable-leg file blocks REGARDLESS of how many production files also + // changed. This closes the WS-2b hybrid cheat (relax sdk-shapes.ts + one + // trivial on-target production edit) that the old "only check legs when + // productionFiles===0" logic passed straight to auto-merge. voice-models.ts is + // dual-classified (harness + legit target) and blocks here (fix #2 precedence). + // + // Reason split (both distinct, both NEEDS-HUMAN in the workflow): + // • leg edit WITH a production change → SUPPRESSION_SUSPECTED — the WS-2b + // hybrid, the dangerous auto-merge vector. + // • leg edit with NO production change → COMPARISON_LEG_ONLY — a pure + // relaxation, no mock fix even attempted. + if (gameableLegFiles.length > 0) { + if (productionFiles.length > 0) { + return { + resolved: false, + reason: PredicateReason.SUPPRESSION_SUSPECTED, + detail: + "Fix edited a gameable comparison/SDK/harness leg " + + `(${gameableLegFiles.join(", ")}) ALONGSIDE a production change — relaxing a leg is never a valid ` + + "drift fix (a real fix updates the mock builder, not the leg). The WS-2b hybrid cheat. Needs human review.", + offendingFiles: gameableLegFiles, + }; + } + return { + resolved: false, + reason: PredicateReason.COMPARISON_LEG_ONLY, + detail: + "Fix changed ONLY comparison/SDK/harness-leg files " + + `(${gameableLegFiles.join(", ")}) with no production mock-builder change — ` + + "this relaxes the drift detector instead of fixing the mock. The exact cheat this gate blocks.", + offendingFiles: gameableLegFiles, + }; + } + + // ---- ALLOWLIST GATE (round-2 CR F1/F2/F3 — the inversion): EVERY changed file + // must be on the allowlist. The suppression + gameable-leg branches above give + // the KNOWN gameable legs their own LOUD, specific reasons; this gate catches + // EVERYTHING ELSE that is not sanctioned — package.json / lockfiles / manifests + // / config, a drift-dir `*.test.ts`, a non-drift `__tests__` file, an imported + // sub-fixture the report did not name, or any unrecognized path. A denylist of + // "known legs" leaks these; an allowlist does not (default-deny). This is what + // closes the in-diff vectors (F3) and the drift-dir `.test.ts` gap. + const unsanctionedFiles = changedFiles.filter((f) => !isAllowlisted(f, targets)); + if (unsanctionedFiles.length > 0) { + return { + resolved: false, + reason: PredicateReason.UNSANCTIONED_CHANGE, + detail: + "Fix changed files that are NOT on the sanctioned allowlist " + + `(${unsanctionedFiles.join(", ")}) — a real drift fix touches only production mock-builder ` + + "source and the fixture targets the report named. Any other change (deps/config/manifests, " + + "unrelated tests, unnamed fixtures) could game the collector. Fail-closed, needs human review.", + offendingFiles: unsanctionedFiles, + }; + } + + // ---- Signal 2: at least one PRODUCTION mock-builder change is present. -- + // The module invariant (Signal 2 in the header) is that a genuine drift fix + // ALWAYS changes at least one production mock-builder file (`src/**` excluding + // `src/__tests__/`) — that is the only place the mock output is produced. A run + // that changed ZERO production files cannot be a real fix, even if it edited a + // report-named fixture target (the canary model-registry.ts case): a + // fixture-target-only change (e.g. adding a model id to the model-list fixture + // with NO production/builder change) is NOT independently verifiable — the + // re-collect's clean signal is derived from the same fixture the change touched, + // so a clean post-fix report there means only "the fixture agrees with itself", + // not "the mock was fixed". Fix #F2 (round-4): require >=1 production change for + // RESOLVED, unconditionally. A fixture-target-only diff is routed to + // needs-human (NO_PRODUCTION_CHANGE) rather than auto-resolved — matching the + // docstring invariant, which the earlier `onTargetFiles`-satisfies-it logic + // violated (it let a canary fixture-only edit reach resolved:true). + const onTargetFiles = changedFiles.filter((f) => targets.has(f)); + if (productionFiles.length === 0) { + return { + resolved: false, + reason: PredicateReason.NO_PRODUCTION_CHANGE, + detail: + "Fix changed zero production mock-builder files — a real drift fix always updates the " + + "production mock builder (src/** outside src/__tests__/). A fixture-target-only change " + + "(e.g. a model-list fixture edit with no builder change) is not independently verifiable " + + "(the re-collect reads the same fixture) and is routed to human review, not auto-resolved.", + offendingFiles: [], + }; + } + + // ---- Signal 3 (on-target): the change must land on a file the report named as + // a fix target. A production change to an UNNAMED file (a shared helper) MAY be + // a legitimate fix, so it WARNS and blocks (distinct from a cheat). + // + // FIX #3 — an EMPTY sanctioned-target set is fail-closed, not a free pass. An + // empty target set means the report could not name where to fix the drift — + // route to human rather than rubber-stamp an unverifiable change. + if (targets.size === 0) { + return { + resolved: false, + reason: PredicateReason.PRODUCTION_CHANGE_OFF_TARGET, + detail: + "Drift report named ZERO sanctioned fix targets (no builderFile/typesFile) — cannot verify the " + + `change (${productionFiles.join(", ")}) landed where the drift lives. Fail-closed, needs human review.`, + offendingFiles: productionFiles, + }; + } + if (onTargetFiles.length === 0) { + return { + resolved: false, + reason: PredicateReason.PRODUCTION_CHANGE_OFF_TARGET, + detail: + "Change did not touch any file the drift report named as a fix target " + + `(changed: ${productionFiles.join(", ")}; sanctioned: ${[...targets].join(", ")}). ` + + "May be a legitimate shared-helper fix — needs human review.", + offendingFiles: productionFiles, + }; + } + + // ---- All signals satisfied. ------------------------------------------- + return { + resolved: true, + reason: PredicateReason.RESOLVED, + detail: + "Drift genuinely resolved: post-fix collector clean, " + + `the change landed on a report-named fix target (${onTargetFiles.join(", ")}), and every ` + + "changed file is on the sanctioned allowlist (production source + report-named fixture targets only).", + offendingFiles: [], + }; +} + +// --------------------------------------------------------------------------- +// CLI wrapper +// --------------------------------------------------------------------------- + +interface CliArgs { + reportPath: string; + postFixReportPath: string; + postFixExit: number; + /** Explicit changed files, or null to derive from git. */ + changedFiles: string[] | null; +} + +/** Parse argv (without node/script) into CliArgs. Throws PredicateConfigError. */ +export function parseCliArgs(argv: string[]): CliArgs { + let reportPath: string | null = null; + let postFixReportPath: string | null = null; + let postFixExit: number | null = null; + const changedFiles: string[] = []; + let sawChangedFlag = false; + + for (let idx = 0; idx < argv.length; idx++) { + const arg = argv[idx]; + const next = argv[idx + 1]; + switch (arg) { + case "--report": + if (!next) throw new PredicateConfigError("--report requires a path argument"); + reportPath = next; + idx++; + break; + case "--post-fix-report": + if (!next) throw new PredicateConfigError("--post-fix-report requires a path argument"); + postFixReportPath = next; + idx++; + break; + case "--post-fix-exit": { + if (next === undefined) + throw new PredicateConfigError("--post-fix-exit requires a numeric argument"); + // FIX #6 — fail CLOSED on an empty/whitespace value. `Number("")` and + // `Number(" ")` are both 0, which Number.isInteger accepts, so a missing + // recollect output (`--post-fix-exit ""`) would masquerade as a clean + // exit 0 and be trusted as authoritative-clean. Reject it explicitly. + if (next.trim() === "") { + throw new PredicateConfigError( + "--post-fix-exit is empty/whitespace — a missing collector exit code must fail closed, not be treated as clean exit 0", + ); + } + const parsed = Number(next); + if (!Number.isInteger(parsed)) { + throw new PredicateConfigError(`--post-fix-exit must be an integer, got "${next}"`); + } + postFixExit = parsed; + idx++; + break; + } + case "--changed-file": + if (!next) throw new PredicateConfigError("--changed-file requires a path argument"); + sawChangedFlag = true; + changedFiles.push(next); + idx++; + break; + default: + throw new PredicateConfigError(`Unknown argument: ${arg}`); + } + } + + if (!reportPath) throw new PredicateConfigError("--report is required"); + if (!postFixReportPath) throw new PredicateConfigError("--post-fix-report is required"); + if (postFixExit === null) throw new PredicateConfigError("--post-fix-exit is required"); + + return { + reportPath, + postFixReportPath, + postFixExit, + changedFiles: sawChangedFlag ? changedFiles : null, + }; +} + +/** Minimal drift-report read + shape validation. Throws PredicateConfigError. */ +export function readReport(path: string): DriftReport { + if (!existsSync(path)) { + throw new PredicateConfigError(`Drift report not found at ${path}`); + } + let parsed: unknown; + try { + parsed = JSON.parse(readFileSync(path, "utf-8")); + } catch (err: unknown) { + throw new PredicateConfigError( + `Drift report at ${path} is not valid JSON: ${err instanceof Error ? err.message : String(err)}`, + ); + } + if ( + !parsed || + typeof parsed !== "object" || + !Array.isArray((parsed as Record).entries) + ) { + throw new PredicateConfigError( + `Drift report at ${path} has invalid structure: expected { entries: [...] }`, + ); + } + // FIX #6 — align this (previously loose) validator with the stricter + // fix-drift.ts:readDriftReport: a report missing/carrying a non-string + // `timestamp` is a corrupt/truncated collector run and must fail-closed rather + // than be silently trusted as a clean "drift is gone" signal. NOTE: an EMPTY + // `entries: []` is intentionally ACCEPTED — that is exactly what the collector + // emits when no drift remains (the legitimate clean signal); the trust anchor + // for "clean" is the collector EXIT CODE plus fix #1's always-block-on-leg-edit + // rule, not a non-empty entries array. + if (typeof (parsed as Record).timestamp !== "string") { + throw new PredicateConfigError( + `Drift report at ${path} is missing a string "timestamp" — corrupt/truncated report, failing closed`, + ); + } + + // F3 — ENTRY-LEVEL validation, aligned with fix-drift.ts:readDriftReport. The + // predicate reads `entry.builderFile` / `entry.typesFile` (sanctionedTargets) + // and `entry.diffs` (countCriticalDiffs); a structurally-valid report whose + // entries are malformed at those fields would otherwise throw a bare TypeError + // deep inside classification, caught only by the outer runCli try/catch and + // surfaced as an UNNAMED config-error. Validate here so every malformed shape + // fails-closed with a DISTINCT, named PredicateConfigError → CONFIG_ERROR. + const entries = (parsed as { entries: unknown[] }).entries; + for (let i = 0; i < entries.length; i++) { + const entry = entries[i] as Record | null | undefined; + if (!entry || typeof entry !== "object") { + throw new PredicateConfigError(`Drift report at ${path} entry[${i}] is not an object`); + } + if (typeof entry.builderFile !== "string" || entry.builderFile === "") { + throw new PredicateConfigError( + `Drift report at ${path} entry[${i}] has a missing/empty "builderFile" — cannot derive the sanctioned target set, failing closed`, + ); + } + if (entry.typesFile !== null && typeof entry.typesFile !== "string") { + throw new PredicateConfigError( + `Drift report at ${path} entry[${i}] "typesFile" must be a string or null, failing closed`, + ); + } + if (!Array.isArray(entry.diffs)) { + throw new PredicateConfigError( + `Drift report at ${path} entry[${i}] is missing a "diffs" array — cannot score criticalCount, failing closed`, + ); + } + } + return parsed as DriftReport; +} + +/** + * Count critical diffs in a (post-fix) report. Belt-and-suspenders against a + * collector exit code that disagrees with the report contents. + */ +export function countCriticalDiffs(report: DriftReport): number { + return report.entries.reduce( + (sum, e) => sum + e.diffs.filter((d) => d.severity === "critical").length, + 0, + ); +} + +/** + * Parse a `git status --porcelain` line into a file path. Handles quoted paths + * and rename notation. Kept in sync with fix-drift.ts:parsePorcelainLine. + */ +export function parsePorcelainLine(line: string): string { + let path = line.slice(3).trim(); + const arrowIdx = path.indexOf(" -> "); + if (arrowIdx !== -1) path = path.slice(arrowIdx + 4); + if (path.startsWith('"') && path.endsWith('"')) path = path.slice(1, -1); + return path; +} + +/** + * Changed files from `git status --porcelain`. `-c core.quotePath=false` keeps + * non-ASCII paths verbatim (UTF-8) rather than C-quoted/octal-escaped, so a leg + * path with a special character is not mangled into a non-matching spelling + * before classification (round-2 CR slot-1/slot-2 path finding). + */ +export function gitChangedFiles(): string[] { + const out = execSync("git -c core.quotePath=false status --porcelain", { + encoding: "utf-8", + }).trimEnd(); + return out.split("\n").filter(Boolean).map(parsePorcelainLine); +} + +/** + * FIX #4 — AUTHORITATIVE changed-file set. The git working tree is the single + * source of truth for what actually changed. An explicit `--changed-file` list + * is only a hint, and a supplied list that OMITS a relaxed leg would BLIND the + * predicate (the exact WS-2b vector, from the other side). So when a list is + * supplied we cross-check it against git as a set and fail-closed + * (PredicateConfigError → exit 2) on ANY disagreement — a missing file (leg + * hidden) OR an extra file (phantom) both mean the caller's view diverges from + * ground truth and the verdict cannot be trusted. When no list is supplied we + * use the git set directly. + */ +export function crossCheckChangedFiles(explicit: string[] | null, git: string[]): string[] { + if (explicit === null) return git; + const gitSet = new Set(git); + const explicitSet = new Set(explicit); + const missing = git.filter((f) => !explicitSet.has(f)); // git has it, list omits it + const extra = explicit.filter((f) => !gitSet.has(f)); // list has it, git does not + if (missing.length > 0 || extra.length > 0) { + throw new PredicateConfigError( + "--changed-file list disagrees with the git working tree (fail-closed): " + + `${missing.length > 0 ? `omitted by the list: ${missing.join(", ")}; ` : ""}` + + `${extra.length > 0 ? `not present in git: ${extra.join(", ")}` : ""}`.trim(), + ); + } + return explicit; +} + +/** + * Run the predicate from CLI args. Returns the process exit code and prints + * `detail` (and offending files for LOUD reasons) to stdout/stderr. + */ +export function runCli(argv: string[]): number { + let args: CliArgs; + try { + args = parseCliArgs(argv); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + console.error(`CONFIG_ERROR: ${msg}`); + console.log(`reason=${PredicateReason.CONFIG_ERROR}`); + return REASON_EXIT_CODE[PredicateReason.CONFIG_ERROR]; + } + + let report: DriftReport; + let postFixReport: DriftReport; + try { + report = readReport(args.reportPath); + postFixReport = readReport(args.postFixReportPath); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + console.error(`CONFIG_ERROR: ${msg}`); + console.log(`reason=${PredicateReason.CONFIG_ERROR}`); + return REASON_EXIT_CODE[PredicateReason.CONFIG_ERROR]; + } + + // FIX #4 — always derive the AUTHORITATIVE changed set from git, and + // cross-check any supplied --changed-file list against it (fail-closed on + // mismatch) so a leg-omitting list cannot blind the predicate. + let changedFiles: string[]; + try { + changedFiles = crossCheckChangedFiles(args.changedFiles, gitChangedFiles()); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + console.error(`CONFIG_ERROR: ${msg}`); + console.log(`reason=${PredicateReason.CONFIG_ERROR}`); + return REASON_EXIT_CODE[PredicateReason.CONFIG_ERROR]; + } + + // FIX #8 — a post-fix report that structurally passes readReport (has a + // string timestamp + an entries array) can still be malformed at the entry + // level (e.g. an entry missing its `diffs` array), which would make + // countCriticalDiffs throw a bare TypeError. evaluateDriftResolved can also + // throw PredicateConfigError from canonicalizePath (a repo-root-escaping or + // absolute changed-file path). Catch both here and map to a NAMED CONFIG_ERROR + // so the human gets a named cause instead of an uncaught stacktrace with an + // empty reason= line. + let verdict: PredicateResult; + try { + verdict = evaluateDriftResolved({ + changedFiles, + report, + postFixCollectorExit: args.postFixExit, + postFixCriticalCount: countCriticalDiffs(postFixReport), + }); + } catch (err: unknown) { + const msg = err instanceof Error ? err.message : String(err); + console.error(`CONFIG_ERROR: unable to score the drift reports: ${msg}`); + console.log(`reason=${PredicateReason.CONFIG_ERROR}`); + return REASON_EXIT_CODE[PredicateReason.CONFIG_ERROR]; + } + + if (verdict.resolved) { + console.log(verdict.detail); + } else { + console.error(`DRIFT NOT RESOLVED [${verdict.reason}]: ${verdict.detail}`); + if (verdict.offendingFiles.length > 0) { + console.error(`Offending files: ${verdict.offendingFiles.join(", ")}`); + } + } + // Emit a machine-readable reason line for the workflow to capture. + console.log(`reason=${verdict.reason}`); + return REASON_EXIT_CODE[verdict.reason]; +} + +// --------------------------------------------------------------------------- +// Entry-point guard (mirrors drift-report-collector.ts:isDirectRun) +// --------------------------------------------------------------------------- + +function isDirectRun(): boolean { + const entry = process.argv[1]; + if (!entry) return false; + try { + return fileURLToPath(import.meta.url) === resolve(entry); + } catch { + return false; + } +} + +if (isDirectRun()) { + process.exit(runCli(process.argv.slice(2))); +} diff --git a/scripts/drift-types.ts b/scripts/drift-types.ts index 5eaec247..c1246aab 100644 --- a/scripts/drift-types.ts +++ b/scripts/drift-types.ts @@ -15,6 +15,29 @@ */ export type DriftSeverity = "critical" | "warning" | "info"; +/** + * Coarse drift-classification enum used by the delta/summary layers to route a + * report to the right terminal outcome (block vs advisory vs quarantine). It is + * orthogonal to per-diff `DriftSeverity`: a report is `Quarantine` when its + * findings could not be trusted (unparseable / unmapped surface), independent of + * whether individual diffs were `critical`. Purely additive — existing consumers + * that never read `class` are unaffected. + */ +export enum DriftClass { + /** At least one critical, trustworthy drift finding — hard failure. */ + Critical = "critical", + /** Non-critical, informational drift — advisory only. */ + Advisory = "advisory", + /** + * A failure that could not be parsed/mapped into a trustworthy drift finding. + * Neither a clean pass nor a confirmed critical — held aside for human review + * so it is never silently swallowed as a green. + */ + Quarantine = "quarantine", + /** No drift detected. */ + None = "none", +} + export interface ParsedDiff { path: string; severity: DriftSeverity; @@ -22,6 +45,35 @@ export interface ParsedDiff { expected: string; real: string; mock: string; + /** + * Optional stable per-item key (e.g. a model id) used by the delta layer to + * key findings by provider+id. Absent on legacy diffs. + */ + id?: string; + /** Optional coarse classification for this diff. Absent on legacy diffs. */ + class?: DriftClass; +} + +/** + * A test failure that could not be parsed into a trustworthy drift finding and + * was NOT positively classified as benign infrastructure. Rather than crash the + * collector (exit 1) or silently drop the failure (exit 0), the failure is + * captured here so it can surface as a distinct quarantine outcome (exit 5) for + * human review. + */ +export interface QuarantineEntry { + /** Provider inferred from the failing assertion, or a best-effort label. */ + provider: string; + /** The failing test's name (ancestor titles + title). */ + testName: string; + /** + * Raw `file:line` captured from the original stack frame BEFORE stack-frame + * stripping, so the human reviewer can locate the failing assertion. Empty + * string when no frame was available. + */ + rawLocation: string; + /** The (possibly truncated) failure message that could not be parsed. */ + message: string; } export interface DriftEntry { @@ -36,5 +88,25 @@ export interface DriftEntry { export interface DriftReport { timestamp: string; + /** + * ISO-8601 alias of `timestamp`, written so the base-report reuse guard + * (`isBaseReportReusable` via `test-drift.yml`) can read `report.generatedAt`. + * `timestamp` is retained for back-compat with existing consumers + * (drift-slack-summary.ts, etc.). Absent on legacy reports. + */ + generatedAt?: string; + /** + * Coarse run outcome derived from the collector exit code + * (0→"clean", 2→"critical", 5→"quarantine"), written so the reuse guard can + * read `report.conclusion` instead of relying solely on the CI run + * conclusion. Absent on legacy reports. + */ + conclusion?: string; entries: DriftEntry[]; + /** + * Optional list of failures held aside for human review (see QuarantineEntry). + * Absent/empty when there was nothing to quarantine — legacy consumers that + * ignore this field are unaffected. + */ + quarantine?: QuarantineEntry[]; } diff --git a/scripts/fix-drift.ts b/scripts/fix-drift.ts index 73d94f4f..40af57bd 100644 --- a/scripts/fix-drift.ts +++ b/scripts/fix-drift.ts @@ -19,6 +19,30 @@ * 3 — unhandled error (e.g. bad arguments, missing report, git/gh command failure) * 124 — Claude Code timed out (default mode) * In default mode, the exit code is passed through from Claude Code. + * + * In --create-pr mode, the drift-success predicate (drift-success-predicate.ts) + * gates PR creation BEFORE any git add/commit. When it rejects the fix (e.g. a + * fixture-relaxation cheat rather than a real mock change), createPr exits with + * the predicate's reason code instead of opening a PR: + * 10 — NO_PRODUCTION_CHANGE (zero production mock-builder files changed) + * 11 — COMPARISON_LEG_ONLY (only comparison-leg files changed — the cheat) + * 12 — SUPPRESSION_SUSPECTED (allowlist / *.drift.ts assertion edited) + * 13 — STILL_DIRTY (post-fix collector still reports critical drift) + * 14 — QUARANTINE_AFTER_FIX (post-fix collector returned quarantine) + * 15 — COLLECTOR_INFRA (post-fix collector infra failure, OR the + * MANDATORY post-fix args were not supplied) + * 16 — PRODUCTION_CHANGE_OFF_TARGET (production change not in report's target set) + * 17 — UNSANCTIONED_CHANGE (a changed file is not on the allowlist) + * 18 — VERSION_BUMP_FAILED (version bump / CHANGELOG step failed) + * 19 — POST_FIX_PARSE_ERROR (unparseable --post-fix-exit / -report) + * 20 — GIT_PUSH_FAILED (git checkout/add/commit/push failed) + * The legacy exit 4 (no source files changed) is subsumed by 10/11. The + * drift-success predicate is MANDATORY in --create-pr mode: BOTH + * --post-fix-report and --post-fix-exit are required (there is no legacy + * no-post-fix fallback — a missing pair fails closed to COLLECTOR_INFRA rather + * than opening a PR). The real workflow also supplies --report pointing at the + * PINNED pre-fix report (see .github/workflows/fix-drift.yml) so the + * sanctioned-target set cannot be forged by the autofix LLM. */ import { spawn, execSync, execFileSync } from "node:child_process"; @@ -26,6 +50,17 @@ import { readFileSync, writeFileSync, existsSync, unlinkSync } from "node:fs"; import { resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { + evaluateDriftResolved, + readReport as readPostFixReport, + countCriticalDiffs, + canonicalizePath, + gitChangedFiles, + isProductionFile, + sanctionedTargets, + REASON_EXIT_CODE, + PredicateReason, +} from "./drift-success-predicate.js"; import type { DriftReport, DriftSeverity } from "./drift-types.js"; // --------------------------------------------------------------------------- @@ -40,8 +75,6 @@ const KILL_GRACE_MS = 10_000; const VALID_SEVERITIES: ReadonlySet = new Set(["critical", "warning", "info"]); -const SKILL_FILE = "skills/write-fixtures/SKILL.md"; - /** * Map builder source files to the corresponding section names in the * write-fixtures skill documentation. Used to flag which skill sections @@ -73,6 +106,29 @@ export function todayStamp(): string { return new Date().toISOString().slice(0, 10); } +/** GitHub hard limit on PR/issue body length. */ +export const GH_BODY_MAX = 65536; +/** Safety margin below the hard limit. */ +export const GH_BODY_SAFE_MAX = 60000; + +/** + * Truncate `body` to at most `max` characters. When truncation occurs, the + * HEAD of the body (summary/diffs) is preserved and the tail is replaced with a + * marker. The full detail is always available as a workflow artifact. + */ +export function truncateBody(body: string, max: number = GH_BODY_SAFE_MAX): string { + // Never exceed the hard GitHub limit, even if a caller passes a larger max. + const effectiveMax = Math.min(max, GH_BODY_MAX); + if (body.length <= effectiveMax) return body; + const marker = + "\n\n---\n" + + "_Body truncated to fit GitHub's 65536-character limit. " + + "Full drift report is attached as the `drift-report` workflow artifact._\n"; + // When the budget can't even fit the marker, hard-cut instead of overflowing. + if (effectiveMax <= marker.length) return body.slice(0, effectiveMax); + return body.slice(0, effectiveMax - marker.length) + marker; +} + /** * Format an exec error into a human-readable Error object. * Includes exit status, signal, and stderr when available. @@ -278,14 +334,6 @@ export function buildPrompt(report: DriftReport): string { lines.push(""); } - lines.push("## Skill file update"); - lines.push(""); - lines.push("If any builder's output format changed (new fields, renamed fields, changed event"); - lines.push("types), update the write-fixtures skill documentation to match:"); - lines.push(` File: ${SKILL_FILE}`); - lines.push("Only update the Response Types and API Endpoints sections that correspond to the"); - lines.push("changed builders. Do not rewrite unrelated sections."); - lines.push(""); // Add AG-UI specific guidance if any AG-UI entries exist const hasAgUiDrift = report.entries.some((e) => e.provider === "AG-UI"); if (hasAgUiDrift) { @@ -319,7 +367,99 @@ export function buildPrompt(report: DriftReport): string { // Claude Code invocation (default mode) // --------------------------------------------------------------------------- -function invokeClaudeCode(prompt: string): Promise { +/** + * Kill an entire process GROUP by its leader pid. + * + * `spawn(..., { detached: true })` makes the child a group leader whose group + * id equals its pid, so `process.kill(-pid, signal)` signals the child AND all + * of its descendants (e.g. the `npx` wrapper's `@anthropic-ai/claude-code` + * grandchild). Signalling the child pid alone (`child.kill()`) reaches only the + * `npx` wrapper and leaves a wedged grandchild alive to burn the job budget. + * + * ESRCH (group already gone) is a benign "nothing left to kill" and is swallowed + * to `false`. EPERM is DIFFERENT and must NOT be treated as benign: `kill(2)` + * returns EPERM when the target process(es) EXIST but the caller lacks + * permission to signal them — e.g. a grandchild that changed credentials (a + * setuid postinstall) or was re-parented under a remapped container user. Such a + * process can be STILL ALIVE and STILL BURNING the job budget — exactly the + * WS-4 leak this guards against. So on EPERM we log a VISIBLE warning (never + * silently claim success) and attempt a single-PID fallback (`kill(pid)` rather + * than the whole group) in case only the group-leader escaped our permission. + * The job's `timeout-minutes: 30` ceiling remains the ultimate backstop. + * + * Returns true if a group signal was delivered, false if the group was already + * gone (ESRCH) or is present-but-unkillable (EPERM after the fallback attempt). + */ +export function killProcessGroup(pid: number, signal: NodeJS.Signals): boolean { + try { + // Negative pid targets the whole process group led by `pid`. + process.kill(-pid, signal); + return true; + } catch (err) { + const code = (err as NodeJS.ErrnoException).code; + if (code === "ESRCH") { + // The group has already fully exited — nothing left to kill. + return false; + } + if (code === "EPERM") { + // NOT a benign "nothing to kill": the group (or part of it) exists but is + // unkillable by us. Surface it loudly and try a single-PID fallback in + // case only the leader escaped our permission. + console.error( + `WARNING: EPERM signalling process group ${pid} with ${signal} — the group may ` + + "still be ALIVE and unkillable by us (re-credentialed / re-parented child). " + + "Attempting single-PID fallback; the 30-min job ceiling is the final backstop.", + ); + try { + process.kill(pid, signal); + return true; + } catch (fallbackErr) { + const fallbackCode = (fallbackErr as NodeJS.ErrnoException).code; + if (fallbackCode === "ESRCH") { + return false; + } + console.error( + `WARNING: single-PID fallback kill of ${pid} with ${signal} also failed ` + + `(${fallbackCode ?? "unknown"}) — process may leak until the job timeout.`, + ); + return false; + } + } + throw err; + } +} + +/** + * Escalating timeout kill for a detached subprocess: deliver SIGTERM to the + * whole GROUP, then after a grace period escalate to SIGKILL on the GROUP — + * but ONLY if the process has NOT already exited. + * + * The has-exited signal is the caller-supplied `hasExited()` predicate, which + * MUST be backed by the real `close` event (not `child.killed`). Node sets + * `child.killed = true` the instant a signal is DELIVERED, long before the + * process actually exits, so a `!child.killed` guard makes the SIGKILL + * escalation dead code — the original WS-4 defect. Gating on a real exit flag + * is what makes the escalation actually fire against a process that ignores + * SIGTERM. + * + * Returns the grace timer so the caller can cancel it from its `close` handler + * (a clean early exit must not leave a pending SIGKILL escalation queued). + */ +export function scheduleEscalatingKill( + pid: number, + hasExited: () => boolean, + graceMs: number = KILL_GRACE_MS, +): NodeJS.Timeout { + killProcessGroup(pid, "SIGTERM"); + return setTimeout(() => { + if (!hasExited()) { + console.error("Process group did not exit after SIGTERM. Sending SIGKILL to the group..."); + killProcessGroup(pid, "SIGKILL"); + } + }, graceMs); +} + +export function invokeClaudeCode(prompt: string): Promise { return new Promise((done, reject) => { const args = [ "@anthropic-ai/claude-code", @@ -347,26 +487,35 @@ function invokeClaudeCode(prompt: string): Promise { "50", ]; + // `detached: true` puts the child in its OWN process group (gpid === pid), + // so a timeout can signal the WHOLE group — the `npx` wrapper AND its + // `@anthropic-ai/claude-code` grandchild — via `process.kill(-pid, …)`. + // Without it, killing the child pid reaches only the wrapper and a wedged + // grandchild survives to burn the 30-min job budget. const child = spawn("npx", args, { stdio: ["inherit", "pipe", "pipe"], + detached: true, }); const logChunks: Buffer[] = []; let killGraceTimer: NodeJS.Timeout | undefined; let timedOut = false; + // REAL has-exited flag, flipped by the `close` handler. The SIGKILL + // escalation is gated on THIS, never `child.killed` (which is true the + // instant SIGTERM is delivered, making the escalation dead code). + let exited = false; const killTimer = setTimeout(() => { timedOut = true; console.error( - `Claude Code timed out after ${CLAUDE_TIMEOUT_MS / 60000} minutes. Sending SIGTERM...`, + `Claude Code timed out after ${CLAUDE_TIMEOUT_MS / 60000} minutes. ` + + "Sending SIGTERM to the process group...", ); - child.kill("SIGTERM"); - killGraceTimer = setTimeout(() => { - if (!child.killed) { - console.error("Process did not exit after SIGTERM. Sending SIGKILL..."); - child.kill("SIGKILL"); - } - }, KILL_GRACE_MS); + // child.pid can be undefined if the spawn failed; the `error` handler + // covers that path, so only escalate when we have a real group leader. + if (typeof child.pid === "number") { + killGraceTimer = scheduleEscalatingKill(child.pid, () => exited); + } }, CLAUDE_TIMEOUT_MS); child.on("error", (err) => { @@ -383,17 +532,37 @@ function invokeClaudeCode(prompt: string): Promise { reject(err); }); - child.stdout.on("data", (chunk: Buffer) => { - process.stdout.write(chunk); - logChunks.push(chunk); - }); - - child.stderr.on("data", (chunk: Buffer) => { - process.stderr.write(chunk); - logChunks.push(chunk); - }); + // Wire the stream + close handlers inside a guard so a SYNCHRONOUS throw + // here (e.g. `child.stdout` is null on a spawn edge case, making + // `child.stdout.on(...)` a TypeError) cannot strand the already-armed + // `killTimer`. Without this, such a throw rejects the Promise via the + // executor's synchronous-throw semantics BUT the 30-min timer stays live and + // would later group-kill a possibly-reused PID. Clear the timer, then reject. + try { + if (!child.stdout || !child.stderr) { + throw new Error( + "Claude Code child has no stdout/stderr pipe (spawn produced null streams)", + ); + } + child.stdout.on("data", (chunk: Buffer) => { + process.stdout.write(chunk); + logChunks.push(chunk); + }); + + child.stderr.on("data", (chunk: Buffer) => { + process.stderr.write(chunk); + logChunks.push(chunk); + }); + } catch (setupErr) { + clearTimeout(killTimer); + reject(setupErr instanceof Error ? setupErr : new Error(String(setupErr))); + return; + } child.on("close", (code, signal) => { + // Mark real exit BEFORE clearing timers so any in-flight grace timer that + // fires in the same tick sees `exited === true` and skips the SIGKILL. + exited = true; clearTimeout(killTimer); if (killGraceTimer) clearTimeout(killGraceTimer); const logContent = Buffer.concat(logChunks).toString("utf-8"); @@ -502,7 +671,11 @@ export function addChangelogEntry(report: DriftReport, version: string): void { } } -export function buildPrBody(report: DriftReport, changedFiles?: string[]): string { +export function buildPrBody( + report: DriftReport, + changedFiles?: string[], + verdictDetail?: string, +): string { const providers: string[] = []; const diffs: string[] = []; @@ -520,6 +693,19 @@ export function buildPrBody(report: DriftReport, changedFiles?: string[]): strin "", "Auto-generated drift remediation.", "", + // Human-approval backstop (WS-2): this PR was auto-FILTERED by the + // drift-success predicate but is NOT auto-merged. A human must review CI + + // this diff + the verdict below and merge. The predicate is a strong filter, + // not a provable merge gate (the re-collect is not independent of the fix — + // WS-2b), so the merge decision stays with a human. + "> **Needs human review + merge.** This drift-fix PR was opened by the", + "> automated pipeline after the drift-success predicate passed. It is NOT", + "> auto-merged — review CI, the diff, and the verdict below, then merge.", + "", + "### Drift-success predicate verdict", + "", + verdictDetail ? `RESOLVED — ${verdictDetail}` : "RESOLVED.", + "", "### Providers affected", ...providers, "", @@ -553,7 +739,7 @@ export function buildPrBody(report: DriftReport, changedFiles?: string[]): strin "", ); - return sections.join("\n"); + return truncateBody(sections.join("\n")); } /** @@ -581,15 +767,129 @@ export function getChangedFiles(): string[] { return exec("git status --porcelain").split("\n").filter(Boolean).map(parsePorcelainLine); } -function createPr(report: DriftReport): void { +/** + * Optional post-fix collector result, supplied by the workflow so createPr does + * not re-shell the (2-3 min) collector. When present, the drift-success + * predicate is the authoritative PR-open gate; when absent, createPr falls back + * to the legacy "no source files changed" guard (exit 4). + */ +export interface PostFixCollectorResult { + /** Parsed post-fix drift report (from --post-fix-report). */ + report: DriftReport; + /** Exit code of the re-run collector (from --post-fix-exit). */ + exitCode: number; +} + +/** + * The GATED commit groups for a RESOLVED verdict (CR round-3 F-C / F2). Given + * the canonicalized changed-file set and the report's sanctioned-target set, + * partition into the ONLY groups createPr is permitted to stage: + * - `builderFiles` — production mock-builder source (always allowlisted). + * - `testFiles` — ONLY report-named fixture targets under src/__tests__/ + * (any other test file would have BLOCKED at the gate, so + * it must never be staged). + * `stragglers` is every canonicalized changed file that is NOT in one of those + * gated groups. On a RESOLVED verdict it MUST be empty (the predicate allowlist + * already rejected any unclassified file); createPr never `git add`s it. The + * version bump + CHANGELOG are added separately by exact path and are not part + * of this changed-file partition. + */ +export function gatedCommitFiles( + changedFiles: string[], + sanctioned: ReadonlySet, +): { + builderFiles: string[]; + testFiles: string[]; + stragglers: string[]; +} { + const builderFiles = changedFiles.filter(isProductionFile); + const testFiles = changedFiles.filter((f) => f.startsWith("src/__tests__/") && sanctioned.has(f)); + const gated = new Set([...builderFiles, ...testFiles]); + const stragglers = changedFiles.filter((f) => !gated.has(f)); + return { builderFiles, testFiles, stragglers }; +} + +export function createPr(report: DriftReport, postFix?: PostFixCollectorResult): void { const stamp = todayStamp(); - // Determine branch name + // Detect uncommitted changes (staged + unstaged) BEFORE any git write ops so + // the drift-success predicate can gate PR-open ahead of branch/commit/push. + // + // Use the predicate's OWN `gitChangedFiles()` (which runs + // `git -c core.quotePath=false status --porcelain`) and canonicalize every + // path with the predicate's `canonicalizePath` (CR round-3 F-C). This keeps + // classification/staging BYTE-FOR-BYTE identical to what the predicate scored: + // both callers now read the same git invocation and the same canonical spelling, + // so a non-ASCII/C-quoted path or a `./`-prefixed spelling cannot be classified + // one way by the verdict and staged another way here. + const changedFiles = gitChangedFiles().map(canonicalizePath); + + // PR-OPEN GATE (WS-2). When the workflow supplies the post-fix collector + // result, the drift-success predicate is the authoritative decision: it + // rejects fixture-relaxation cheats (a diff that changed ONLY comparison-leg + // files, or silenced the detector) and drifts that were not actually + // resolved. This runs BEFORE any git add/commit — a blocked verdict opens no + // PR (and therefore never reaches auto-merge). + // FIX #5 — the drift-success predicate is MANDATORY; there is NO legacy + // no-post-fix fallback. The old fallback (accept a PR when + // `builderFiles.length || testFiles.length`) re-opened the original + // fixture-only cheat: a run that changed only comparison-leg test files + // satisfied `testFiles.length > 0` and sailed through to a PR. There is no + // safe "no post-fix result" path — without the authoritative post-fix + // collector signal we cannot tell a real fix from a relaxation, so we + // fail-closed to human review (COLLECTOR_INFRA) rather than open a PR. The + // real workflow (fix-drift.yml "Create PR" step) ALWAYS supplies --report + // (the PINNED pre-fix report), --post-fix-report, and --post-fix-exit, so this + // only fires on a misconfigured invocation — which must NOT auto-merge. + if (!postFix) { + console.error( + "ERROR: PR-open gate requires the post-fix collector result " + + "(--post-fix-report + --post-fix-exit). Refusing to open a PR without the " + + "authoritative drift-success signal — routing to human review.", + ); + console.log(`reason=${PredicateReason.COLLECTOR_INFRA}`); + process.exit(REASON_EXIT_CODE[PredicateReason.COLLECTOR_INFRA]); + } + let verdict; + try { + verdict = evaluateDriftResolved({ + changedFiles, + report, + postFixCollectorExit: postFix.exitCode, + postFixCriticalCount: countCriticalDiffs(postFix.report), + }); + } catch (err: unknown) { + // A malformed report or a repo-escaping changed-file path throws from the + // predicate — fail-closed to a NAMED config-error rather than an uncaught + // stacktrace, so the workflow's Slack alert names the cause (mirrors runCli). + const msg = err instanceof Error ? err.message : String(err); + console.error(`ERROR: unable to score the drift reports: ${msg}`); + console.log(`reason=${PredicateReason.CONFIG_ERROR}`); + process.exit(REASON_EXIT_CODE[PredicateReason.CONFIG_ERROR]); + } + if (!verdict.resolved) { + console.error(`ERROR: Drift NOT resolved [${verdict.reason}]: ${verdict.detail}`); + if (verdict.offendingFiles.length > 0) { + console.error(`Offending files: ${verdict.offendingFiles.join(", ")}`); + } + console.error("Aborting PR creation — this fix will be routed to human review."); + console.log(`reason=${verdict.reason}`); + process.exit(REASON_EXIT_CODE[verdict.reason]); + } + console.log(`Drift-success predicate: RESOLVED — ${verdict.detail}`); + + // Determine branch name. A git failure here fails CLOSED with a NAMED reason + // (git-push-failed) rather than a blank-reason exit 3 (see the catch at the + // end of the git-op sequence below). let currentBranch: string; try { currentBranch = exec("git rev-parse --abbrev-ref HEAD"); } catch (err: unknown) { - throw new Error(`Cannot determine current branch for PR creation: ${(err as Error).message}`); + console.error( + `ERROR: cannot determine current branch for PR creation: ${(err as Error).message}`, + ); + console.log(`reason=${PredicateReason.GIT_PUSH_FAILED}`); + process.exit(REASON_EXIT_CODE[PredicateReason.GIT_PUSH_FAILED]); } const branchName = @@ -597,48 +897,84 @@ function createPr(report: DriftReport): void { ? `fix/drift-${stamp}` : currentBranch; + // A git checkout/stage/commit/push failure anywhere below fails CLOSED (no PR + // — the push never completes, so no partial/unversioned PR ships) but the raw + // throw would reach the top-level catch with a BLANK `reason=`. `gitOp` names + // the cause (git-push-failed) so the operator alert is not blank. The + // version-bump block keeps its OWN distinct VERSION_BUMP_FAILED reason. + const gitOp = (label: string, fn: () => void): void => { + try { + fn(); + } catch (err) { + console.error( + `ERROR: git operation failed (${label}) while opening the drift-fix PR — no PR opened:`, + err instanceof Error ? err.message : err, + ); + console.log(`reason=${PredicateReason.GIT_PUSH_FAILED}`); + process.exit(REASON_EXIT_CODE[PredicateReason.GIT_PUSH_FAILED]); + } + }; + if (branchName !== currentBranch) { - execFileSafe("git", ["checkout", "-b", branchName]); + gitOp("checkout -b", () => execFileSafe("git", ["checkout", "-b", branchName])); console.log(`Created branch ${branchName}`); } - // Stage and commit in groups — detect uncommitted changes (staged + unstaged) - const changedFiles = getChangedFiles(); - - const builderFiles = changedFiles.filter( - (f) => f.startsWith("src/") && !f.startsWith("src/__tests__/"), - ); - const testFiles = changedFiles.filter((f) => f.startsWith("src/__tests__/")); - const skillFiles = changedFiles.filter((f) => f.startsWith("skills/")); - - // Abort if no source files were changed — a version-bump-only PR would be misleading - if (builderFiles.length === 0 && testFiles.length === 0) { + // Stage ONLY the GATED set (CR round-3 F-C / F2). The predicate already + // verified that EVERY changed file is on the sanctioned allowlist — production + // mock-builder source OR a fixture the report explicitly named. We stage + // exactly that allowlisted set here (grouped by commit purpose) and NEVER a + // catch-all `git add` of "whatever is still dirty": a straggler-add re-widened + // the PR past the verdict (any unclassified file the predicate did not judge — + // an unnamed fixture, a config file — would have BLOCKED at the gate, so + // sweeping it in AFTER the pass silently defeats the allowlist). + const sanctioned = sanctionedTargets(report); + const { builderFiles, testFiles, stragglers } = gatedCommitFiles(changedFiles, sanctioned); + + // FIX #F5 (round-4) — on a RESOLVED verdict `stragglers` MUST be empty: the + // predicate allowlist already rejected any file that is neither production + // source nor a report-named fixture target, so every changed file must fall + // into one of the gated groups above. If a straggler survives here, the + // verdict and the staging partition have DIVERGED (e.g. a future predicate + // change admitted a file gatedCommitFiles does not classify) — silently + // dropping it would ship an incomplete fix behind a green verdict. Fail closed + // to human review rather than open a PR whose diff differs from what was scored. + if (stragglers.length > 0) { console.error( - "ERROR: No source files changed. Claude Code may not have made any fixes, " + - "or all changes were reverted during verification. Aborting PR creation.", + `ERROR: RESOLVED verdict but ${stragglers.length} changed file(s) are not in any gated ` + + `commit group (${stragglers.join(", ")}) — the verdict and the staging partition have ` + + "diverged. Refusing to open a PR that would silently drop these from the diff.", ); - process.exit(4); // no source files changed (distinct from exit 2 = critical drift) + console.log(`reason=${PredicateReason.UNSANCTIONED_CHANGE}`); + process.exit(REASON_EXIT_CODE[PredicateReason.UNSANCTIONED_CHANGE]); } if (builderFiles.length > 0) { - execFileSafe("git", ["add", ...builderFiles]); - execFileSafe("git", ["commit", "-m", "fix: auto-remediate API drift in builder functions"]); + gitOp("add builders", () => execFileSafe("git", ["add", ...builderFiles])); + gitOp("commit builders", () => + execFileSafe("git", ["commit", "-m", "fix: auto-remediate API drift in builder functions"]), + ); } if (testFiles.length > 0) { - execFileSafe("git", ["add", ...testFiles]); - execFileSafe("git", ["commit", "-m", "test: update SDK shapes for drift remediation"]); - } - - if (skillFiles.length > 0) { - execFileSafe("git", ["add", ...skillFiles]); - execFileSafe("git", [ - "commit", - "-m", - "docs: update write-fixtures skill for builder format changes", - ]); + gitOp("add tests", () => execFileSafe("git", ["add", ...testFiles])); + gitOp("commit tests", () => + execFileSafe("git", ["commit", "-m", "test: update SDK shapes for drift remediation"]), + ); } + // The version bump + CHANGELOG are an EXPLICIT, gated part of the fix set — a + // release always accompanies an auto-remediation — not an unclassified + // straggler. They are workflow-authored (never LLM-authored) and staged by an + // exact path list, so they do not re-open the allowlist. + // + // WS-8 — this step is MANDATORY, so a failure here is a HARD, fail-closed + // error: opening an UNVERSIONED PR would ship a "fix" that a human might merge + // but which never publishes a release, silently delivering no value. We exit + // with a distinct VERSION_BUMP_FAILED reason (routed to the workflow's + // human-review alert) rather than warn-and-continue. No push has happened yet, + // and the builder/test commits above are local-only until the push below — + // which we never reach — so no partial/unversioned PR is ever opened. try { const newVersion = patchBumpVersion(); console.log(`Bumped version to ${newVersion}`); @@ -646,25 +982,23 @@ function createPr(report: DriftReport): void { addChangelogEntry(report, newVersion); console.log("Added CHANGELOG.md entry"); - // Always commit version bump + changelog + // Commit version bump + changelog by EXACT path (not a catch-all). execFileSafe("git", ["add", "package.json", "CHANGELOG.md"]); execFileSafe("git", ["commit", "-m", `chore: bump version to ${newVersion}`, "--allow-empty"]); } catch (err) { - console.warn("Version bump failed, skipping:", err); - // Continue with PR creation without version bump - } - - // Catch any remaining files - const remaining = getChangedFiles(); - if (remaining.length > 0) { - execFileSafe("git", ["add", ...remaining]); - execFileSafe("git", ["commit", "-m", "fix: remaining drift remediation changes"]); + console.error( + "ERROR: version bump / CHANGELOG step failed — refusing to open an UNVERSIONED " + + "drift-fix PR that would merge a fix which never publishes a release:", + err instanceof Error ? err.message : err, + ); + console.log(`reason=${PredicateReason.VERSION_BUMP_FAILED}`); + process.exit(REASON_EXIT_CODE[PredicateReason.VERSION_BUMP_FAILED]); } - execFileSafe("git", ["push", "-u", "origin", branchName]); + gitOp("push", () => execFileSafe("git", ["push", "-u", "origin", branchName])); console.log(`Pushed branch ${branchName}`); - const prBody = buildPrBody(report, changedFiles); + const prBody = buildPrBody(report, changedFiles, verdict.detail); const prTitle = `fix: auto-remediate API drift (${stamp})`; const prBodyFile = `/tmp/aimock-drift-${process.pid}-pr-body.md`; @@ -706,29 +1040,31 @@ function createIssue(report: DriftReport | null): void { const claudeOutput = readFileIfExists(resolve("claude-code-output.log")) ?? "(no output captured)"; - const issueBody = [ - "## Drift detected but auto-fix failed", - "", - "The automated drift remediation pipeline detected API drift but was unable", - "to fix it automatically. Manual intervention is required.", - "", - "### Drift Report", - "", - "```json", - reportJson, - "```", - "", - "### Claude Code Output", - "", - "

    ", - "Full output", - "", - "```", - claudeOutput, - "```", - "", - "
    ", - ].join("\n"); + const issueBody = truncateBody( + [ + "## Drift detected but auto-fix failed", + "", + "The automated drift remediation pipeline detected API drift but was unable", + "to fix it automatically. Manual intervention is required.", + "", + "### Drift Report", + "", + "```json", + reportJson, + "```", + "", + "### Claude Code Output", + "", + "
    ", + "Full output", + "", + "```", + claudeOutput, + "```", + "", + "
    ", + ].join("\n"), + ); const issueTitle = `Drift detected — auto-fix failed (${stamp})`; @@ -769,6 +1105,43 @@ export function parseMode(args: string[]): "pr" | "issue" | "default" { return "default"; } +/** + * FIX #5 — true only when BOTH post-fix collector flags are present with values. + * PR mode requires both (the drift-success predicate is the authoritative + * PR-open gate); there is no legacy no-post-fix fallback that could re-open the + * fixture-only cheat. + */ +export function hasPostFixArgs(args: string[]): boolean { + const reportIdx = args.indexOf("--post-fix-report"); + const exitIdx = args.indexOf("--post-fix-exit"); + const hasReport = reportIdx !== -1 && args[reportIdx + 1] !== undefined; + const hasExit = exitIdx !== -1 && args[exitIdx + 1] !== undefined; + return hasReport && hasExit; +} + +/** + * FIX #F7 (round-4) — parse the `--post-fix-exit` value, failing CLOSED on an + * empty/whitespace or non-integer value. `Number("")` and `Number(" ")` are + * both 0, which Number.isInteger accepts, so a missing recollect output + * (`--post-fix-exit ""`, e.g. a skipped step) would masquerade as a clean + * collector exit 0 and open a PR on an unverified fix. This mirrors the + * predicate CLI's guard (drift-success-predicate.ts:parseCliArgs). Throws on any + * empty/whitespace/non-integer input; the caller must NOT treat that as clean. + */ +export function parsePostFixExit(raw: string): number { + if (raw.trim() === "") { + throw new Error( + "--post-fix-exit is empty/whitespace — a missing collector exit code must fail closed, " + + "not be treated as clean exit 0", + ); + } + const parsed = Number(raw); + if (!Number.isInteger(parsed)) { + throw new Error(`--post-fix-exit must be an integer, got "${raw}"`); + } + return parsed; +} + async function main(): Promise { const args = process.argv.slice(2); const mode = parseMode(args); @@ -801,7 +1174,43 @@ async function main(): Promise { console.log(`Loaded drift report: ${report.entries.length} entries from ${report.timestamp}`); if (mode === "pr") { - createPr(report); + const postFixReportIdx = args.indexOf("--post-fix-report"); + const postFixExitIdx = args.indexOf("--post-fix-exit"); + + // FIX #5 — the post-fix collector result is REQUIRED in PR mode. The old + // path allowed --create-pr with NO post-fix args, which fell back to the + // gameable legacy guard (a test-file-only change satisfied it and opened a + // PR). Both flags must be present; a missing/partial pair throws rather than + // silently skipping the authoritative drift-success gate. + if (!hasPostFixArgs(args)) { + throw new Error( + "--create-pr requires BOTH --post-fix-report and --post-fix-exit (the drift-success " + + "predicate is the authoritative PR-open gate; there is no legacy no-post-fix path)", + ); + } + // FIX #F7 (round-4) — fail CLOSED on an empty/whitespace/non-integer + // --post-fix-exit (see parsePostFixExit): a missing recollect output must + // not masquerade as a clean collector exit 0 and open a PR on an unverified + // fix. Mirrors the predicate CLI's guard. + // + // These parse/read failures fail closed (no PR) — but the raw throw would + // reach the top-level catch with a BLANK `reason=`, so the operator alert is + // uninformative. Emit a NAMED reason (post-fix-parse-error) before + // rethrowing so the workflow's `grep '^reason='` names the cause. + let postFixExit: number; + let postFixReport: DriftReport; + try { + postFixExit = parsePostFixExit(args[postFixExitIdx + 1]); + postFixReport = readPostFixReport(resolve(args[postFixReportIdx + 1])); + } catch (err) { + const msg = err instanceof Error ? err.message : String(err); + console.error(`ERROR: could not parse/read the post-fix collector result: ${msg}`); + console.log(`reason=${PredicateReason.POST_FIX_PARSE_ERROR}`); + process.exit(REASON_EXIT_CODE[PredicateReason.POST_FIX_PARSE_ERROR]); + } + const postFix: PostFixCollectorResult = { report: postFixReport, exitCode: postFixExit }; + + createPr(report, postFix); } else { const prompt = buildPrompt(report); console.log("Invoking Claude Code CLI..."); diff --git a/skills/write-fixtures/SKILL.md b/skills/write-fixtures/SKILL.md index 8c3de1a4..c9c7e05a 100644 --- a/skills/write-fixtures/SKILL.md +++ b/skills/write-fixtures/SKILL.md @@ -19,25 +19,26 @@ aimock is a zero-dependency mock infrastructure for AI apps. Fixture-driven. Mul ## Match Field Reference -| Field | Type | Matches Against | -| ---------------- | ----------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| `userMessage` | `string` | Substring of last `role: "user"` message text | -| `userMessage` | `RegExp` | Pattern test on last `role: "user"` message text | -| `systemMessage` | `string` | Substring of the concatenated text of every `role: "system"` message in the request. Use to gate a fixture on host-supplied context (persona, agent-context entries) so changes to that context cause the fixture to fall through instead of returning a stale baked response | -| `systemMessage` | `string[]` | Array of substrings — ALL must be present in the joined system text (AND semantics). Use when the gate must combine multiple non-adjacent tokens whose serialisation order isn't stable | -| `systemMessage` | `RegExp` | Pattern test on the concatenated system-message text | -| `inputText` | `string` | Substring of embedding input text (concatenated if multiple inputs) | -| `inputText` | `RegExp` | Pattern test on embedding input text | -| `toolName` | `string` | Exact match on any tool in request's `tools[]` array (by `function.name`) | -| `toolCallId` | `string` | Exact match on `tool_call_id` of last `role: "tool"` message | -| `model` | `string` | Exact match on `req.model` | -| `model` | `RegExp` | Pattern test on `req.model` | -| `responseFormat` | `string` | Exact match on `req.response_format.type` (`"json_object"`, `"json_schema"`) | -| `sequenceIndex` | `number` | Matches only when this fixture's match count equals the given index (0-based) | -| `turnIndex` | `number` | Stateless conversation-depth matching. Counts `role: "assistant"` messages in the request; matches when that count equals the value. `turnIndex: 0` = first turn (no prior assistant messages). Use instead of `sequenceIndex` for shared/deployed instances where stateful counters break under concurrency | -| `hasToolResult` | `boolean` | Stateless tool-message presence matching. `true` matches when any `role: "tool"` message exists in the request; `false` matches when none exist. Provider-consistent across all aimock handlers (OpenAI, Claude, Gemini, Bedrock, Ollama, Cohere) | -| `endpoint` | `string` | Restrict to endpoint type: `"chat"`, `"image"`, `"speech"`, `"transcription"`, `"video"`, `"embedding"` | -| `predicate` | `(req: ChatCompletionRequest) => boolean` | Custom function — full access to request | +| Field | Type | Matches Against | +| -------------------- | ----------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `userMessage` | `string` | Substring of last `role: "user"` message text | +| `userMessage` | `RegExp` | Pattern test on last `role: "user"` message text | +| `systemMessage` | `string` | Substring of the concatenated text of every `role: "system"` message in the request. Use to gate a fixture on host-supplied context (persona, agent-context entries) so changes to that context cause the fixture to fall through instead of returning a stale baked response | +| `systemMessage` | `string[]` | Array of substrings — ALL must be present in the joined system text (AND semantics). Use when the gate must combine multiple non-adjacent tokens whose serialisation order isn't stable | +| `systemMessage` | `RegExp` | Pattern test on the concatenated system-message text | +| `inputText` | `string` | Substring of embedding input text (concatenated if multiple inputs) | +| `inputText` | `RegExp` | Pattern test on embedding input text | +| `toolName` | `string` | Exact match on any tool in request's `tools[]` array (by `function.name`) | +| `toolCallId` | `string` | Exact match on `tool_call_id` of last `role: "tool"` message | +| `toolResultContains` | `string` | Substring of the last tool message's text content, gated on that message being the request's LAST message (same rule as `toolCallId`). Discriminates resume paths that share a `tool_call_id` and differ only inside the tool-result payload (e.g. approve `{"chosen_time": …}` vs cancel `{"cancelled": true}`) | +| `model` | `string` | Exact match on `req.model` | +| `model` | `RegExp` | Pattern test on `req.model` | +| `responseFormat` | `string` | Exact match on `req.response_format.type` (`"json_object"`, `"json_schema"`) | +| `sequenceIndex` | `number` | Matches only when this fixture's match count equals the given index (0-based) | +| `turnIndex` | `number` | Stateless conversation-depth matching. Counts `role: "assistant"` messages in the request; matches when that count equals the value. `turnIndex: 0` = first turn (no prior assistant messages). Use instead of `sequenceIndex` for shared/deployed instances where stateful counters break under concurrency | +| `hasToolResult` | `boolean` | Stateless tool-message presence matching. `true` matches when any `role: "tool"` message exists in the request; `false` matches when none exist. Provider-consistent across all aimock handlers (OpenAI, Claude, Gemini, Bedrock, Ollama, Cohere) | +| `endpoint` | `string` | Restrict to endpoint type: `"chat"`, `"image"`, `"speech"`, `"transcription"`, `"video"`, `"embedding"` | +| `predicate` | `(req: ChatCompletionRequest) => boolean` | Custom function — full access to request | **AND logic**: all specified fields must match. Empty match `{}` = catch-all. @@ -45,14 +46,15 @@ Multi-part content (e.g., `[{type: "text", text: "hello"}]`) is automatically ex ### When to Use Each Multi-turn Matching Approach -| Approach | Stateless? | Best For | -| --------------- | ---------- | --------------------------------------------------------------------------------------------------------- | -| `turnIndex` | Yes | Shared/deployed instances; matches on conversation depth (count of assistant messages in request) | -| `hasToolResult` | Yes | Simplest option for 2-step tool flows — boolean: are there tool results in the request? | -| `sequenceIndex` | No | Single-client unit tests with repeated identical requests (server-side counter, breaks under concurrency) | -| `toolCallId` | Yes | Matching specific tool result IDs in the conversation history | +| Approach | Stateless? | Best For | +| -------------------- | ---------- | --------------------------------------------------------------------------------------------------------- | +| `turnIndex` | Yes | Shared/deployed instances; matches on conversation depth (count of assistant messages in request) | +| `hasToolResult` | Yes | Simplest option for 2-step tool flows — boolean: are there tool results in the request? | +| `sequenceIndex` | No | Single-client unit tests with repeated identical requests (server-side counter, breaks under concurrency) | +| `toolCallId` | Yes | Matching specific tool result IDs in the conversation history | +| `toolResultContains` | Yes | Same tool call id, different outcomes — match on the tool-result payload (approve vs cancel legs) | -**Prefer stateless approaches** (`turnIndex`, `hasToolResult`) for shared aimock instances (deployed via Docker, used by multiple test runners). Use `sequenceIndex` only in isolated single-client unit tests where the counter won't be corrupted by concurrent requests. +**Prefer stateless approaches** (`turnIndex`, `hasToolResult`, `toolResultContains`) for shared aimock instances (deployed via Docker, used by multiple test runners). Use `sequenceIndex` only in isolated single-client unit tests where the counter won't be corrupted by concurrent requests. ### Multi-turn fixture examples @@ -64,6 +66,11 @@ Multi-part content (e.g., `[{type: "text", text: "hello"}]`) is automatically ex // Same thing with hasToolResult (simpler for 2-step) {"match": {"userMessage": "trip to mars", "hasToolResult": false}, "response": {"toolCalls": [{"id": "call_001", "name": "generate_steps", "arguments": "{}"}]}} {"match": {"userMessage": "trip to mars", "hasToolResult": true}, "response": {"content": "Great choices!"}} + +// HITL suspend tool where approve and cancel resume with the SAME tool call id — +// discriminate on the tool-result payload; put the cancel leg first (first match wins) +{"match": {"toolCallId": "call_001", "toolResultContains": "\"cancelled\""}, "response": {"content": "No problem — nothing was booked."}} +{"match": {"toolCallId": "call_001"}, "response": {"content": "Booked: Monday 9:00 AM confirmed."}} ``` ## Response Types @@ -255,7 +262,7 @@ mock.on( mock.on({ userMessage: "status", sequenceIndex: 1 }, { content: "All systems operational." }); ``` -Match counts are tracked per fixture group and reset with `reset()` or `resetMatchCounts()`. +Match counts are tracked per fixture group. Use `resetMatchCounts()` between tests to reset counts while keeping loaded fixtures. `reset()` also clears the fixture pool, so avoid it between tests that share a loaded fixture set. ### Streaming physics (realistic timing) @@ -500,7 +507,7 @@ These fields map correctly across all provider formats — for example, `finishR 10. **Embeddings auto-generate if no fixture matches** — deterministic vectors are generated from the input text hash. You don't need a catch-all for embedding requests. -11. **Sequential response counts are tracked per fixture** — counts reset with `reset()` or `resetMatchCounts()`. The count increments after each match of that fixture group (all fixtures sharing the same non-`sequenceIndex` match fields). +11. **Sequential response counts are tracked per fixture** — use `resetMatchCounts()` between tests to reset counts while keeping loaded fixtures; `reset()` also clears the fixture pool, so don't use it between tests that share a loaded fixture set. The count increments after each match of that fixture group (all fixtures sharing the same non-`sequenceIndex` match fields). 12. **Bedrock uses Anthropic Messages format internally** — the adapter normalizes Bedrock requests to `ChatCompletionRequest`, so the same fixtures work. Bedrock supports both non-streaming (`/invoke`, `/converse`) and streaming (`/invoke-with-response-stream`, `/converse-stream`) endpoints. @@ -549,7 +556,7 @@ await suite.start(); // suite.llm — the LLMock instance // suite.url — base URL -afterEach(() => suite.reset()); // resets everything +afterEach(() => suite.llm.resetMatchCounts()); // reset sequence counts, keep fixtures afterAll(() => suite.stop()); ``` @@ -684,8 +691,8 @@ mock.loadFixtureDir("./fixtures"); await mock.start(); process.env.OPENAI_BASE_URL = `${mock.url}/v1`; -// Per-test cleanup -afterEach(() => mock.reset()); // clears fixtures AND journal +// Per-test cleanup — reset sequence match counts, keep the loaded fixtures +afterEach(() => mock.resetMatchCounts()); // Teardown afterAll(async () => await mock.stop()); @@ -736,6 +743,8 @@ const mock = await LLMock.create({ port: 0 }); // creates + starts in one call | `url` / `baseUrl` | Server URL (throws if not started) | | `port` | Server port number | +Between tests that share a loaded fixture set, use `resetMatchCounts()` (not `reset()`, which also clears fixtures). For a `MockSuite`, call `suite.llm.resetMatchCounts()` — the suite itself has no `resetMatchCounts()`. + Sequential responses use `on()` with `sequenceIndex` in the match — there is no dedicated convenience method. ## Record-and-Replay (VCR Mode) diff --git a/src/__tests__/agui-record-buffer-cap.test.ts b/src/__tests__/agui-record-buffer-cap.test.ts new file mode 100644 index 00000000..85ec8ddf --- /dev/null +++ b/src/__tests__/agui-record-buffer-cap.test.ts @@ -0,0 +1,186 @@ +import { describe, it, expect, afterEach, vi } from "vitest"; +import * as http from "node:http"; +import * as fs from "node:fs"; +import * as os from "node:os"; +import * as path from "node:path"; +import { AGUIMock } from "../agui-mock.js"; +import { setAGUIRecordBufferCeilingForTests } from "../agui-recorder.js"; +import { Logger } from "../logger.js"; + +// --------------------------------------------------------------------------- +// AG-UI record-buffer cap. +// +// The AG-UI recorder tees every upstream SSE chunk to the client AND buffers a +// copy so it can `Buffer.concat(chunks).toString()` + parse the events into a +// fixture once the stream ends. With no cap, a large upstream response builds a +// string past V8's ~512 MiB max string length and throws +// `RangeError: Invalid string length` (the ~1/sec prod crash). These tests +// exercise the REAL proxyAndRecordAGUI/teeUpstreamStream path via a raw local +// upstream streaming a large SSE body, with a low test-only cap so the run +// stays fast: the client must still receive every relayed byte, the server must +// not crash, and recording must be skipped (no fixture written). +// --------------------------------------------------------------------------- + +let upstream: http.Server | undefined; +let agui: AGUIMock | undefined; +let tmpDir: string | undefined; + +afterEach(async () => { + if (agui) { + try { + await agui.stop(); + } catch { + /* already stopped */ + } + agui = undefined; + } + if (upstream) { + await new Promise((resolve) => upstream!.close(() => resolve())); + upstream = undefined; + } + if (tmpDir) { + fs.rmSync(tmpDir, { recursive: true, force: true }); + tmpDir = undefined; + } + setAGUIRecordBufferCeilingForTests(undefined); + vi.restoreAllMocks(); +}); + +/** + * A raw upstream that streams `totalBytes` of valid AG-UI SSE frames in + * `chunkBytes`-sized frames, then closes. Each frame is a TEXT_MESSAGE_CONTENT + * event so the recorder's SSE parser exercises normally under the cap. + */ +function createLargeAguiUpstream(totalBytes: number, chunkBytes: number): Promise { + return new Promise((resolve) => { + upstream = http.createServer((_req, res) => { + res.writeHead(200, { + "Content-Type": "text/event-stream", + "Cache-Control": "no-cache", + Connection: "keep-alive", + }); + let sent = 0; + const writeNext = () => { + if (res.writableEnded || res.destroyed) return; + if (sent >= totalBytes) { + res.end(); + return; + } + const payload = JSON.stringify({ + type: "TEXT_MESSAGE_CONTENT", + messageId: "m1", + delta: "x".repeat(Math.max(1, chunkBytes)), + }); + const frame = `data: ${payload}\n\n`; + sent += Buffer.byteLength(frame); + if (res.write(frame)) { + setImmediate(writeNext); + } else { + res.once("drain", writeNext); + } + }; + writeNext(); + }); + upstream.listen(0, "127.0.0.1", () => { + const { port } = upstream!.address() as { port: number }; + resolve(`http://127.0.0.1:${port}`); + }); + }); +} + +/** Stream a POST and count relayed bytes without buffering a huge string. */ +function postStreaming(url: string): Promise<{ status: number; bytesReceived: number }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify({ + threadId: "t1", + runId: "r1", + messages: [{ id: "u1", role: "user", content: "stream me a lot" }], + tools: [], + context: [], + state: {}, + forwardedProps: {}, + }); + const parsed = new URL(url); + const req = http.request( + { + hostname: parsed.hostname, + port: parsed.port, + path: parsed.pathname, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + }, + }, + (res) => { + let bytesReceived = 0; + res.on("data", (c: Buffer) => { + bytesReceived += c.length; + }); + res.on("end", () => resolve({ status: res.statusCode ?? 0, bytesReceived })); + res.on("error", reject); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +describe("AG-UI record-buffer cap", () => { + it("relays the full streamed body, does not throw, and skips recording when over the cap", async () => { + // 4 MB total, ~4 KB frames, with a low 256 KB test-only ceiling so the + // buffer must truncate well before the full body accumulates. The client + // must still receive every relayed byte. + const TOTAL = 4 * 1024 * 1024; + const CHUNK = 4 * 1024; + setAGUIRecordBufferCeilingForTests(256 * 1024); + + const upstreamUrl = await createLargeAguiUpstream(TOTAL, CHUNK); + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "agui-bufcap-")); + + const warnings: string[] = []; + vi.spyOn(Logger.prototype, "warn").mockImplementation((...args: unknown[]) => { + warnings.push(args.map(String).join(" ")); + }); + + agui = new AGUIMock({ port: 0, logLevel: "warn" }); + agui.enableRecording({ upstream: upstreamUrl, proxyOnly: false, fixturePath: tmpDir }); + await agui.start(); + + const resp = await postStreaming(agui.url); + + // Client correctness: full relay regardless of the cap. If the server had + // crashed with Invalid string length the relay would have been cut short. + expect(resp.status).toBe(200); + expect(resp.bytesReceived).toBeGreaterThanOrEqual(TOTAL); + + // The record-buffer cap tripped and recording was skipped. + expect(warnings.some((w) => /record buffer cap/i.test(w))).toBe(true); + + // No fixture written (the partial buffer was dropped, not journaled). + const files = fs.readdirSync(tmpDir).filter((f) => f.endsWith(".json")); + expect(files).toHaveLength(0); + }); + + it("records normally when the response stays under the cap", async () => { + const TOTAL = 32 * 1024; + const CHUNK = 4 * 1024; + setAGUIRecordBufferCeilingForTests(4 * 1024 * 1024); + + const upstreamUrl = await createLargeAguiUpstream(TOTAL, CHUNK); + tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "agui-bufcap-ok-")); + + agui = new AGUIMock({ port: 0 }); + agui.enableRecording({ upstream: upstreamUrl, proxyOnly: false, fixturePath: tmpDir }); + await agui.start(); + + const resp = await postStreaming(agui.url); + expect(resp.status).toBe(200); + expect(resp.bytesReceived).toBeGreaterThanOrEqual(TOTAL); + + // Under the cap → a fixture IS written. + const files = fs.readdirSync(tmpDir).filter((f) => f.endsWith(".json")); + expect(files.length).toBeGreaterThan(0); + }); +}); diff --git a/src/__tests__/config-loader.test.ts b/src/__tests__/config-loader.test.ts index 78a814a4..7f03af2e 100644 --- a/src/__tests__/config-loader.test.ts +++ b/src/__tests__/config-loader.test.ts @@ -1,9 +1,11 @@ -import { describe, it, expect, beforeEach, afterEach } from "vitest"; +import { describe, it, expect, beforeEach, afterEach, vi } from "vitest"; import { mkdtempSync, writeFileSync, mkdirSync, rmSync } from "node:fs"; import { tmpdir } from "node:os"; import { join } from "node:path"; import { loadConfig, startFromConfig } from "../config-loader.js"; import type { AimockConfig } from "../config-loader.js"; +import type { RecordedTimings } from "../types.js"; +import { Logger } from "../logger.js"; function makeTmpDir(): string { return mkdtempSync(join(tmpdir(), "config-loader-test-")); @@ -32,6 +34,47 @@ function writeFixtureFile(dir: string, name = "fixtures.json"): string { return filePath; } +// Content is long enough to split into several chunks so per-chunk delays are observable +function writeStreamFixtureFile( + dir: string, + recordedTimings?: RecordedTimings, + name = "stream-fixtures.json", +): string { + const filePath = join(dir, name); + writeFileSync( + filePath, + JSON.stringify({ + fixtures: [ + { + match: { userMessage: "hello" }, + response: { content: "Hello world from the streaming config test!" }, + ...(recordedTimings ? { recordedTimings } : {}), + }, + ], + }), + "utf-8", + ); + return filePath; +} + +// Streams a completion and counts SSE frames. Without stream: true there is no SSE and no +// chunk delays, so the timing assertions below would pass whether or not the options are wired. +async function streamChunkCount(url: string): Promise { + const resp = await fetch(`${url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + model: "gpt-4", + stream: true, + messages: [{ role: "user", content: "hello" }], + }), + }); + const body = await resp.text(); + return body + .split("\n\n") + .filter((frame) => frame.startsWith("data: ") && !frame.includes("[DONE]")).length; +} + describe("loadConfig", () => { let tmpDir: string; @@ -156,6 +199,108 @@ describe("startFromConfig", () => { expect(resp.status).toBe(500); }); + it("with llm.chunkSize, the response is split into more chunks", async () => { + const fixturePath = writeStreamFixtureFile(tmpDir); + + const small = await startFromConfig({ llm: { fixtures: fixturePath, chunkSize: 5 } }); + cleanups.push(() => small.llmock.stop()); + const smallChunks = await streamChunkCount(small.url); + + const large = await startFromConfig({ llm: { fixtures: fixturePath, chunkSize: 1000 } }); + cleanups.push(() => large.llmock.stop()); + const largeChunks = await streamChunkCount(large.url); + + expect(smallChunks).toBeGreaterThan(largeChunks); + }); + + it("with llm.latency, streamed chunks are delayed", async () => { + const fixturePath = writeStreamFixtureFile(tmpDir); + const config: AimockConfig = { + llm: { fixtures: fixturePath, chunkSize: 5, latency: 100 }, + }; + const { llmock, url } = await startFromConfig(config); + cleanups.push(() => llmock.stop()); + + const start = Date.now(); + const chunks = await streamChunkCount(url); + const elapsed = Date.now() - start; + + // 100ms per chunk across >= 5 chunks. Asserted as a lower bound so a slow + // machine can never fail it; without the passthrough latency is 0. + expect(chunks).toBeGreaterThanOrEqual(5); + expect(elapsed).toBeGreaterThanOrEqual(250); + }); + + it("with llm.replaySpeed, recorded timings replay faster", async () => { + // 500ms TTFT + 4 x 500ms inter-chunk = ~2500ms at 1x speed + const fixturePath = writeStreamFixtureFile(tmpDir, { + ttftMs: 500, + interChunkDelaysMs: [500, 500, 500, 500], + totalDurationMs: 2500, + }); + const config: AimockConfig = { + llm: { fixtures: fixturePath, chunkSize: 5, replaySpeed: 100 }, + }; + const { llmock, url } = await startFromConfig(config); + cleanups.push(() => llmock.stop()); + + const start = Date.now(); + await streamChunkCount(url); + const elapsed = Date.now() - start; + + // At 100x this lands in ~25ms; the bound is deliberately generous so only a + // dropped passthrough (which costs the full ~2500ms) can fail it. + expect(elapsed).toBeLessThan(1000); + }); + + it("with non-positive llm.replaySpeed, warns and ignores it", async () => { + const fixturePath = writeStreamFixtureFile(tmpDir); + // Spy on the logger the code path actually uses rather than console, so the + // assertion verifies the intended warn call without coupling to the + // "[aimock]" prefix or the console transport. + const warn = vi.spyOn(Logger.prototype, "warn").mockImplementation(() => {}); + + // 0 looks like "no delay" but hits the `speed > 0` guard in calculateDelay, + // which would replay at full recorded speed. + const { llmock } = await startFromConfig({ + llm: { fixtures: fixturePath, replaySpeed: 0 }, + }); + cleanups.push(() => llmock.stop()); + + expect(warn).toHaveBeenCalledWith( + expect.stringContaining("llm.replaySpeed must be a positive number"), + ); + warn.mockRestore(); + }); + + it("with llm.logLevel debug, the server logs debug output that silent suppresses", async () => { + const fixturePath = writeFixtureFile(tmpDir); + // The server logger writes debug/info lines via console.log; capture them so + // we can assert the configured level actually reaches the running server. + const log = vi.spyOn(console, "log").mockImplementation(() => {}); + + // A matched request emits a "Fixture matched" debug line, but only when the + // configured logLevel is high enough — so it is a direct probe of the passthrough. + async function debugLineCount(logLevel: "debug" | "silent"): Promise { + log.mockClear(); + const { llmock, url } = await startFromConfig({ + llm: { fixtures: fixturePath, logLevel }, + }); + cleanups.push(() => llmock.stop()); + await fetch(`${url}/v1/chat/completions`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model: "gpt-4", messages: [{ role: "user", content: "hello" }] }), + }); + return log.mock.calls.filter((args) => String(args[1]).startsWith("Fixture matched")).length; + } + + expect(await debugLineCount("debug")).toBeGreaterThan(0); + expect(await debugLineCount("silent")).toBe(0); + + log.mockRestore(); + }); + it("with mcp tools config, MCPMock created and tools/list works", async () => { const config: AimockConfig = { mcp: { @@ -649,11 +794,8 @@ describe("startFromConfig", () => { { pattern: "stream", events: [ - { - kind: "status-update", - taskId: "t1", - status: { state: "working", message: { parts: [{ text: "streaming..." }] } }, - }, + { type: "status", state: "TASK_STATE_WORKING" }, + { type: "artifact", parts: [{ text: "streaming..." }], name: "out" }, ], delayMs: 0, }, @@ -670,6 +812,28 @@ describe("startFromConfig", () => { expect(cardRes.status).toBe(200); const card = await cardRes.json(); expect(card.name).toBe("stream-agent"); + + // Verify the streaming task actually streams the configured events over SSE + const streamRes = await fetch(`${url}/a2a`, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ + jsonrpc: "2.0", + method: "SendStreamingMessage", + params: { message: { parts: [{ text: "please stream" }] } }, + id: 1, + }), + }); + expect(streamRes.status).toBe(200); + expect(streamRes.headers.get("content-type")).toBe("text/event-stream"); + const streamBody = await streamRes.text(); + const events = streamBody + .split("\n\n") + .filter((frame) => frame.startsWith("data: ")) + .map((frame) => JSON.parse(frame.slice("data: ".length))); + expect(events.length).toBe(2); + expect(events[0].result.task.status.state).toBe("TASK_STATE_WORKING"); + expect(events[1].result.artifact.parts[0].text).toBe("streaming..."); }); it("with a2a custom path, mounts at specified path for tasks", async () => { diff --git a/src/__tests__/drift-collector.test.ts b/src/__tests__/drift-collector.test.ts index d749934b..5c095188 100644 --- a/src/__tests__/drift-collector.test.ts +++ b/src/__tests__/drift-collector.test.ts @@ -1,40 +1,71 @@ /** - * Tests for key functions in scripts/drift-report-collector.ts + * Tests for the drift-report collector's pure functions. * - * Since scripts/ is outside the rootDir for the main tsconfig (and vitest - * only covers src/__tests__), these functions are duplicated here as local - * test helpers to keep the test runner config intact. Any changes to the - * originals must be reflected here. + * These tests import and exercise the REAL exported functions from + * scripts/drift-report-collector.ts — NOT local reimplementations. Importing + * the module does NOT run main(): the collector guards its entry point with an + * `isDirectRun()` check (main() only fires under `tsx scripts/…` invocation), + * so importing here is side-effect-free. + * + * The canary fixtures below are REAL vitest failure-message shapes captured by + * running the canary / drift / infra assertions under + * `vitest run … --reporter=json` (see PR #291 RED/GREEN logs under tmp/). They + * are NOT hand-authored: the single-glyph Unicode ellipsis `…(N)`, the + * `AssertionError:` prefix, the leading blank line before a formatted drift + * report, and the stack-frame layout are exactly what vitest emits. */ import { describe, it, expect } from "vitest"; import { formatDriftReport } from "./drift/schema.js"; import type { ShapeDiff } from "./drift/schema.js"; +import { + parseDriftBlock, + extractProviderName, + extractSurfaceKey, + extractScenario, + parseKnownModelsCanary, + collectDriftEntries, + computeExitCode, + conclusionForExitCode, + classifyUnparseableAsInfra, + INFRA_INDICATOR_SOURCES, + infraIndicatorSample, +} from "../../scripts/drift-report-collector.js"; +import type { DriftEntry, QuarantineEntry } from "../../scripts/drift-types.js"; +import { SURFACE_REGISTRY, KNOWN_SURFACE_SLUGS, isKnownSurface } from "./drift/surface-registry.js"; +import { existsSync, readFileSync, readdirSync } from "node:fs"; +import { resolve } from "node:path"; +import { isBaseReportReusable } from "../../scripts/drift-delta.js"; +import type { DriftReport } from "../../scripts/drift-types.js"; // --------------------------------------------------------------------------- -// Local copies of the types and functions under test -// (mirrors scripts/drift-report-collector.ts — keep in sync) +// Helpers for the A1.3 CollectResult shape ({ entries, quarantine }). +// collectDriftEntries no longer returns a bare array nor throws on unmapped / +// unparseable-not-infra failures — those are quarantined (→ exit 5). // --------------------------------------------------------------------------- -type DriftSeverity = "critical" | "warning" | "info"; - -interface ParsedDiff { - path: string; - severity: DriftSeverity; - issue: string; - expected: string; - real: string; - mock: string; +function entriesOf(result: VitestJsonResult): DriftEntry[] { + return collectDriftEntries(result).entries; } -interface VitestJsonResult { - testResults: VitestTestFile[]; +function quarantineOf(result: VitestJsonResult): QuarantineEntry[] { + return collectDriftEntries(result).quarantine; } -interface VitestTestFile { - assertionResults: VitestAssertion[]; +/** The exit code main() would emit for a given collect result (agUiSkipped=false). */ +function exitCodeOf(result: VitestJsonResult): 0 | 1 | 2 | 5 { + const { entries, quarantine } = collectDriftEntries(result); + const criticalCount = entries.reduce( + (sum, e) => sum + e.diffs.filter((d) => d.severity === "critical").length, + 0, + ); + return computeExitCode(criticalCount, quarantine.length, false); } +// --------------------------------------------------------------------------- +// Vitest JSON reporter fixture builders +// --------------------------------------------------------------------------- + interface VitestAssertion { status: string; ancestorTitles: string[]; @@ -42,226 +73,10 @@ interface VitestAssertion { failureMessages: string[]; } -interface ProviderMapping { - builderFile: string; - builderFunctions: string[]; - typesFile: string | null; - sdkShapesFile?: string; -} - -const PROVIDER_MAP: Record = { - "OpenAI Chat": { - builderFile: "src/helpers.ts", - builderFunctions: [ - "buildTextCompletion", - "buildToolCallCompletion", - "buildTextChunks", - "buildToolCallChunks", - ], - typesFile: "src/types.ts", - }, - "OpenAI Responses": { - builderFile: "src/responses.ts", - builderFunctions: [ - "buildTextResponse", - "buildToolCallResponse", - "buildTextStreamEvents", - "buildToolCallStreamEvents", - ], - typesFile: null, - }, - Anthropic: { - builderFile: "src/messages.ts", - builderFunctions: [ - "buildClaudeTextResponse", - "buildClaudeToolCallResponse", - "buildClaudeTextStreamEvents", - "buildClaudeToolCallStreamEvents", - ], - typesFile: null, - }, - "Anthropic Claude": { - builderFile: "src/messages.ts", - builderFunctions: [ - "buildClaudeTextResponse", - "buildClaudeToolCallResponse", - "buildClaudeTextStreamEvents", - "buildClaudeToolCallStreamEvents", - ], - typesFile: null, - }, - "Google Gemini": { - builderFile: "src/gemini.ts", - builderFunctions: [ - "buildGeminiTextResponse", - "buildGeminiToolCallResponse", - "buildGeminiTextStreamChunks", - "buildGeminiToolCallStreamChunks", - ], - typesFile: null, - }, - Gemini: { - builderFile: "src/gemini.ts", - builderFunctions: [ - "buildGeminiTextResponse", - "buildGeminiToolCallResponse", - "buildGeminiTextStreamChunks", - "buildGeminiToolCallStreamChunks", - ], - typesFile: null, - }, - "OpenAI Realtime": { - builderFile: "src/ws-realtime.ts", - builderFunctions: ["handleWebSocketRealtime", "realtimeItemsToMessages"], - typesFile: null, - }, - "OpenAI Responses WS": { - builderFile: "src/ws-responses.ts", - builderFunctions: ["handleWebSocketResponses"], - typesFile: null, - }, - "Gemini Live": { - builderFile: "src/ws-gemini-live.ts", - builderFunctions: ["handleWebSocketGeminiLive"], - typesFile: null, - }, - "OpenAI Embeddings": { - builderFile: "src/helpers.ts", - builderFunctions: ["buildEmbeddingResponse", "generateDeterministicEmbedding"], - typesFile: null, - sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", - }, - "Gemini Interactions": { - builderFile: "src/gemini-interactions.ts", - builderFunctions: [ - "buildInteractionsTextResponse", - "buildInteractionsToolCallResponse", - "buildInteractionsContentWithToolCallsResponse", - "buildInteractionsTextSSEEvents", - "buildInteractionsToolCallSSEEvents", - "buildInteractionsContentWithToolCallsSSEEvents", - ], - typesFile: null, - }, -}; - -const SDK_SHAPES_FILE = "src/__tests__/drift/sdk-shapes.ts"; - -const VALID_SEVERITIES = new Set(["critical", "warning", "info"]); - -function parseDriftBlock(text: string): { context: string; diffs: ParsedDiff[] } | null { - const headerMatch = text.match(/API DRIFT DETECTED:\s*(.+)/); - if (!headerMatch) return null; - - const context = headerMatch[1].trim(); - const diffs: ParsedDiff[] = []; - - const entryPattern = - /\d+\.\s*\[(\w+)\]\s*(.+)\n\s*Path:\s*(.+)\n\s*SDK:\s*(.+)\n\s*Real:\s*(.+)\n\s*Mock:\s*(.+)/g; - - let match: RegExpExecArray | null; - while ((match = entryPattern.exec(text)) !== null) { - const severity = match[1].trim(); - if (!VALID_SEVERITIES.has(severity as DriftSeverity)) continue; - diffs.push({ - severity: severity as DriftSeverity, - issue: match[2].trim(), - path: match[3].trim(), - expected: match[4].trim(), - real: match[5].trim(), - mock: match[6].trim(), - }); - } - - return { context, diffs }; -} - -function extractProviderName(text: string): string | null { - const sorted = Object.keys(PROVIDER_MAP).sort((a, b) => b.length - a.length); - for (const key of sorted) { - if (text.includes(key)) return key; - } - return null; -} - -function extractScenario(context: string): string { - const parenMatch = context.match(/\(([^)]+)\)/); - return parenMatch ? parenMatch[1] : context; -} - -function collectDriftEntries(results: VitestJsonResult): Array<{ - provider: string; - scenario: string; - builderFile: string; - builderFunctions: string[]; - typesFile: string | null; - sdkShapesFile: string; - diffs: ParsedDiff[]; -}> { - const entries: Array<{ - provider: string; - scenario: string; - builderFile: string; - builderFunctions: string[]; - typesFile: string | null; - sdkShapesFile: string; - diffs: ParsedDiff[]; - }> = []; - const unmapped: string[] = []; - let unparseable = 0; - - for (const file of results.testResults) { - for (const assertion of file.assertionResults) { - if (assertion.status !== "failed") continue; - if (assertion.failureMessages.length === 0) continue; - - const fullMessage = assertion.failureMessages.join("\n"); - const parsed = parseDriftBlock(fullMessage); - if (!parsed || parsed.diffs.length === 0) { - unparseable++; - continue; - } - - const ancestorText = assertion.ancestorTitles.join(" "); - const provider = extractProviderName(ancestorText) ?? extractProviderName(parsed.context); - if (!provider) { - unmapped.push(`${ancestorText} > ${assertion.title}`); - continue; - } - - const mapping = PROVIDER_MAP[provider]; - if (!mapping) { - unmapped.push(`${ancestorText} > ${assertion.title} (provider: ${provider})`); - continue; - } - - entries.push({ - provider, - scenario: extractScenario(parsed.context), - builderFile: mapping.builderFile, - builderFunctions: mapping.builderFunctions, - typesFile: mapping.typesFile, - sdkShapesFile: SDK_SHAPES_FILE, - diffs: parsed.diffs, - }); - } - } - - if (unmapped.length > 0) { - throw new Error(`${unmapped.length} unmapped drift entries — update PROVIDER_MAP`); - } - - if (unparseable > 0 && entries.length === 0) { - throw new Error(`${unparseable} unparseable test failures with 0 drift entries — investigate`); - } - - return entries; +interface VitestJsonResult { + testResults: { assertionResults: VitestAssertion[] }[]; } -// --------------------------------------------------------------------------- -// Helpers for building test fixtures -// --------------------------------------------------------------------------- - function makeResult(assertions: VitestAssertion[]): VitestJsonResult { return { testResults: [{ assertionResults: assertions }] }; } @@ -295,7 +110,87 @@ const SAMPLE_DIFF_WARNING: ShapeDiff = { }; // --------------------------------------------------------------------------- -// parseDriftBlock tests +// REAL captured vitest --reporter=json failure-message fixtures. +// Captured via throwaway `*.drift.ts` capture tests run under the drift config; +// see tmp/canary-fixtures.json + the PR #291 RED/GREEN logs. +// --------------------------------------------------------------------------- + +// Canary tripped with FOUR unknown models. The printed array is truncated by +// vitest to `…(3)` (single-glyph Unicode ellipsis), but the custom assertion +// message `UNKNOWN_REALTIME_MODELS=…` carries the full list verbatim. +// NOTE (A4): the ids below are HYPOTHETICAL future models that are NOT in the +// knownModels set in ws-realtime.drift.ts — so the real canary really could +// emit them as unknown. (gpt-realtime-2.1 / -2.1-mini ARE in knownModels and +// therefore can never appear here — the earlier fixture that used them was +// impossible.) +const CANARY_MARKER_MULTI = + "AssertionError: UNKNOWN_REALTIME_MODELS=gpt-realtime-3,gpt-realtime-3-mini,gpt-realtime-3-preview,gpt-realtime-ultra: expected [ 'gpt-realtime-3', …(3) ] to deeply equal []\n" + + " at /repo/src/__tests__/drift/ws-realtime.drift.ts:108:69\n" + + " at file:///repo/node_modules/@vitest/runner/dist/chunk-hooks.js:155:11\n" + + " at processTicksAndRejections (node:internal/process/task_queues:104:5)"; + +// A GENUINE drift report carried inside an AssertionError. formatDriftReport +// prepends "\n", so line 0 is just "AssertionError: " and the "API DRIFT +// DETECTED" / "mismatch" markers live on LATER lines. This is a fully-formatted +// report body (parseDriftBlock parses it into one critical diff). +const GENUINE_DRIFT_WITH_STACK = + "AssertionError: \nAPI DRIFT DETECTED: OpenAI Chat (non-streaming text)\n\n" + + " 1. [critical] LLMOCK DRIFT — mismatch detected\n" + + " Path: choices[0].message.refusal\n" + + " SDK: null\n" + + " Real: null\n" + + " Mock: \n" + + ": expected [ Array(1) ] to deeply equal []\n" + + " at /repo/src/__tests__/drift/openai-chat.drift.ts:42:30\n" + + " at file:///repo/node_modules/@vitest/runner/dist/chunk-hooks.js:155:11"; + +// A genuinely-unparseable failure whose ONLY infra token (ECONNREFUSED) sits in +// a STACK FRAME; the assertion body is neutral (no infra token, no drift +// marker). This is the A3 asymmetry surface — a raw scan would see the frame +// token and wrongly swallow the failure; a stack-stripped scan does not. +const INFRA_TOKEN_IN_STACKFRAME_ONLY = + "AssertionError: expected 1 to be 2 // Object.is equality\n" + + " at ECONNREFUSED (/repo/src/__tests__/drift/some.drift.ts:8:13)\n" + + " at file:///repo/node_modules/@vitest/runner/dist/chunk-hooks.js:155:11"; + +// A genuine infra failure whose token is in the BODY (the well-behaved case). +const REAL_INFRA_BODY = "fetch failed\n at handler (file:///repo/src/x.drift.ts:5:1)"; + +// CLASS 2 — the canary `hasGA`-false mode. Captured REAL from a throwaway +// `*.drift.ts` capture run under the drift config (see tmp/captured-vitest-shapes.json). +// When OpenAI renames/removes the GA realtime family, the canary emits the +// NO_GA_REALTIME_MODELS= marker (symmetric to UNKNOWN_REALTIME_MODELS=) and the +// assertion fails with "expected false to be true". The collector must map this +// to a CRITICAL OpenAI-Realtime DriftEntry (exit-2), NOT crash to exit-1. +const CANARY_NO_GA_MARKER = + "AssertionError: NO_GA_REALTIME_MODELS=gpt-foo,gpt-bar | UNKNOWN_REALTIME_MODELS=: expected false to be true // Object.is equality\n" + + " at /repo/src/__tests__/drift/ws-realtime.drift.ts:96:44\n" + + " at file:///repo/node_modules/@vitest/runner/dist/chunk-hooks.js:155:11\n" + + " at processTicksAndRejections (node:internal/process/task_queues:104:5)"; + +// CLASS 2 combined case — the run is BOTH no-GA AND carries new unknown models. +// The hasGA assertion throws first (short-circuiting the unknown-models +// assertion), so the NO_GA marker carries BOTH lists. The collector must +// preserve the unknown list (no information loss into the auto-fix prompt). +const CANARY_NO_GA_WITH_UNKNOWN = + "AssertionError: NO_GA_REALTIME_MODELS=gpt-foo,gpt-bar | UNKNOWN_REALTIME_MODELS=gpt-realtime-99,gpt-realtime-99-mini: expected false to be true // Object.is equality\n" + + " at /repo/src/__tests__/drift/ws-realtime.drift.ts:96:44\n" + + " at file:///repo/node_modules/@vitest/runner/dist/chunk-hooks.js:155:11"; + +// A NO_GA marker with an empty observed list (key could not see ANY realtime +// models — still a critical signal that the GA family is unreachable/gone). +const CANARY_NO_GA_EMPTY = + "AssertionError: NO_GA_REALTIME_MODELS= | UNKNOWN_REALTIME_MODELS=: expected false to be true // Object.is equality\n" + + " at /repo/src/__tests__/drift/ws-realtime.drift.ts:96:44"; + +// CLASS 3 — a marker-less truncated canary array. The truncation fact must +// become a boolean flag, never a prose sentinel occupying a model-id slot. +const CANARY_FALLBACK_TRUNCATED = + "AssertionError: expected [ 'gpt-realtime-9', …(2) ] to deeply equal []\n" + + " at /repo/src/__tests__/drift/ws-realtime.drift.ts:108:69"; + +// --------------------------------------------------------------------------- +// parseDriftBlock // --------------------------------------------------------------------------- describe("parseDriftBlock", () => { @@ -337,7 +232,6 @@ describe("parseDriftBlock", () => { }); it("skips entries with unknown severity", () => { - // Manually construct a report with a bad severity const text = ` API DRIFT DETECTED: OpenAI Chat (test) @@ -355,7 +249,6 @@ API DRIFT DETECTED: OpenAI Chat (test) `; const result = parseDriftBlock(text); expect(result).not.toBeNull(); - // Only the critical entry should be in diffs expect(result!.diffs).toHaveLength(1); expect(result!.diffs[0].severity).toBe("critical"); expect(result!.diffs[0].path).toBe("baz.qux"); @@ -401,7 +294,7 @@ API DRIFT DETECTED: OpenAI Chat (test) }); // --------------------------------------------------------------------------- -// extractProviderName tests +// extractProviderName // --------------------------------------------------------------------------- describe("extractProviderName", () => { @@ -412,7 +305,6 @@ describe("extractProviderName", () => { }); it("uses longest match — Anthropic Claude over Anthropic", () => { - // "Anthropic Claude" is longer and should win over "Anthropic" expect(extractProviderName("Anthropic Claude drift")).toBe("Anthropic Claude"); expect(extractProviderName("Anthropic Claude (streaming tool call)")).toBe("Anthropic Claude"); }); @@ -441,23 +333,44 @@ describe("extractProviderName", () => { }); // --------------------------------------------------------------------------- -// collectDriftEntries tests +// extractScenario +// --------------------------------------------------------------------------- + +describe("extractScenario", () => { + it("extracts the parenthetical scenario", () => { + expect(extractScenario("OpenAI Chat (non-streaming text)")).toBe("non-streaming text"); + expect(extractScenario("Anthropic Claude (streaming tool call)")).toBe("streaming tool call"); + }); + + it("returns the whole context when there is no parenthetical", () => { + expect(extractScenario("OpenAI Chat")).toBe("OpenAI Chat"); + }); +}); + +// --------------------------------------------------------------------------- +// collectDriftEntries (HTTP drift path) // --------------------------------------------------------------------------- describe("collectDriftEntries", () => { - it("returns empty array when no failed tests", () => { + it("returns empty entries+quarantine when no failed tests", () => { const result = makeResult([ makeAssertion({ status: "passed" }), makeAssertion({ status: "pending" }), ]); - expect(collectDriftEntries(result)).toEqual([]); + expect(entriesOf(result)).toEqual([]); + expect(quarantineOf(result)).toEqual([]); + expect(exitCodeOf(result)).toBe(0); }); - it("returns empty array when there are no test files at all", () => { - expect(collectDriftEntries({ testResults: [] })).toEqual([]); + it("returns empty entries+quarantine when there are no test files at all", () => { + const result: VitestJsonResult = { testResults: [] }; + expect(entriesOf(result)).toEqual([]); + expect(quarantineOf(result)).toEqual([]); }); - it("throws when an unmapped provider is found in drift report", () => { + it("QUARANTINES (does NOT throw) an unmapped provider found in a drift report → exit 5", () => { + // A1.3: an unmapped provider is untrusted, not a collector crash. It is held + // for review (exit 5) instead of throwing (was exit 1). const driftText = formatDriftReport("UnknownProvider (non-streaming text)", [SAMPLE_DIFF]); const result = makeResult([ makeAssertion({ @@ -466,21 +379,43 @@ describe("collectDriftEntries", () => { failureMessages: [driftText], }), ]); - expect(() => collectDriftEntries(result)).toThrow(/unmapped drift entries/); + expect(() => collectDriftEntries(result)).not.toThrow(); + const q = quarantineOf(result); + expect(q).toHaveLength(1); + expect(entriesOf(result)).toEqual([]); + expect(exitCodeOf(result)).toBe(5); }); - it("throws when all failures are unparseable and no drift entries collected", () => { + it("QUARANTINES (does NOT throw) all-unparseable-not-infra failures → exit 5 (incident-5)", () => { + // A1.3: the incident-5 surface. Genuine-but-unparseable failures are no + // longer a fail-loud crash (exit 1) — they are quarantined (exit 5) so they + // surface for review without being swallowed. const result = makeResult([ makeAssertion({ status: "failed", - failureMessages: ["Error: expected true to equal false\n at Object."], + ancestorTitles: ["Some Suite"], + title: "a", + failureMessages: [ + "AssertionError: expected 1 to be 2 // Object.is equality\n at foo (/repo/src/__tests__/drift/some.drift.ts:8:13)", + ], }), makeAssertion({ status: "failed", - failureMessages: ["TypeError: Cannot read property 'foo' of undefined"], + ancestorTitles: ["Other Suite"], + title: "b", + failureMessages: [ + "TypeError: Cannot read property 'foo' of undefined\n at bar (/repo/src/__tests__/drift/other.drift.ts:3:1)", + ], }), ]); - expect(() => collectDriftEntries(result)).toThrow(/unparseable test failures/); + expect(() => collectDriftEntries(result)).not.toThrow(); + const q = quarantineOf(result); + expect(q).toHaveLength(2); + // O-1: raw file:line captured from the stack frame BEFORE stripping. + expect(q[0].rawLocation).toBe("/repo/src/__tests__/drift/some.drift.ts:8:13"); + expect(q[1].rawLocation).toBe("/repo/src/__tests__/drift/other.drift.ts:3:1"); + expect(entriesOf(result)).toEqual([]); + expect(exitCodeOf(result)).toBe(5); }); it("returns valid entries and tolerates unparseable failures mixed in", () => { @@ -500,13 +435,19 @@ describe("collectDriftEntries", () => { }), ]); - const entries = collectDriftEntries(result); + const entries = entriesOf(result); expect(entries).toHaveLength(1); expect(entries[0].provider).toBe("OpenAI Chat"); expect(entries[0].scenario).toBe("non-streaming text"); expect(entries[0].builderFile).toBe("src/helpers.ts"); expect(entries[0].diffs).toHaveLength(1); expect(entries[0].diffs[0].severity).toBe("critical"); + // When real drift entries ARE present, a mixed-in unparseable sibling stays + // TOLERATED (warn-only) rather than quarantined — the quarantine path only + // fires for the all-unparseable, zero-entries case (former throw site). + expect(quarantineOf(result)).toEqual([]); + // A critical entry present → exit 2 (dominates any tolerated sibling). + expect(exitCodeOf(result)).toBe(2); }); it("ignores passed assertions in a mixed result set", () => { @@ -521,7 +462,7 @@ describe("collectDriftEntries", () => { }), ]); - const entries = collectDriftEntries(result); + const entries = entriesOf(result); expect(entries).toHaveLength(1); expect(entries[0].provider).toBe("OpenAI Chat"); }); @@ -555,9 +496,1254 @@ describe("collectDriftEntries", () => { ], }; - const entries = collectDriftEntries(results); + const entries = entriesOf(results); expect(entries).toHaveLength(2); expect(entries[0].provider).toBe("OpenAI Chat"); expect(entries[1].provider).toBe("Google Gemini"); }); + + // ------------------------------------------------------------------------- + // INTEGRATION: the canary → critical → exit-2 contract, end to end through + // the REAL collectDriftEntries. This is the whole reason PR #291 exists. + // ------------------------------------------------------------------------- + + it("emits ONE critical DriftEntry carrying the FULL unknown-model list from a real canary failure (RED without the marker fix)", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "canary: GA realtime models available", + failureMessages: [CANARY_MARKER_MULTI], + }), + ]); + + const entries = entriesOf(result); + expect(entries).toHaveLength(1); + + const entry = entries[0]; + expect(entry.provider).toBe("OpenAI Realtime"); + expect(entry.scenario).toBe("known-models canary"); + expect(entry.builderFile).toBe("src/ws-realtime.ts"); + + // FULL list recovered from the marker (NOT truncated to just the first id). + const reals = entry.diffs.map((d) => d.real); + expect(reals).toEqual([ + "gpt-realtime-3", + "gpt-realtime-3-mini", + "gpt-realtime-3-preview", + "gpt-realtime-ultra", + ]); + + // Every diff is critical so the collector exits 2 and the Fix Drift + // workflow reaches the auto-fix step. + expect(entry.diffs.every((d) => d.severity === "critical")).toBe(true); + // Real-API-only canary: the model id must NOT be mislabeled as a mock value. + for (const d of entry.diffs) { + expect(d.mock).not.toBe(d.real); + expect(d.mock).toContain("no mock leg"); + } + + // The exit-2 gate condition (criticalCount > 0) that main() checks. + const criticalCount = entries.reduce( + (sum, e) => sum + e.diffs.filter((d) => d.severity === "critical").length, + 0, + ); + expect(criticalCount).toBe(4); + }); + + it("does NOT misattribute a non-canary toEqual([]) failure from another provider as OpenAI-Realtime drift → quarantine (exit 5)", () => { + // A different provider's test failed with the generic vitest shape + // `expected [ 'sk-leaked' ] to deeply equal []`. Pre-fix, the unguarded + // canary fallback matched this and emitted a CRITICAL "OpenAI Realtime + // known-models canary" entry with `real: 'sk-leaked'`, pointing the auto-fix + // at src/ws-realtime.ts and relabeling arbitrary array contents as a model + // id. It must NOT be claimed as a canary. A1.3: because the message is + // neither a parseable drift block, a canary, nor infra, it is QUARANTINED + // (exit 5) — never fabricated into a false entry, never silently dropped, + // and (A1.3) no longer a fail-loud crash. + const NON_CANARY_TOEQUAL_EMPTY = + "AssertionError: expected [ 'sk-leaked' ] to deeply equal []\n" + + " at /repo/src/__tests__/drift/openai-chat.drift.ts:42:30\n" + + " at file:///repo/node_modules/@vitest/runner/dist/chunk-hooks.js:155:11"; + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + title: "non-streaming text matches real API", + failureMessages: [NON_CANARY_TOEQUAL_EMPTY], + }), + ]); + + expect(() => collectDriftEntries(result)).not.toThrow(); + // No OpenAI-Realtime (nor any) entry may be produced. + expect(entriesOf(result)).toEqual([]); + expect(quarantineOf(result)).toHaveLength(1); + // The arbitrary array content is NOT relabeled as a model id anywhere. + expect(quarantineOf(result)[0].message).toContain("sk-leaked"); + expect(exitCodeOf(result)).toBe(5); + }); + + it("surfaces a genuine drift report carried in an AssertionError with a leading blank line (does not swallow)", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + title: "non-streaming text matches real API", + failureMessages: [GENUINE_DRIFT_WITH_STACK], + }), + ]); + + const entries = entriesOf(result); + expect(entries).toHaveLength(1); + expect(entries[0].provider).toBe("OpenAI Chat"); + expect(entries[0].diffs).toHaveLength(1); + expect(entries[0].diffs[0].severity).toBe("critical"); + }); + + it("does NOT exit 0 (quarantines → exit 5) when the only failure has an infra token confined to a stack frame (A3)", () => { + // The raw-vs-stripped asymmetry classified this as benign infra and swallowed + // it (returned []). The fix normalizes both scans, so an infra token that + // survives ONLY in a stripped-away stack frame no longer flips the gate — the + // failure is surfaced. A1.3: surfaced now means QUARANTINE (exit 5), not a + // crash; the invariant that matters is it is NEVER a green (exit 0). + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "canary: GA realtime models available", + failureMessages: [INFRA_TOKEN_IN_STACKFRAME_ONLY], + }), + ]); + expect(() => collectDriftEntries(result)).not.toThrow(); + expect(quarantineOf(result)).toHaveLength(1); + expect(exitCodeOf(result)).toBe(5); + expect(exitCodeOf(result)).not.toBe(0); + }); + + // ------------------------------------------------------------------------- + // CLASS 2 — hasGA-false canary maps to a CRITICAL OpenAI-Realtime entry + // (exit-2 path) instead of crashing to exit-1. + // ------------------------------------------------------------------------- + it("maps a NO_GA_REALTIME_MODELS canary failure to a CRITICAL OpenAI-Realtime entry (exit-2, not a throw)", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "canary: GA realtime models available", + failureMessages: [CANARY_NO_GA_MARKER], + }), + ]); + + const entries = entriesOf(result); + expect(entries).toHaveLength(1); + const entry = entries[0]; + expect(entry.provider).toBe("OpenAI Realtime"); + expect(entry.builderFile).toBe("src/ws-realtime.ts"); + expect(entry.diffs.length).toBeGreaterThan(0); + expect(entry.diffs.every((d) => d.severity === "critical")).toBe(true); + + const criticalCount = entries.reduce( + (sum, e) => sum + e.diffs.filter((d) => d.severity === "critical").length, + 0, + ); + expect(criticalCount).toBeGreaterThan(0); + }); + + it("preserves the unknown-model list in the NO_GA entry (no info loss when both fire)", () => { + // A run that is BOTH no-GA AND has new unknown models must surface the + // unknown ids as critical diffs, not lose them to the short-circuited + // unknown-models assertion. + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "canary: GA realtime models available", + failureMessages: [CANARY_NO_GA_WITH_UNKNOWN], + }), + ]); + const entries = entriesOf(result); + expect(entries).toHaveLength(1); + const entry = entries[0]; + expect(entry.provider).toBe("OpenAI Realtime"); + expect(entry.diffs.every((d) => d.severity === "critical")).toBe(true); + // The unknown model ids survive as `real` values on knownModels diffs. + const knownModelReals = entry.diffs.filter((d) => d.path === "knownModels").map((d) => d.real); + expect(knownModelReals).toEqual(["gpt-realtime-99", "gpt-realtime-99-mini"]); + // The GA-family diff is still present. + expect(entry.diffs.some((d) => d.path === "gaModels")).toBe(true); + }); + + it("maps an EMPTY NO_GA marker (no realtime models observed) to a CRITICAL entry too", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "canary: GA realtime models available", + failureMessages: [CANARY_NO_GA_EMPTY], + }), + ]); + const entries = entriesOf(result); + expect(entries).toHaveLength(1); + expect(entries[0].provider).toBe("OpenAI Realtime"); + expect(entries[0].diffs.every((d) => d.severity === "critical")).toBe(true); + }); + + // ------------------------------------------------------------------------- + // CLASS 3 — no DriftEntry.real is a non-model prose sentinel, even when the + // canary array was truncated in CI output. + // ------------------------------------------------------------------------- + it("never lands a prose sentinel in DriftEntry.real when the canary array is truncated (CLASS 3)", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "canary: GA realtime models available", + failureMessages: [CANARY_FALLBACK_TRUNCATED], + }), + ]); + const entries = entriesOf(result); + expect(entries).toHaveLength(1); + for (const d of entries[0].diffs) { + // No `real` value may be a prose annotation (e.g. "(additional models…)"). + expect(d.real.startsWith("(")).toBe(false); + } + }); + + // ------------------------------------------------------------------------- + // CLASS 1 — corpus/table test asserting the SAFE outcome for the recurring + // classifier failure shapes. A1.3 replaces the old binary throws/no-throw + // with a three-way `outcome`: + // "entry" → surfaced as a structured drift entry (exit 2), NEVER a + // silent exit-0; + // "quarantine" → held for human review (exit 5), NEVER a crash and NEVER a + // silent exit-0 (was: fail-loud throw / exit 1); + // "infra" → benign infra, collector returns [] (exit 0). + // The invariant the corpus protects: an untrusted failure is never a green. + // ------------------------------------------------------------------------- + describe("CLASS 1 safe-outcome corpus", () => { + const DRIFT_VALUE_WITH_STATUS_200 = formatDriftReport("OpenAI Chat (non-streaming text)", [ + { + path: "choices[0].message.content", + severity: "critical", + issue: "LLMOCK DRIFT — value mismatch", + expected: "status 200", + real: "status 200", + mock: "", + }, + ]); + + const rows: { + name: string; + messages: string[]; + outcome: "entry" | "quarantine" | "infra"; + }[] = [ + { + name: "a drift body containing the substring 'status 200' is surfaced as an entry, NOT swallowed as infra", + messages: [DRIFT_VALUE_WITH_STATUS_200], + outcome: "entry", + }, + { + name: "a genuine drift report with a leading blank line is surfaced as an entry", + messages: [GENUINE_DRIFT_WITH_STACK], + outcome: "entry", + }, + { + name: "an 'expected false to be true' hasGA shape is surfaced as a (canary) entry, not swallowed", + messages: [CANARY_NO_GA_MARKER], + outcome: "entry", + }, + { + name: "an 'expected […] to deeply equal []' canary shape is surfaced as an entry, not swallowed", + messages: [CANARY_MARKER_MULTI], + outcome: "entry", + }, + { + name: "a bare AssertionError with no infra token and no drift marker is quarantined (exit 5), not swallowed", + messages: ["AssertionError: expected 1 to be 2 // Object.is equality\n at foo (x:1:1)"], + outcome: "quarantine", + }, + { + name: "an infra token confined to a stack frame is quarantined (exit 5) (A3)", + messages: [INFRA_TOKEN_IN_STACKFRAME_ONLY], + outcome: "quarantine", + }, + ]; + + for (const row of rows) { + it(row.name, () => { + const result = makeResult( + row.messages.map((m) => + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "canary: GA realtime models available", + failureMessages: [m], + }), + ), + ); + expect(() => collectDriftEntries(result)).not.toThrow(); + const { entries, quarantine } = collectDriftEntries(result); + if (row.outcome === "entry") { + expect(entries.length).toBeGreaterThan(0); + expect(exitCodeOf(result)).toBe(2); + } else if (row.outcome === "quarantine") { + expect(entries).toEqual([]); + expect(quarantine.length).toBeGreaterThan(0); + expect(exitCodeOf(result)).toBe(5); + expect(exitCodeOf(result)).not.toBe(0); + } else { + expect(entries).toEqual([]); + expect(quarantine).toEqual([]); + expect(exitCodeOf(result)).toBe(0); + } + }); + } + + it("still classifies genuine infra (body token) as benign — collector returns [] entries+quarantine (exit 0)", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + title: "non-streaming text matches real API", + failureMessages: [REAL_INFRA_BODY], + }), + ]); + expect(entriesOf(result)).toEqual([]); + expect(quarantineOf(result)).toEqual([]); + expect(exitCodeOf(result)).toBe(0); + }); + }); + + // ------------------------------------------------------------------------- + // A1.4 TAXONOMY — the exit-code taxonomy end-to-end through the REAL collector + // + computeExitCode. Each row asserts the full path from a vitest failure + // shape to the process exit code main() would emit. + // ------------------------------------------------------------------------- + describe("exit-code taxonomy (collector → computeExitCode)", () => { + it("critical drift → exit 2", () => { + const driftText = formatDriftReport("OpenAI Chat (non-streaming text)", [SAMPLE_DIFF]); + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + failureMessages: [driftText], + }), + ]); + expect(entriesOf(result)).toHaveLength(1); + expect(quarantineOf(result)).toEqual([]); + expect(exitCodeOf(result)).toBe(2); + }); + + it("incident-5 unparseable failure → quarantine + exit 5 (NOT a throw)", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["Broken Suite"], + title: "a", + failureMessages: [ + "AssertionError: expected 1 to be 2 // Object.is equality\n at foo (/repo/src/__tests__/drift/x.drift.ts:8:13)", + ], + }), + ]); + expect(() => collectDriftEntries(result)).not.toThrow(); + expect(entriesOf(result)).toEqual([]); + expect(quarantineOf(result)).toHaveLength(1); + expect(exitCodeOf(result)).toBe(5); + }); + + it("all-infra failures → exit 0 (benign, collector returns nothing)", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + failureMessages: [REAL_INFRA_BODY], + }), + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Responses drift"], + failureMessages: [ + "INFRA_ERROR: upstream down\n at h (file:///repo/src/y.drift.ts:1:1)", + ], + }), + ]); + expect(entriesOf(result)).toEqual([]); + expect(quarantineOf(result)).toEqual([]); + expect(exitCodeOf(result)).toBe(0); + }); + + it("canary (unknown-model) failure → critical entry + exit 2", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "canary: GA realtime models available", + failureMessages: [CANARY_MARKER_MULTI], + }), + ]); + const entries = entriesOf(result); + expect(entries).toHaveLength(1); + expect(entries[0].diffs.every((d) => d.severity === "critical")).toBe(true); + expect(exitCodeOf(result)).toBe(2); + }); + + it("empty → fail-loud invariant: an all-unparseable batch is NEVER classified as a benign all-clear", () => { + // CLASS 1 root invariant surfaced at the collector: an empty evidence set + // (no positive infra evidence) must NOT be treated as infra. The batch is + // quarantined (exit 5), never a silent exit 0. + expect(classifyUnparseableAsInfra([])).toBe(false); + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["Broken Suite"], + title: "unrecognized", + failureMessages: [ + "AssertionError: expected 1 to be 2\n at foo (/repo/src/z.drift.ts:1:1)", + ], + }), + ]); + expect(exitCodeOf(result)).not.toBe(0); + expect(exitCodeOf(result)).toBe(5); + }); + + it("critical + quarantine together → exit 2 (critical dominates quarantine)", () => { + const driftText = formatDriftReport("OpenAI Chat (non-streaming text)", [SAMPLE_DIFF]); + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + title: "non-streaming text matches real API", + failureMessages: [driftText], + }), + // An unmapped-provider drift block → quarantined (never dropped) even + // though a real critical entry is also present. + makeAssertion({ + status: "failed", + ancestorTitles: ["UnknownProvider drift"], + title: "some scenario", + failureMessages: [formatDriftReport("UnknownProvider (streaming text)", [SAMPLE_DIFF])], + }), + ]); + const { entries, quarantine } = collectDriftEntries(result); + expect(entries).toHaveLength(1); + expect(entries[0].provider).toBe("OpenAI Chat"); + expect(quarantine).toHaveLength(1); + // Critical dominates: exit 2, not 5. + expect(exitCodeOf(result)).toBe(2); + }); + }); +}); + +// --------------------------------------------------------------------------- +// parseKnownModelsCanary +// --------------------------------------------------------------------------- + +describe("parseKnownModelsCanary", () => { + it("recovers the FULL unknown-model list from the UNKNOWN_REALTIME_MODELS marker (not truncated)", () => { + // The printed array is truncated to `…(3)` but the marker carries all four. + const result = parseKnownModelsCanary(CANARY_MARKER_MULTI); + expect(result).not.toBeNull(); + expect(result!.ids).toEqual([ + "gpt-realtime-3", + "gpt-realtime-3-mini", + "gpt-realtime-3-preview", + "gpt-realtime-ultra", + ]); + // CLASS 3: the marker carries the full list, so nothing is truncated and no + // prose sentinel may ever occupy an id slot. + expect(result!.truncated).toBeFalsy(); + expect(result!.ids.every((id) => !id.startsWith("("))).toBe(true); + }); + + it("returns null when the marker is present but the unknown list was empty", () => { + // Empty unknown list = no drift to surface. + const msg = "AssertionError: UNKNOWN_REALTIME_MODELS=: expected [] to deeply equal []"; + expect(parseKnownModelsCanary(msg)).toBeNull(); + }); + + it("falls through to the printed-array fallback when the marker value is mangled/empty (A2)", () => { + // A2: an empty/mangled marker must NOT short-circuit to null; it must fall + // through so a recoverable id in the printed array is still surfaced. + const msg = + "AssertionError: UNKNOWN_REALTIME_MODELS=: expected [ 'gpt-realtime-3', …(1) ] to deeply equal []"; + const result = parseKnownModelsCanary(msg); + expect(result).not.toBeNull(); + expect(result!.ids[0]).toBe("gpt-realtime-3"); + // CLASS 3: truncation is a boolean flag, NOT a prose id in the list. + expect(result!.truncated).toBe(true); + expect(result!.ids.every((id) => !id.startsWith("("))).toBe(true); + }); + + it("returns null for a non-canary message", () => { + expect(parseKnownModelsCanary("TypeError: something unrelated")).toBeNull(); + expect(parseKnownModelsCanary("")).toBeNull(); + }); + + describe("fallback (no marker — legacy message shape)", () => { + // NOTE: the fallback fires ONLY in a confirmed ws-realtime canary context. + // A REAL marker-less canary failure ALWAYS carries the canary's origin frame + // (`at …/ws-realtime.drift.ts`), which these fixtures include — that frame + // is the recognizer that distinguishes a genuine canary from a generic + // non-canary `toEqual([])` failure in some other provider's test. + const CANARY_ORIGIN = "\n at /repo/src/__tests__/drift/ws-realtime.drift.ts:108:69"; + + it("detects the single-glyph Unicode ellipsis `…(1)` truncation (as a flag, not a sentinel id)", () => { + const msg = + "AssertionError: expected [ 'gpt-realtime-3', …(1) ] to deeply equal []" + CANARY_ORIGIN; + const result = parseKnownModelsCanary(msg); + expect(result).not.toBeNull(); + expect(result!.ids[0]).toBe("gpt-realtime-3"); + // CLASS 3: no prose sentinel in the id list; truncation is a boolean. + expect(result!.truncated).toBe(true); + expect(result!.ids.every((id) => !id.startsWith("("))).toBe(true); + }); + + it("also detects the three-dot ASCII ellipsis `... (1)` truncation", () => { + const msg = + "AssertionError: expected [ 'gpt-realtime-3', ... (1) ] to deeply equal []" + CANARY_ORIGIN; + const result = parseKnownModelsCanary(msg); + expect(result!.truncated).toBe(true); + expect(result!.ids.every((id) => !id.startsWith("("))).toBe(true); + }); + + it("parses a small untruncated printed array", () => { + const msg = + "AssertionError: expected [ 'gpt-realtime-3', 'gpt-realtime-3-mini' ] to deeply equal []" + + CANARY_ORIGIN; + const result = parseKnownModelsCanary(msg); + expect(result!.ids).toEqual(["gpt-realtime-3", "gpt-realtime-3-mini"]); + expect(result!.truncated).toBeFalsy(); + }); + + it("returns null for an empty printed array (genuinely no unknown models)", () => { + const msg = "AssertionError: expected [] to deeply equal []"; + expect(parseKnownModelsCanary(msg)).toBeNull(); + }); + + it("flags truncation-only content (glyph present, no extractable id) without inventing a prose id", () => { + // Inner had a truncation glyph but no quoted ids we could extract. Carries + // the ws-realtime canary origin path so the fallback gate recognizes it as + // a genuine canary failure (a real canary failure ALWAYS carries this + // frame). Without a canary-origin token the fallback must NOT fire. + const msg = + "AssertionError: expected [ …(4) ] to deeply equal []\n" + + " at /repo/src/__tests__/drift/ws-realtime.drift.ts:108:69"; + const result = parseKnownModelsCanary(msg); + expect(result).not.toBeNull(); + // CLASS 3: no non-model prose id — the fact lives entirely in `truncated`. + expect(result!.ids.every((id) => !id.startsWith("("))).toBe(true); + expect(result!.truncated).toBe(true); + }); + }); + + // ------------------------------------------------------------------------- + // MISATTRIBUTION GUARD (bucket (a) finding): the printed-array fallback must + // fire ONLY for a genuine ws-realtime known-models canary failure. A generic + // `expected [...] to deeply equal []` from ANY OTHER provider/test (no + // realtime-canary marker AND not originating from ws-realtime.drift.ts) must + // NOT be claimed as OpenAI-Realtime known-models drift — its arbitrary array + // contents (which could be a leaked secret, an object shape, anything) must + // never be relabeled as "unknown model ids". + // ------------------------------------------------------------------------- + describe("fallback gating — non-canary toEqual([]) is NOT misattributed", () => { + it("returns null for a non-canary toEqual([]) failure carrying arbitrary array contents (RED before gate)", () => { + // A DIFFERENT provider's test asserted `toEqual([])` and the array held an + // arbitrary value — here a leaked-looking secret. NO realtime-canary marker + // and NO ws-realtime.drift.ts origin: this is not the canary and must not + // be parsed as one. + const msg = + "AssertionError: expected [ 'sk-leaked' ] to deeply equal []\n" + + " at /repo/src/__tests__/drift/openai-chat.drift.ts:42:30"; + expect(parseKnownModelsCanary(msg)).toBeNull(); + }); + + it("returns null for a bare non-canary toEqual([]) failure with no origin frame at all", () => { + const msg = "AssertionError: expected [ 'sk-leaked' ] to deeply equal []"; + expect(parseKnownModelsCanary(msg)).toBeNull(); + }); + + it("still parses a genuine marker-less canary failure that carries the ws-realtime.drift.ts origin", () => { + // No structured marker (mangled/stripped), but the stack frame identifies + // the canary — the fallback SHOULD still recover the id. + const result = parseKnownModelsCanary(CANARY_FALLBACK_TRUNCATED); + expect(result).not.toBeNull(); + expect(result!.ids[0]).toBe("gpt-realtime-9"); + expect(result!.truncated).toBe(true); + }); + }); + + // ------------------------------------------------------------------------- + // CLASS 2 — the NO_GA_REALTIME_MODELS marker (hasGA-false mode) + // ------------------------------------------------------------------------- + describe("NO_GA_REALTIME_MODELS marker (hasGA-false)", () => { + it("recognizes the marker and returns the observed model ids with noGA=true", () => { + const result = parseKnownModelsCanary(CANARY_NO_GA_MARKER); + expect(result).not.toBeNull(); + expect(result!.noGA).toBe(true); + expect(result!.ids).toEqual(["gpt-foo", "gpt-bar"]); + }); + + it("recognizes an EMPTY NO_GA marker (no realtime models observed at all) as noGA=true", () => { + const result = parseKnownModelsCanary(CANARY_NO_GA_EMPTY); + expect(result).not.toBeNull(); + expect(result!.noGA).toBe(true); + expect(result!.ids).toEqual([]); + expect(result!.unknownIds ?? []).toEqual([]); + }); + + it("preserves the unknown-model list carried alongside the NO_GA marker (info-loss fix)", () => { + // Combined case: no-GA AND new unknown models. The hasGA assertion + // short-circuits the unknown-models assertion, so the NO_GA marker carries + // BOTH lists. The observed and unknown lists must be split cleanly. + const result = parseKnownModelsCanary(CANARY_NO_GA_WITH_UNKNOWN); + expect(result).not.toBeNull(); + expect(result!.noGA).toBe(true); + expect(result!.ids).toEqual(["gpt-foo", "gpt-bar"]); + expect(result!.unknownIds).toEqual(["gpt-realtime-99", "gpt-realtime-99-mini"]); + }); + }); +}); + +// --------------------------------------------------------------------------- +// classifyUnparseableAsInfra (A3 — symmetric normalization safety net) +// --------------------------------------------------------------------------- + +describe("classifyUnparseableAsInfra", () => { + it("returns false for an EMPTY evidence array — no evidence is NOT proof of infra (CLASS 1)", () => { + // Vacuous `.every` on [] returns true; the fix must NOT treat "no evidence" + // as "all clear". Unrecognized ⇒ fail loud, never a false all-clear. + expect(classifyUnparseableAsInfra([])).toBe(false); + }); + + it("does NOT swallow a failure whose only infra token is confined to a stack frame (A3)", () => { + // Pre-fix: the raw scan saw ECONNREFUSED in the frame → allInfraErrors true + // → swallowed. The fix strips frames for BOTH scans, so the token is gone + // and the failure is not classified as infra. + expect(classifyUnparseableAsInfra([INFRA_TOKEN_IN_STACKFRAME_ONLY])).toBe(false); + }); + + it("does NOT swallow genuine drift carried in an AssertionError with a leading blank line", () => { + expect(classifyUnparseableAsInfra([GENUINE_DRIFT_WITH_STACK])).toBe(false); + }); + + it("does NOT treat a bare AssertionError as benign infra", () => { + const msg = "AssertionError: expected [ 'x' ] to deeply equal []\n at foo (file:///x)"; + expect(classifyUnparseableAsInfra([msg])).toBe(false); + }); + + it("still classifies genuine infra errors (token in the body) as infra", () => { + expect(classifyUnparseableAsInfra([REAL_INFRA_BODY])).toBe(true); + expect(classifyUnparseableAsInfra(["INFRA_ERROR: upstream down\n at foo (file:///x)"])).toBe( + true, + ); + expect(classifyUnparseableAsInfra(["API returned 503 Service Unavailable"])).toBe(true); + }); + + it("does NOT classify a drift body whose VALUE contains 'status 200' as infra (CLASS 1 anchoring)", () => { + // A real drift value like "status 200" appearing anywhere in the body must + // not trip the infra gate. The infra 'status \\d{3}' indicator must anchor + // to the failure reason/line, not a bare substring inside a drift value. + const msg = + "AssertionError: \nAPI DRIFT DETECTED: OpenAI Chat (non-streaming text)\n\n" + + " 1. [critical] LLMOCK DRIFT — mismatch detected\n" + + " Path: choices[0].message.content\n" + + " SDK: status 200\n" + + " Real: status 200\n" + + " Mock: \n"; + expect(classifyUnparseableAsInfra([msg])).toBe(false); + }); + + it("does NOT classify a labelled 'Real: API returned 503' drift VALUE as infra (CLASS 1 anchoring)", () => { + // Symmetric to the 'status 200' anchoring case above, and to the already- + // anchored 'status \\d{3}' sibling. A drift *value* like "API returned 503" + // appearing AFTER a `Field:` label must NOT trip the infra gate. The + // 'API returned \\d{3}' indicator must anchor to the failure reason/line + // (line start, optional `HTTP ` prefix) exactly like 'status \\d{3}' does — + // an anchoring-defeating `(?:.*:\\s*)?` prefix lets a labelled value match + // and silently swallow genuine drift. This message is deliberately NOT + // drift-like (no "drift"/"mismatch"/"expected…to" markers) so the + // anchoring of the infra indicator is the SOLE determinant of the outcome. + const msg = + " Path: choices[0].message.content\n" + + " SDK: n/a\n" + + " Real: API returned 503\n" + + " Mock: \n"; + expect(classifyUnparseableAsInfra([msg])).toBe(false); + }); + + it("still classifies a bare line-start 'API returned 503' reason as infra (anchoring preserved)", () => { + // The anchoring fix must NOT break the genuine infra case: a line whose + // reason IS "API returned " (optionally `HTTP `-prefixed, at line + // start) is still infra. Guards against over-tightening the anchor. + expect(classifyUnparseableAsInfra(["API returned 503 Service Unavailable"])).toBe(true); + expect(classifyUnparseableAsInfra([" HTTP API returned 500"])).toBe(true); + }); + + it("does not false-positive drift from a stack-trace filename like ws-realtime.drift.ts", () => { + // A recognized infra error (token in BODY) whose stack frame mentions + // "ws-realtime.drift.ts" stays infra — the frame filename is stripped. + const msg = "fetch failed\n at handler (file:///repo/src/ws-realtime.drift.ts:5:1)"; + expect(classifyUnparseableAsInfra([msg])).toBe(true); + }); + + // ------------------------------------------------------------------------- + // PROPERTY-BASED uniform-anchoring test — the NON-RECURRING deliverable. + // + // Iterates the REAL exported infra-indicator list (INFRA_INDICATOR_SOURCES), + // NOT a hand-copied subset. For EVERY indicator it asserts through the REAL + // exported classifyUnparseableAsInfra that: + // (a) a labelled drift-body line `Real: ` (no drift marker) is + // NOT classified as infra — genuine drift is surfaced/fail-loud; and + // (b) a bare line-start `` failure reason IS classified as infra. + // + // This is what makes the class non-recurring: if a future indicator is added + // to INFRA_INDICATOR_SPECS but individually mis-anchored (e.g. with the + // old `(?:.*:\s*)?` prefix or an unanchored `/i`), row (a) fails automatically + // for that indicator — no one has to remember to add a bespoke test. + // + // RED before the uniform-anchoring fix: at minimum the `empty response`, + // `returned no SSE events`, and `returned empty body` rows fail case (a) + // (they were `(?:.*:\s*)?`-prefixed or unanchored, so a labelled value matched + // and swallowed genuine drift). GREEN after: all rows pass both cases. + // ------------------------------------------------------------------------- + describe("uniform anchoring across the REAL infra-indicator list (property)", () => { + it("exports a non-empty indicator list to iterate", () => { + expect(INFRA_INDICATOR_SOURCES.length).toBeGreaterThan(0); + }); + + for (const source of INFRA_INDICATOR_SOURCES) { + const sample = infraIndicatorSample(source); + + it(`[${source}] a labelled drift-body value "Real: ${sample}" is NOT swallowed as infra (a)`, () => { + // A labelled body line carrying the phrase as a drift VALUE. No drift + // marker present, so the infra-indicator anchoring is the SOLE + // determinant: if the indicator is properly line-anchored it does NOT + // match here (the phrase follows a `Real:` label), so the batch is not + // all-infra and the failure is surfaced (classify → false). + const msg = + " Path: choices[0].message.content\n" + + " SDK: n/a\n" + + ` Real: ${sample}\n` + + " Mock: \n"; + expect(classifyUnparseableAsInfra([msg])).toBe(false); + }); + + it(`[${source}] a bare line-start "${sample}" failure reason IS classified as infra (b)`, () => { + // The phrase AS the failure reason at line start must still be infra — + // the anchoring fix must not over-tighten and break genuine infra. + expect(classifyUnparseableAsInfra([sample])).toBe(true); + }); + + it(`[${source}] taxonomy (c): a bare "${sample}" reason → collector exit 0 (benign, no quarantine)`, () => { + // A1.4 extension: tie the infra classification to the exit-code taxonomy + // at the REAL collector surface. A bare infra-reason failure must be a + // benign exit 0 — NOT quarantined (exit 5) and NOT a crash. + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + title: "non-streaming text matches real API", + failureMessages: [sample], + }), + ]); + expect(() => collectDriftEntries(result)).not.toThrow(); + expect(entriesOf(result)).toEqual([]); + expect(quarantineOf(result)).toEqual([]); + expect(exitCodeOf(result)).toBe(0); + }); + + it(`[${source}] taxonomy (c'): a labelled "Real: ${sample}" drift value → NOT exit 0 (quarantined, exit 5)`, () => { + // Symmetric to (a) at the collector surface: a labelled body value that + // merely CONTAINS the infra phrase must never be swallowed as a green. + // It is not a full parseable drift block, so it is quarantined (exit 5). + const msg = + " Path: choices[0].message.content\n" + + " SDK: n/a\n" + + ` Real: ${sample}\n` + + " Mock: \n"; + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Chat Completions drift"], + title: "non-streaming text matches real API", + failureMessages: [msg], + }), + ]); + expect(exitCodeOf(result)).not.toBe(0); + expect(exitCodeOf(result)).toBe(5); + }); + } + }); +}); + +// --------------------------------------------------------------------------- +// D6.2 — per-item `id` field on ParsedDiff +// +// The delta layer (D6.1) keys findings by `provider+id`. For N distinct unknown +// model ids, the collector must produce N DISTINCT per-item `id` values so that +// a downstream `provider+id` keying yields N distinct keys — not 1 collapsed +// key under `path:"knownModels"` (pre-fix behaviour when `id` was absent/undefined). +// +// RED (pre-fix): `id` is unset on every ParsedDiff produced by the canary path, +// so all 3 diffs have `id === undefined` → only 1 distinct key. +// GREEN (post-fix): each diff carries `id` = the model id stored in `diff.real`, +// so 3 distinct unknown ids → 3 distinct `id` values → 3 distinct keys. +// --------------------------------------------------------------------------- + +describe("D6.2 — per-item id on ParsedDiff", () => { + // Three distinct hypothetical unknown model ids (not in the knownModels set + // in ws-realtime.drift.ts — A4 note: use future/hypothetical ids only). + const THREE_UNKNOWN_IDS_CANARY = + "AssertionError: UNKNOWN_REALTIME_MODELS=gpt-realtime-x1,gpt-realtime-x2,gpt-realtime-x3: " + + "expected [ 'gpt-realtime-x1', …(2) ] to deeply equal []\n" + + " at /repo/src/__tests__/drift/ws-realtime.drift.ts:108:69"; + + it("D6.2 RED→GREEN: 3 distinct canary model ids produce 3 DISTINCT per-item id fields (not collapsed under undefined)", () => { + const result = makeResult([ + makeAssertion({ + status: "failed", + ancestorTitles: ["OpenAI Realtime API drift"], + title: "canary: GA realtime models available", + failureMessages: [THREE_UNKNOWN_IDS_CANARY], + }), + ]); + + const entries = entriesOf(result); + expect(entries).toHaveLength(1); + const entry = entries[0]; + expect(entry.provider).toBe("OpenAI Realtime"); + + // There should be exactly 3 diffs — one per unknown model id. + expect(entry.diffs).toHaveLength(3); + + // D6.2 core assertion: each diff must carry a populated `id` field equal to + // the model id in `diff.real`, and all three must be distinct. + const ids = entry.diffs.map((d) => d.id); + expect(ids).toEqual(["gpt-realtime-x1", "gpt-realtime-x2", "gpt-realtime-x3"]); + + // All three ids are defined (not undefined). + expect(ids.every((id) => id !== undefined)).toBe(true); + + // All three ids are DISTINCT — a downstream provider+id key would yield 3 + // different keys, not 1 collapsed key under undefined/absent id. + const distinctIds = new Set(ids); + expect(distinctIds.size).toBe(3); + + // The model ids must match what's in `diff.real` (the source of truth). + for (const diff of entry.diffs) { + expect(diff.id).toBe(diff.real); + } + }); + + it("D6.2: parseDriftBlock-path diffs carry a stable id derived from path", () => { + // For regular drift-block diffs (not canary), `id` is derived from + // the `path` field so different paths → different ids. + const formatted = formatDriftReport("OpenAI Chat (non-streaming text)", [ + { ...SAMPLE_DIFF, path: "choices[0].message.refusal" }, + { ...SAMPLE_DIFF, path: "choices[0].message.content", severity: "warning" as const }, + ]); + const parsed = parseDriftBlock(formatted); + expect(parsed).not.toBeNull(); + expect(parsed!.diffs).toHaveLength(2); + + const ids = parsed!.diffs.map((d) => d.id); + // Both diffs must have a non-empty id. + expect(ids.every((id) => id !== undefined && id !== "")).toBe(true); + // The two paths are different → two distinct ids. + expect(new Set(ids).size).toBe(2); + // Each id must be derived from (or equal to) the path. + for (const diff of parsed!.diffs) { + expect(diff.id).toBe(diff.path); + } + }); +}); + +// =========================================================================== +// WS-5 — structural surface keying via SURFACE_REGISTRY +// =========================================================================== + +describe("WS-5 extractSurfaceKey", () => { + it("reads the Surface: marker line emitted by formatDriftReport(surface)", () => { + const block = formatDriftReport( + "Cohere /v2/chat (non-streaming)", + [SAMPLE_DIFF], + "cohere-chat", + ); + expect(extractSurfaceKey(block)).toBe("cohere-chat"); + }); + + it("returns null for a legacy block with no Surface: marker", () => { + const block = formatDriftReport("Cohere /v2/chat (non-streaming)", [SAMPLE_DIFF]); + expect(extractSurfaceKey(block)).toBeNull(); + }); +}); + +describe("WS-5 — previously-quarantined surfaces now route to exit-2 entries", () => { + // A drift block for a surface that today is NOT a PROVIDER_MAP key. On old + // code these route to a quarantine (exit 5) because extractProviderName + // returns null. With the surface marker + registry they become auto-fixable + // exit-2 entries. + // + // CRITICAL — each title below is NEUTRAL PROSE that contains NO registry + // provider label (nor a legacy alias) as a substring, in either the ancestor + // title OR the emitted context. That is deliberate: it means the legacy + // `extractProviderName` fallback returns null for every one of these, so the + // ONLY thing that can route them to an entry is the `Surface:` marker seam. If + // the WS-5 seam is reverted, ALL of these go RED (verified). This closes the + // F1 gap where fal/elevenlabs stayed green on revert because their titles + // happened to contain the legacy label substring. (≥3 cells; 4 for margin.) + const CASES: { surface: string; title: string; provider: string; builderFile: string }[] = [ + { + surface: "moderation", + title: "content-safety endpoint 400 payload", + provider: "OpenAI Moderations", + builderFile: "src/moderation.ts", + }, + { + surface: "video", + title: "async media generation status poll", + provider: "OpenAI Video", + builderFile: "src/video.ts", + }, + { + surface: "transcription", + title: "audio-to-text multipart upload", + provider: "Transcription", + builderFile: "src/transcription.ts", + }, + { + surface: "rerank", + title: "document relevance scoring endpoint", + provider: "Cohere Rerank", + builderFile: "src/rerank.ts", + }, + ]; + + for (const c of CASES) { + it(`RED→GREEN: "${c.surface}" drift → exit-2 entry (marker-only, legacy label CANNOT rescue)`, () => { + const block = formatDriftReport(c.title, [SAMPLE_DIFF], c.surface); + const result = makeResult([ + makeAssertion({ + ancestorTitles: [`${c.title} drift`], + title: "shape matches SDK", + failureMessages: [`AssertionError: ${block}`], + }), + ]); + + // Guard the guard: the neutral prose title must NOT be resolvable via the + // legacy provider-label path, so the marker seam is genuinely required. If + // this ever starts returning non-null, the RED→GREEN below is a false lock. + expect(extractProviderName(`${c.title} drift`)).toBeNull(); + expect(extractProviderName(c.title)).toBeNull(); + + const { entries, quarantine } = collectDriftEntries(result); + // The fix: routed to a trustworthy entry, NOT quarantined. + expect(quarantine).toHaveLength(0); + expect(entries).toHaveLength(1); + + const entry = entries[0]; + expect(entry.provider).toBe(c.provider); + expect(entry.builderFile).toBe(c.builderFile); + expect(entry.builderFunctions.length).toBeGreaterThan(0); + expect(entry.sdkShapesFile.length).toBeGreaterThan(0); + + // Exit code: 2 (auto-fixable), not 5 (quarantine). + expect(exitCodeOf(result)).toBe(2); + }); + } + + it("legacy no-marker fallback: a truly un-keyable block still quarantines (exit 5)", () => { + // A marker-less block whose prose title matches NO registry provider label + // routes to quarantine exactly as before WS-5 — the defensive legacy net is + // preserved for genuinely un-attributable output. + const block = formatDriftReport("SomeBrandNewProvider /v9/widgets", [SAMPLE_DIFF]); + const result = makeResult([ + makeAssertion({ + ancestorTitles: ["SomeBrandNewProvider drift"], + title: "shape matches SDK", + failureMessages: [`AssertionError: ${block}`], + }), + ]); + const { entries, quarantine } = collectDriftEntries(result); + expect(entries).toHaveLength(0); + expect(quarantine).toHaveLength(1); + expect(exitCodeOf(result)).toBe(5); + }); + + it("legacy no-marker fallback still resolves a known provider LABEL to an entry", () => { + // Back-compat: an unmigrated block that carries no Surface: marker but whose + // prose title contains a registered provider label still routes to an entry. + const block = formatDriftReport("Cohere Chat completions", [SAMPLE_DIFF]); + const result = makeResult([ + makeAssertion({ + ancestorTitles: ["Cohere Chat drift"], + title: "shape matches SDK", + failureMessages: [`AssertionError: ${block}`], + }), + ]); + const { entries, quarantine } = collectDriftEntries(result); + expect(quarantine).toHaveLength(0); + expect(entries).toHaveLength(1); + expect(entries[0].builderFile).toBe("src/cohere.ts"); + }); +}); + +describe("WS-5 — unknown surface slug fails LOUD (throws), never silent quarantine", () => { + it("collectDriftEntries throws on a marker with a slug not in the registry", () => { + // Build the block manually — formatDriftReport(surface) would itself throw + // on an unknown slug, so synthesize the marker directly to exercise the + // COLLECTOR's runtime throw. + const block = + "\nAPI DRIFT DETECTED: Totally New Surface\n" + + " Surface: totally-new-surface\n\n" + + " 1. [critical] LLMOCK DRIFT — mismatch detected\n" + + " Path: a.b.c\n" + + " SDK: null\n" + + " Real: null\n" + + " Mock: \n"; + const result = makeResult([ + makeAssertion({ + ancestorTitles: ["Totally New Surface drift"], + title: "shape matches SDK", + failureMessages: [`AssertionError: ${block}`], + }), + ]); + + expect(() => collectDriftEntries(result)).toThrow( + /Unknown drift surface "totally-new-surface"/, + ); + }); + + it("formatDriftReport throws at emit time on an unknown slug", () => { + expect(() => formatDriftReport("X", [SAMPLE_DIFF], "not-a-real-surface")).toThrow( + /unknown drift surface "not-a-real-surface"/, + ); + }); + + it.each(["constructor", "__proto__", "hasOwnProperty", "toString", "valueOf"])( + "throws (not garbage entry) when the marker slug is the Object.prototype member %s", + (protoSlug) => { + // A prototype-chain bracket lookup (SURFACE_REGISTRY[slug]) would resolve + // these to a truthy INHERITED member and skip the throw, emitting a + // DriftEntry with builderFile: undefined. The Object.hasOwn / isKnownSurface + // guard must treat them as unknown and THROW loudly. + const block = + `\nAPI DRIFT DETECTED: Prototype Slug\n` + + ` Surface: ${protoSlug}\n\n` + + " 1. [critical] LLMOCK DRIFT — mismatch detected\n" + + " Path: a.b.c\n" + + " SDK: null\n" + + " Real: null\n" + + " Mock: \n"; + const result = makeResult([ + makeAssertion({ + ancestorTitles: ["Prototype Slug drift"], + title: "shape matches SDK", + failureMessages: [`AssertionError: ${block}`], + }), + ]); + + expect(() => collectDriftEntries(result)).toThrow( + new RegExp(`Unknown drift surface "${protoSlug.replace(/[$]/g, "\\$&")}"`), + ); + }, + ); +}); + +describe("WS-5 — base-report reuse contract (generatedAt + conclusion)", () => { + it("conclusionForExitCode maps exit codes to coarse conclusions", () => { + expect(conclusionForExitCode(0)).toBe("clean"); + expect(conclusionForExitCode(2)).toBe("critical"); + expect(conclusionForExitCode(5)).toBe("quarantine"); + expect(conclusionForExitCode(1)).toBe("skipped"); + }); + + it("isBaseReportReusable accepts a written clean report (reuse works)", () => { + // A report shaped like what main() now writes for a clean run. + const timestamp = new Date().toISOString(); + const report: DriftReport = { + timestamp, + generatedAt: timestamp, + conclusion: conclusionForExitCode(0), + entries: [ + { + provider: "OpenAI Chat", + scenario: "non-streaming text", + builderFile: "src/helpers.ts", + builderFunctions: ["buildTextCompletion"], + typesFile: "src/types.ts", + sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", + diffs: [], + }, + ], + }; + // Same-UTC-day + known-good conclusion + non-empty entries → reusable. + expect(isBaseReportReusable(report, report.conclusion, true)).toBe(true); + }); + + it("a report WITHOUT conclusion is not reusable (documents the pre-fix gap)", () => { + const timestamp = new Date().toISOString(); + const legacy: DriftReport = { + timestamp, + entries: [ + { + provider: "OpenAI Chat", + scenario: "non-streaming text", + builderFile: "src/helpers.ts", + builderFunctions: ["buildTextCompletion"], + typesFile: "src/types.ts", + sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", + diffs: [], + }, + ], + }; + // No conclusion field → falls back to undefined → not reusable. + expect(isBaseReportReusable(legacy, legacy.conclusion, true)).toBe(false); + }); + + it("generatedAt drives sameUtcDay staleness: same-day report reuses, prior-day does not", () => { + // Mirrors the sameUtcDay derivation the drift workflow computes from + // report.generatedAt (.github/workflows/test-drift.yml). This locks the + // *semantics* of generatedAt (a stale-day base is rejected), not merely that + // the field is written. A clean report identical in every way EXCEPT its + // generatedAt day must flip reusability. + const sameUtcDay = (generatedAt: string, now: Date): boolean => { + const g = new Date(generatedAt); + return ( + g.getUTCFullYear() === now.getUTCFullYear() && + g.getUTCMonth() === now.getUTCMonth() && + g.getUTCDate() === now.getUTCDate() + ); + }; + + const now = new Date("2026-07-15T12:00:00.000Z"); + const cleanReport = (generatedAt: string): DriftReport => ({ + timestamp: generatedAt, + generatedAt, + conclusion: conclusionForExitCode(0), + entries: [ + { + provider: "OpenAI Chat", + scenario: "non-streaming text", + builderFile: "src/helpers.ts", + builderFunctions: ["buildTextCompletion"], + typesFile: "src/types.ts", + sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", + diffs: [], + }, + ], + }); + + // Same UTC day (later hour, same date) → derivation true → reusable. + const today = cleanReport("2026-07-15T03:00:00.000Z"); + expect(sameUtcDay(today.generatedAt!, now)).toBe(true); + expect(isBaseReportReusable(today, today.conclusion, sameUtcDay(today.generatedAt!, now))).toBe( + true, + ); + + // Prior UTC day → derivation false → NOT reusable, despite an otherwise + // identical clean report. generatedAt is what makes the difference. + const yesterday = cleanReport("2026-07-14T23:59:59.000Z"); + expect(sameUtcDay(yesterday.generatedAt!, now)).toBe(false); + expect( + isBaseReportReusable( + yesterday, + yesterday.conclusion, + sameUtcDay(yesterday.generatedAt!, now), + ), + ).toBe(false); + }); +}); + +/** + * Statically extract every `surface` slug that a `*.drift.ts` emitter passes as + * the THIRD argument of `formatDriftReport(context, diffs, surface)`. + * + * This scans the real source via the TypeScript AST (NOT regex/text lexing — a + * text scan would mis-hit `formatDriftReport` inside strings/comments and cannot + * reliably pick the 3rd argument across multiline calls). Only string-literal + * 3rd args are collected: a 2-arg call (legacy, no marker — e.g. models.drift.ts) + * or a non-literal arg contributes no slug and is intentionally skipped. + */ +function collectEmittedSurfaceSlugs(): { slugs: Set; scannedFiles: string[] } { + // eslint-disable-next-line @typescript-eslint/no-require-imports + const ts = require("typescript") as typeof import("typescript"); + const driftDir = resolve(__dirname, "drift"); + const files = readdirSync(driftDir).filter((f) => f.endsWith(".drift.ts")); + const slugs = new Set(); + + for (const file of files) { + const abs = resolve(driftDir, file); + const source = readFileSync(abs, "utf8"); + const sf = ts.createSourceFile(abs, source, ts.ScriptTarget.Latest, true); + + const visit = (node: import("typescript").Node): void => { + if ( + ts.isCallExpression(node) && + ts.isIdentifier(node.expression) && + node.expression.text === "formatDriftReport" && + node.arguments.length >= 3 + ) { + const third = node.arguments[2]; + if (ts.isStringLiteral(third) || ts.isNoSubstitutionTemplateLiteral(third)) { + slugs.add(third.text); + } + } + ts.forEachChild(node, visit); + }; + visit(sf); + } + return { slugs, scannedFiles: files }; +} + +describe("WS-5 — SURFACE_REGISTRY coverage & integrity", () => { + it("every slug an emitter passes to formatDriftReport is a registered surface", () => { + // Independent derivation (F2/F3): scan the ACTUAL emitter call sites rather + // than iterating the registry's own keys (which is a tautology). This locks + // the "every emitter is registered" invariant at TEST time, so a new + // unregistered emitter fails CI even on a credential-less run where no drift + // is ever emitted and the collector's runtime throw is never reached. + const { slugs, scannedFiles } = collectEmittedSurfaceSlugs(); + expect(scannedFiles.length, "found *.drift.ts files to scan").toBeGreaterThan(0); + expect(slugs.size, "found at least one emitted surface slug").toBeGreaterThan(0); + + const unregistered = [...slugs].filter((slug) => !isKnownSurface(slug)); + expect( + unregistered, + `emitter slug(s) missing from SURFACE_REGISTRY: ${unregistered.join(", ")}`, + ).toEqual([]); + }); + + it("SURFACE_REGISTRY has no orphan slugs (every registered surface is emitted)", () => { + // Reverse direction: an entry no emitter uses is dead weight. Kept as a + // separate assertion so a future intentional pre-registration is easy to see. + const { slugs } = collectEmittedSurfaceSlugs(); + const orphans = KNOWN_SURFACE_SLUGS.filter((slug) => !slugs.has(slug)); + expect(orphans, `registered but never emitted: ${orphans.join(", ")}`).toEqual([]); + }); + + it("every registry entry resolves to an existing builderFile with non-empty builderFunctions", () => { + // Mirrors the fix-drift.ts validation so a bad entry fails locally, not in CI. + const repoRoot = resolve(__dirname, "..", ".."); + for (const [slug, mapping] of Object.entries(SURFACE_REGISTRY)) { + expect(mapping.provider.length, `${slug} provider`).toBeGreaterThan(0); + expect(mapping.builderFunctions.length, `${slug} builderFunctions`).toBeGreaterThan(0); + expect( + mapping.builderFunctions.every((f) => typeof f === "string" && f.length > 0), + `${slug} builderFunctions all non-empty strings`, + ).toBe(true); + const abs = resolve(repoRoot, mapping.builderFile); + expect(existsSync(abs), `${slug} builderFile exists: ${mapping.builderFile}`).toBe(true); + if (mapping.typesFile !== null) { + expect( + existsSync(resolve(repoRoot, mapping.typesFile)), + `${slug} typesFile exists: ${mapping.typesFile}`, + ).toBe(true); + } + } + }); + + it("provider labels are unique (legacy fallback reverse-index has no collisions)", () => { + const labels = Object.values(SURFACE_REGISTRY).map((m) => m.provider); + expect(new Set(labels).size).toBe(labels.length); + }); }); diff --git a/src/__tests__/drift-delta.test.ts b/src/__tests__/drift-delta.test.ts new file mode 100644 index 00000000..833e08a3 --- /dev/null +++ b/src/__tests__/drift-delta.test.ts @@ -0,0 +1,195 @@ +import { describe, it, expect } from "vitest"; + +import { computeDelta, isBaseReportReusable } from "../../scripts/drift-delta.js"; +import type { DeltaKey } from "../../scripts/drift-delta.js"; +import { DriftClass } from "../../scripts/drift-types.js"; +import type { DriftReport, ParsedDiff } from "../../scripts/drift-types.js"; + +// --------------------------------------------------------------------------- +// drift-delta: the delta-gating core. +// +// The gate must BLOCK only on drift attributable to the PR diff (new-in-head), +// treat drift already present on main as ADVISORY (environmental / world drift), +// and report base-only drift as FIXED. The block/advisory decision is by KEY +// PRESENCE alone — `DriftClass` is annotation and must NEVER route. +// --------------------------------------------------------------------------- + +function diff(overrides: Partial = {}): ParsedDiff { + return { + path: "knownModels", + severity: "critical", + issue: "model drift", + expected: "x", + real: "y", + mock: "z", + ...overrides, + }; +} + +function report( + provider: string, + diffs: ParsedDiff[], + timestamp = "2026-07-14T00:00:00.000Z", +): DriftReport { + return { + timestamp, + entries: [ + { + provider, + scenario: "s", + builderFile: "b.ts", + builderFunctions: ["f"], + typesFile: null, + sdkShapesFile: "shapes.ts", + diffs, + }, + ], + }; +} + +function keys(list: DeltaKey[]): string[] { + return list.map((k) => `${k.provider}:${k.id}`).sort(); +} + +// --------------------------------------------------------------------------- +// The M-1 golden regression (#292). +// +// A real-drift/critical failure that is NEW-in-head MUST BLOCK. The old broken +// rule routed by CLASS (real-drift/critical → advisory), which would have +// greenlit #292. This test proves the class-routed rule fails and computeDelta +// blocks regardless of class. +// --------------------------------------------------------------------------- +describe("M-1 golden: new-in-head critical MUST block regardless of class", () => { + // Head introduces a critical/real-drift finding that base does not have. + const base = report("anthropic", [diff({ id: "claude-3-opus", class: DriftClass.None })]); + const head = report("anthropic", [ + diff({ id: "claude-3-opus", class: DriftClass.None }), + diff({ id: "claude-4-new-model", class: DriftClass.Critical }), + ]); + + // Simulate the OLD broken rule: route purely by class. A critical drift is + // sent to advisory, so a new-in-head #292 failure is NOT blocked. This stub + // encodes the pre-fix behavior we are regressing against. + const classRouted = (r: DriftReport) => { + const block: DeltaKey[] = []; + const advisory: DeltaKey[] = []; + for (const entry of r.entries) { + for (const d of entry.diffs) { + const dk: DeltaKey = { provider: entry.provider, id: d.id ?? d.path, class: d.class }; + if (d.class === DriftClass.Critical) advisory.push(dk); + else block.push(dk); + } + } + return { block, advisory }; + }; + + it("RED regression: the CLASS-ROUTED rule fails to block the new-in-head critical (would greenlight #292)", () => { + const result = classRouted(head); + const newCriticalBlocked = result.block.some((k) => k.id === "claude-4-new-model"); + // The broken rule sends the critical to advisory, NOT block. If this ever + // starts blocking, the class-routed regression is no longer being exercised. + expect(newCriticalBlocked).toBe(false); + expect(result.advisory.some((k) => k.id === "claude-4-new-model")).toBe(true); + }); + + it("GREEN: computeDelta blocks the new-in-head critical regardless of class", () => { + const { block, advisory } = computeDelta(base, head); + expect(keys(block)).toEqual(["anthropic:claude-4-new-model"]); + expect(keys(advisory)).toEqual(["anthropic:claude-3-opus"]); + // The blocked key is critical — proving class did not route it to advisory. + expect(block[0].class).toBe(DriftClass.Critical); + }); +}); + +describe("computeDelta routing by key presence", () => { + it("same key in base+head → advisory (even critical)", () => { + const base = report("openai", [diff({ id: "gpt-4", class: DriftClass.Critical })]); + const head = report("openai", [diff({ id: "gpt-4", class: DriftClass.Critical })]); + const { block, advisory, fixed } = computeDelta(base, head); + expect(block).toEqual([]); + expect(keys(advisory)).toEqual(["openai:gpt-4"]); + expect(fixed).toEqual([]); + }); + + it("head-only transient → block (keyed by provider+id)", () => { + const base = report("openai", []); + const head = report("openai", [diff({ id: "gpt-5-preview", class: DriftClass.Critical })]); + const { block, advisory, fixed } = computeDelta(base, head); + expect(keys(block)).toEqual(["openai:gpt-5-preview"]); + expect(advisory).toEqual([]); + expect(fixed).toEqual([]); + }); + + it("base-only failure → fixed (informational, not block/advisory)", () => { + const base = report("openai", [diff({ id: "gpt-3.5", class: DriftClass.Critical })]); + const head = report("openai", []); + const { block, advisory, fixed } = computeDelta(base, head); + expect(block).toEqual([]); + expect(advisory).toEqual([]); + expect(keys(fixed)).toEqual(["openai:gpt-3.5"]); + }); + + it("keys by provider+id (path bucket must NOT collapse N distinct ids into one)", () => { + const base = report("anthropic", []); + const head = report("anthropic", [ + diff({ path: "knownModels", id: "a" }), + diff({ path: "knownModels", id: "b" }), + diff({ path: "knownModels", id: "c" }), + ]); + const { block } = computeDelta(base, head); + expect(keys(block)).toEqual(["anthropic:a", "anthropic:b", "anthropic:c"]); + }); + + it("same id across different providers → distinct keys", () => { + const base = report("openai", [diff({ id: "shared" })]); + const head = { + timestamp: base.timestamp, + entries: [...base.entries, ...report("anthropic", [diff({ id: "shared" })]).entries], + }; + const { block, advisory } = computeDelta(base, head); + expect(keys(block)).toEqual(["anthropic:shared"]); + expect(keys(advisory)).toEqual(["openai:shared"]); + }); + + it("falls back to path when id is absent (legacy diffs still participate)", () => { + const base = report("cohere", []); + const head = report("cohere", [diff({ path: "legacyBucket" })]); // no id + const { block } = computeDelta(base, head); + expect(keys(block)).toEqual(["cohere:legacyBucket"]); + }); +}); + +describe("isBaseReportReusable (O-2)", () => { + const good = report("openai", [diff({ id: "gpt-4" })]); + + it("accepts a non-empty, known-good, same-UTC-day report", () => { + expect(isBaseReportReusable(good, "clean", true)).toBe(true); + expect(isBaseReportReusable(good, "success", true)).toBe(true); + }); + + it("rejects an empty-entries report (malformed cached base)", () => { + const empty: DriftReport = { timestamp: "2026-07-14T00:00:00.000Z", entries: [] }; + expect(isBaseReportReusable(empty, "clean", true)).toBe(false); + }); + + it("rejects a null / malformed report object", () => { + expect(isBaseReportReusable(null, "clean", true)).toBe(false); + expect(isBaseReportReusable(undefined, "clean", true)).toBe(false); + // Malformed: entries missing entirely. + expect(isBaseReportReusable({ timestamp: "t" } as unknown as DriftReport, "clean", true)).toBe( + false, + ); + }); + + it("rejects an unknown / bad conclusion (crash, quarantine, empty)", () => { + expect(isBaseReportReusable(good, "failure", true)).toBe(false); + expect(isBaseReportReusable(good, "quarantine", true)).toBe(false); + expect(isBaseReportReusable(good, "", true)).toBe(false); + expect(isBaseReportReusable(good, null, true)).toBe(false); + expect(isBaseReportReusable(good, undefined, true)).toBe(false); + }); + + it("rejects a stale (different-UTC-day) report", () => { + expect(isBaseReportReusable(good, "clean", false)).toBe(false); + }); +}); diff --git a/src/__tests__/drift-retry.test.ts b/src/__tests__/drift-retry.test.ts index 373a0b23..9e34c13f 100644 --- a/src/__tests__/drift-retry.test.ts +++ b/src/__tests__/drift-retry.test.ts @@ -1,6 +1,6 @@ import { describe, it, expect } from "vitest"; -import { retryUntilStable } from "../../scripts/drift-retry.js"; +import { EXIT_QUARANTINE, retryUntilStable } from "../../scripts/drift-retry.js"; import type { RetryAttempt, RetryOptions, RetryResult } from "../../scripts/drift-retry.js"; // --------------------------------------------------------------------------- @@ -137,4 +137,21 @@ describe("retryUntilStable", () => { expect(result.attempts.map((a: RetryAttempt) => a.exitCode)).toEqual([2, 2, 0]); }); + + it("treats exit 5 (quarantine) as a distinct terminal outcome — no retry, quarantine:true", () => { + // exit 5 = collector quarantined unparseable output; must propagate immediately + // without retrying and mark the result with quarantine:true. + const runner = fakeRunner([EXIT_QUARANTINE, 0, 0]); + const opts = makeOptions({ runCollector: runner.run }); + const result: RetryResult = retryUntilStable(opts); + + // Propagated as exit 5, not swallowed into the crash branch + expect(result.exitCode).toBe(EXIT_QUARANTINE); + // Quarantine flag set + expect(result.quarantine).toBe(true); + // No retry — only one attempt + expect(runner.calls()).toBe(1); + // Not treated as a transient drift event + expect(result.transient).toBe(false); + }); }); diff --git a/src/__tests__/drift-scripts.test.ts b/src/__tests__/drift-scripts.test.ts index 268fb68d..6ce34527 100644 --- a/src/__tests__/drift-scripts.test.ts +++ b/src/__tests__/drift-scripts.test.ts @@ -19,7 +19,7 @@ import { import { summarizeDriftReport } from "../../scripts/drift-slack-summary.js"; -import type { DriftEntry, DriftReport } from "../../scripts/drift-types.js"; +import type { DriftEntry, DriftReport, QuarantineEntry } from "../../scripts/drift-types.js"; // --------------------------------------------------------------------------- // Helpers @@ -356,7 +356,9 @@ describe("summarizeDriftReport", () => { expect(summary).toContain("*OpenAI Chat*"); expect(summary).toContain("1 critical"); expect(summary).toContain("`choices[0].message.refusal`"); - expect(summary.startsWith("•")).toBe(true); + // The first line is now the classification header; bullet lines follow. + const bulletLines = summary.split("\n").filter((l) => l.startsWith("•")); + expect(bulletLines.length).toBeGreaterThan(0); }); it("merges multiple entries for the same provider into one line with combined counts", () => { @@ -404,11 +406,12 @@ describe("summarizeDriftReport", () => { makeEntry(), // OpenAI Chat with a critical ], }); - const lines = summary.split("\n"); - expect(lines).toHaveLength(2); + // First line is the classification header; provider bullets follow. + const bulletLines = summary.split("\n").filter((l) => l.startsWith("•")); + expect(bulletLines).toHaveLength(2); // Provider with a critical diff sorts before the warning-only provider - expect(lines[0]).toContain("*OpenAI Chat*"); - expect(lines[1]).toContain("*Anthropic*"); + expect(bulletLines[0]).toContain("*OpenAI Chat*"); + expect(bulletLines[1]).toContain("*Anthropic*"); }); it("caps example paths per provider and reports the remainder", () => { @@ -439,3 +442,147 @@ describe("summarizeDriftReport", () => { expect(summary).not.toContain("\\n"); }); }); + +// --------------------------------------------------------------------------- +// summarizeDriftReport — headline class + per-item detail (C5.2) +// --------------------------------------------------------------------------- + +function makeQuarantineEntry(overrides?: Partial): QuarantineEntry { + return { + provider: "OpenAI Chat", + testName: "OpenAI Chat > non-streaming text > response shape", + rawLocation: "src/__tests__/drift/openai-chat.drift.ts:42", + message: "Cannot read properties of undefined (reading 'choices')", + ...overrides, + }; +} + +describe("summarizeDriftReport — headline class", () => { + it("class: real-drift — report has entries with critical diffs", () => { + const report: DriftReport = { + timestamp: "t", + entries: [ + makeEntry({ + diffs: [ + { + severity: "critical", + issue: "field missing from mock", + path: "choices[0].message.refusal", + expected: "null", + real: "null", + mock: "", + }, + ], + }), + ], + }; + const summary = summarizeDriftReport(report); + expect(summary).toContain("real-drift"); + // per-item: provider name present + expect(summary).toContain("OpenAI Chat"); + // per-item: offending path or id + expect(summary).toContain("choices[0].message.refusal"); + // per-item: one-line issue + expect(summary).toContain("field missing from mock"); + // per-item: file reference (builderFile) + expect(summary).toContain("src/helpers.ts"); + }); + + it("class: quarantine — report has quarantine[] entries", () => { + const report: DriftReport = { + timestamp: "t", + entries: [], + quarantine: [makeQuarantineEntry()], + }; + const summary = summarizeDriftReport(report); + expect(summary).toContain("quarantine"); + // quarantine: provider + expect(summary).toContain("OpenAI Chat"); + // quarantine: rawLocation (file:line) + expect(summary).toContain("src/__tests__/drift/openai-chat.drift.ts:42"); + // quarantine: one-line message + expect(summary).toContain("Cannot read properties of undefined"); + }); + + it("class: stale-key — InfraError status 401", () => { + const report: DriftReport = { timestamp: "t", entries: [] }; + const summary = summarizeDriftReport(report, { infraErrorStatus: 401 }); + expect(summary).toContain("stale-key"); + }); + + it("class: stale-key — InfraError status 403", () => { + const report: DriftReport = { timestamp: "t", entries: [] }; + const summary = summarizeDriftReport(report, { infraErrorStatus: 403 }); + expect(summary).toContain("stale-key"); + }); + + it("class: infra-transient — InfraError status 429", () => { + const report: DriftReport = { timestamp: "t", entries: [] }; + const summary = summarizeDriftReport(report, { infraErrorStatus: 429 }); + expect(summary).toContain("infra-transient"); + }); + + it("class: infra-transient — InfraError status 503", () => { + const report: DriftReport = { timestamp: "t", entries: [] }; + const summary = summarizeDriftReport(report, { infraErrorStatus: 503 }); + expect(summary).toContain("infra-transient"); + }); + + it("class: test-infra-false-positive — exit 0, empty entries, no infraError", () => { + const report: DriftReport = { timestamp: "t", entries: [] }; + const summary = summarizeDriftReport(report, { exitCode: 0 }); + expect(summary).toContain("test-infra-false-positive"); + }); + + it("quarantine entries appear on their own distinct line with testName", () => { + const report: DriftReport = { + timestamp: "t", + entries: [], + quarantine: [ + makeQuarantineEntry({ provider: "Anthropic", testName: "Anthropic > streaming > shape" }), + ], + }; + const summary = summarizeDriftReport(report); + const lines = summary.split("\n"); + const quarantineLine = lines.find((l) => l.includes("Anthropic")); + expect(quarantineLine).toBeDefined(); + expect(quarantineLine).toContain("Anthropic > streaming > shape"); + }); + + it("mixed: real-drift entries + quarantine both appear in summary", () => { + const report: DriftReport = { + timestamp: "t", + entries: [makeEntry()], + quarantine: [makeQuarantineEntry({ provider: "Gemini" })], + }; + const summary = summarizeDriftReport(report); + // Class is real-drift when entries have critical diffs (quarantine is secondary) + expect(summary).toContain("real-drift"); + // Quarantine section still present + expect(summary).toContain("Gemini"); + expect(summary).toContain("quarantine"); + }); + + it("per-item id is used over path when present", () => { + const report: DriftReport = { + timestamp: "t", + entries: [ + makeEntry({ + diffs: [ + { + severity: "critical", + issue: "model removed from mock", + path: "knownModels", + id: "gpt-4o-mini", + expected: "present", + real: "present", + mock: "", + }, + ], + }), + ], + }; + const summary = summarizeDriftReport(report); + expect(summary).toContain("gpt-4o-mini"); + }); +}); diff --git a/src/__tests__/drift-success-predicate.test.ts b/src/__tests__/drift-success-predicate.test.ts new file mode 100644 index 00000000..3ad04a74 --- /dev/null +++ b/src/__tests__/drift-success-predicate.test.ts @@ -0,0 +1,1400 @@ +/** + * Tests for the WS-2 drift-success predicate. + * + * These exercise the REAL exported pure function `evaluateDriftResolved` and + * the CLI helpers from scripts/drift-success-predicate.ts. The predicate is a + * pure function over a small `DriftReport` fixture + a synthetic changed-file + * array — no live API, no LLM, no aimock needed. + * + * The headline case is the fixture-relaxation cheat: a run that changes ONLY + * `src/__tests__/drift/sdk-shapes.ts` (relaxing the SDK leg) must be REJECTED + * (COMPARISON_LEG_ONLY), whereas the OLD fix-drift.ts guard (`builderFiles>0 || + * testFiles>0`) would have ACCEPTED it — demonstrated by contrast below. + */ + +import { execFileSync } from "node:child_process"; +import { mkdtempSync, writeFileSync, rmSync, mkdirSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { describe, it, expect, afterEach, vi } from "vitest"; + +import type { DriftEntry, DriftReport, ParsedDiff } from "../../scripts/drift-types.js"; +import { + evaluateDriftResolved, + PredicateReason, + REASON_EXIT_CODE, + isProductionFile, + isComparisonLeg, + isSuppressionSurface, + isGameableLeg, + canonicalizePath, + isAllowlisted, + sanctionedTargets, + countCriticalDiffs, + parseCliArgs, + parsePorcelainLine, + crossCheckChangedFiles, + PredicateConfigError, + readReport, + runCli, +} from "../../scripts/drift-success-predicate.js"; + +// --------------------------------------------------------------------------- +// Fixture builders +// --------------------------------------------------------------------------- + +function diff(overrides: Partial = {}): ParsedDiff { + return { + path: "choices[0].message.content", + severity: "critical", + issue: "field present in SDK+real but missing from mock", + expected: "string", + real: "string", + mock: "", + ...overrides, + }; +} + +function entry(overrides: Partial = {}): DriftEntry { + return { + provider: "OpenAI", + scenario: "chat completion", + builderFile: "src/helpers.ts", + builderFunctions: ["buildChatCompletion"], + typesFile: "src/types.ts", + sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", + diffs: [diff()], + ...overrides, + }; +} + +function report(entries: DriftEntry[] = [entry()]): DriftReport { + return { timestamp: "2026-07-16T00:00:00.000Z", entries }; +} + +/** The OLD, gameable guard from fix-drift.ts:638 — for contrast assertions. */ +function oldGuardWouldAccept(changedFiles: string[]): boolean { + const builderFiles = changedFiles.filter( + (f) => f.startsWith("src/") && !f.startsWith("src/__tests__/"), + ); + const testFiles = changedFiles.filter((f) => f.startsWith("src/__tests__/")); + // OLD guard aborts ONLY when BOTH are empty; otherwise it proceeds. + return !(builderFiles.length === 0 && testFiles.length === 0); +} + +// --------------------------------------------------------------------------- +// RED cases — resolved:false +// --------------------------------------------------------------------------- + +describe("evaluateDriftResolved — RED (cheat/failure) cases", () => { + it("HEADLINE: fixture-relaxation-only (sdk-shapes.ts) → COMPARISON_LEG_ONLY, and the OLD guard would have ACCEPTED it", () => { + const changedFiles = ["src/__tests__/drift/sdk-shapes.ts"]; + const verdict = evaluateDriftResolved({ + changedFiles, + report: report(), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.COMPARISON_LEG_ONLY); + expect(verdict.offendingFiles).toContain("src/__tests__/drift/sdk-shapes.ts"); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(11); + + // Contrast: the OLD guard would have proceeded (testFiles non-empty). + expect(oldGuardWouldAccept(changedFiles)).toBe(true); + }); + + it("schema/allowlist edit + real builder change → SUPPRESSION_SUSPECTED (blocks even with a prod change)", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/__tests__/drift/schema.ts", "src/helpers.ts"], + report: report(), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.SUPPRESSION_SUSPECTED); + expect(verdict.offendingFiles).toContain("src/__tests__/drift/schema.ts"); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(12); + }); + + it("*.drift.ts assertion loosened only → SUPPRESSION_SUSPECTED", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/__tests__/drift/openai-chat.drift.ts"], + report: report(), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.SUPPRESSION_SUSPECTED); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(12); + }); + + it("no changes at all → NO_PRODUCTION_CHANGE", () => { + const verdict = evaluateDriftResolved({ + changedFiles: [], + report: report(), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.NO_PRODUCTION_CHANGE); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(10); + }); + + it("production change but collector still dirty (exit 2) → STILL_DIRTY", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts"], + report: report(), + postFixCollectorExit: 2, + postFixCriticalCount: 1, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.STILL_DIRTY); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(13); + }); + + it("production change + collector exit 0 but criticalCount>0 → STILL_DIRTY (belt-and-suspenders)", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts"], + report: report(), + postFixCollectorExit: 0, + postFixCriticalCount: 3, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.STILL_DIRTY); + }); + + it("post-fix quarantine (exit 5) → QUARANTINE_AFTER_FIX", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts"], + report: report(), + postFixCollectorExit: 5, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.QUARANTINE_AFTER_FIX); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(14); + }); + + it("post-fix collector infra (exit 1) → COLLECTOR_INFRA", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts"], + report: report(), + postFixCollectorExit: 1, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.COLLECTOR_INFRA); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(15); + }); + + it("production change off-target → PRODUCTION_CHANGE_OFF_TARGET", () => { + // report names src/helpers.ts; the change is an unrelated production file. + const verdict = evaluateDriftResolved({ + changedFiles: ["src/gemini.ts"], + report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.PRODUCTION_CHANGE_OFF_TARGET); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(16); + }); + + // ------------------------------------------------------------------------- + // WS-2b HYBRID CHEAT — a gameable-leg edit ACCOMPANIED by a trivial, on-target + // production edit. Pre-fix the predicate ignored the leg once ANY production + // file changed → RESOLVED (the exact WS-2b auto-merge cheat). Post-fix a leg + // edit ALWAYS blocks (SUPPRESSION_SUSPECTED), regardless of production files. + // ------------------------------------------------------------------------- + it("HEADLINE WS-2b: sdk-shapes.ts relaxation + on-target production edit → SUPPRESSION_SUSPECTED (block), NOT resolved", () => { + const changedFiles = ["src/__tests__/drift/sdk-shapes.ts", "src/helpers.ts"]; + const verdict = evaluateDriftResolved({ + changedFiles, + report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.SUPPRESSION_SUSPECTED); + expect(verdict.offendingFiles).toContain("src/__tests__/drift/sdk-shapes.ts"); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(12); + // Contrast: the OLD guard would have proceeded (both builder + test present). + expect(oldGuardWouldAccept(changedFiles)).toBe(true); + }); + + it("harness leg (providers.ts) relaxation + on-target production edit → SUPPRESSION_SUSPECTED (block)", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/__tests__/drift/providers.ts", "src/helpers.ts"], + report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.SUPPRESSION_SUSPECTED); + expect(verdict.offendingFiles).toContain("src/__tests__/drift/providers.ts"); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(12); + }); + + it("harness-only leg edit (ws-providers.ts), no production change → COMPARISON_LEG_ONLY (block, pure relaxation)", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/__tests__/drift/ws-providers.ts"], + report: report(), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(false); + // Leg edit with NO production change → COMPARISON_LEG_ONLY (a pure + // relaxation; no mock fix even attempted). Still a hard block (exit 11). + expect(verdict.reason).toBe(PredicateReason.COMPARISON_LEG_ONLY); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(11); + }); + + // ------------------------------------------------------------------------- + // FIX #2 — dual-classification precedence: voice-models.ts is BOTH a harness + // leg AND a legit fixture target. Block-classification MUST win (fail-closed). + // ------------------------------------------------------------------------- + it("voice-models.ts (dual-classified harness+target) + on-target production edit → SUPPRESSION_SUSPECTED (block wins)", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/__tests__/drift/voice-models.ts", "src/ws-realtime.ts"], + report: report([entry({ builderFile: "src/ws-realtime.ts", typesFile: null })]), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.SUPPRESSION_SUSPECTED); + expect(verdict.offendingFiles).toContain("src/__tests__/drift/voice-models.ts"); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(12); + }); + + // ------------------------------------------------------------------------- + // FIX #3 — empty sanctioned-target set must fail closed (needs-human), not + // silently accept any production change by disabling the off-target guard. + // ------------------------------------------------------------------------- + it("empty sanctionedTargets (report entries have no usable target) → PRODUCTION_CHANGE_OFF_TARGET (fail-closed)", () => { + // Fabricate a report whose entries yield ZERO sanctioned targets: builderFile + // "" and typesFile null. (evaluateDriftResolved does not re-validate the + // report shape — it only reads builderFile/typesFile via sanctionedTargets.) + const emptyTargetReport: DriftReport = { + timestamp: "2026-07-16T00:00:00.000Z", + entries: [ + { + provider: "OpenAI", + scenario: "chat completion", + builderFile: "", + builderFunctions: ["buildChatCompletion"], + typesFile: null, + sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", + diffs: [diff()], + }, + ], + }; + expect(sanctionedTargets(emptyTargetReport).size).toBe(0); + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts"], + report: emptyTargetReport, + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.PRODUCTION_CHANGE_OFF_TARGET); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(16); + }); + + // ------------------------------------------------------------------------- + // FIX #6 — exit 5/1 WITH parseable criticalCount>0 gets its OWN reason + // (quarantine/infra), NOT STILL_DIRTY. The collector-state classification + // wins over the belt-and-suspenders criticalCount check. + // ------------------------------------------------------------------------- + it("post-fix quarantine (exit 5) with criticalCount>0 → QUARANTINE_AFTER_FIX (not STILL_DIRTY)", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts"], + report: report(), + postFixCollectorExit: 5, + postFixCriticalCount: 4, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.QUARANTINE_AFTER_FIX); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(14); + }); + + it("post-fix infra (exit 1) with criticalCount>0 → COLLECTOR_INFRA (not STILL_DIRTY)", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts"], + report: report(), + postFixCollectorExit: 1, + postFixCriticalCount: 4, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.COLLECTOR_INFRA); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(15); + }); +}); + +// --------------------------------------------------------------------------- +// GREEN cases — resolved:true +// --------------------------------------------------------------------------- + +describe("evaluateDriftResolved — GREEN (real fix) cases", () => { + it("real src/helpers.ts fix + clean collector → RESOLVED", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts", "src/types.ts"], + report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(true); + expect(verdict.reason).toBe(PredicateReason.RESOLVED); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(0); + }); + + it("legit canary: model-registry.ts + ws-realtime.ts (report sanctions BOTH) → RESOLVED", () => { + // The known-models canary routes its fix to the production ws-realtime.ts + // (builderFile) AND the model list fixture (typesFile). Under the allowlist a + // fixture is accepted ONLY when the report names it as a target — so the + // report here sanctions model-registry.ts via typesFile. + const canary = report([ + entry({ + provider: "OpenAI Realtime", + scenario: "known-models canary", + builderFile: "src/ws-realtime.ts", + builderFunctions: ["buildRealtimeSession"], + typesFile: "src/__tests__/drift/model-registry.ts", + diffs: [ + diff({ path: "knownModels", issue: "Unknown realtime model detected", mock: "" }), + ], + }), + ]); + const verdict = evaluateDriftResolved({ + changedFiles: ["src/__tests__/drift/model-registry.ts", "src/ws-realtime.ts"], + report: canary, + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(true); + expect(verdict.reason).toBe(PredicateReason.RESOLVED); + }); + + it("AG-UI: report names src/agui-types.ts; change to that file → RESOLVED", () => { + const agui = report([ + entry({ + provider: "AG-UI", + scenario: "missing event types", + builderFile: "src/agui-types.ts", + builderFunctions: ["AGUIEventType"], + typesFile: "src/agui-types.ts", + sdkShapesFile: "src/__tests__/drift/agui-schema.drift.ts", + diffs: [diff({ path: "AGUIEventType", issue: "missing event type" })], + }), + ]); + const verdict = evaluateDriftResolved({ + changedFiles: ["src/agui-types.ts"], + report: agui, + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(true); + expect(verdict.reason).toBe(PredicateReason.RESOLVED); + }); + + it("production change + accompanying report-NAMED canary fixture → RESOLVED", () => { + // model-family.ts is accepted as an accompanying change ONLY because the + // report names it (typesFile) — under the allowlist a fixture is not a free + // pass by static membership; it must be sanctioned by the report for this run. + const verdict = evaluateDriftResolved({ + changedFiles: ["src/ws-realtime.ts", "src/__tests__/drift/model-family.ts"], + report: report([ + entry({ + builderFile: "src/ws-realtime.ts", + typesFile: "src/__tests__/drift/model-family.ts", + }), + ]), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(true); + expect(verdict.reason).toBe(PredicateReason.RESOLVED); + }); +}); + +// --------------------------------------------------------------------------- +// ALLOWLIST INVERSION (round-2 CR F1/F2/F3) — a fix is RESOLVED only when EVERY +// changed file is on the allowlist. Anything not recognized as production source +// or a report-sanctioned fixture target BLOCKS. This closes path-spelling +// sneak-ins, stale-denylist gaps, and in-diff vectors (package.json / lockfiles +// / sub-fixtures / unknown paths). +// --------------------------------------------------------------------------- + +describe("allowlist inversion — non-allowlisted changed files ALWAYS block", () => { + // A report that sanctions src/helpers.ts as the fix target, so the production + // edit itself is legitimately allowlisted; the SECOND file is the attack. + const sanctioned = () => + report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]); + + const cleanSignal = { postFixCollectorExit: 0, postFixCriticalCount: 0 }; + + it("package.json changed alongside a real production fix → UNSANCTIONED_CHANGE (block)", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts", "package.json"], + report: sanctioned(), + ...cleanSignal, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.UNSANCTIONED_CHANGE); + expect(verdict.offendingFiles).toContain("package.json"); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(17); + }); + + it("pnpm-lock.yaml changed alongside a real production fix → UNSANCTIONED_CHANGE (block)", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts", "pnpm-lock.yaml"], + report: sanctioned(), + ...cleanSignal, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.UNSANCTIONED_CHANGE); + expect(verdict.offendingFiles).toContain("pnpm-lock.yaml"); + }); + + it("tsconfig.json changed alongside a real production fix → UNSANCTIONED_CHANGE (block)", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts", "tsconfig.json"], + report: sanctioned(), + ...cleanSignal, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.UNSANCTIONED_CHANGE); + }); + + it("an unknown/unrecognized path alongside a real production fix → UNSANCTIONED_CHANGE (block)", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts", "scripts/fix-drift.ts"], + report: sanctioned(), + ...cleanSignal, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.UNSANCTIONED_CHANGE); + expect(verdict.offendingFiles).toContain("scripts/fix-drift.ts"); + }); + + it("a drift-dir *.test.ts (NOT a *.drift.ts) alongside a real production fix → UNSANCTIONED_CHANGE (closes the drift-dir .test.ts gap)", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts", "src/__tests__/drift/model-registry.test.ts"], + report: sanctioned(), + ...cleanSignal, + }); + expect(verdict.resolved).toBe(false); + // A drift-dir unit test is not a gameable comparison leg (isGameableLeg is + // false for it) and is not allowlisted → UNSANCTIONED_CHANGE. + expect(verdict.reason).toBe(PredicateReason.UNSANCTIONED_CHANGE); + expect(verdict.offendingFiles).toContain("src/__tests__/drift/model-registry.test.ts"); + }); + + it("a non-drift __tests__ file alongside a real production fix → UNSANCTIONED_CHANGE (block)", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts", "src/__tests__/server.test.ts"], + report: sanctioned(), + ...cleanSignal, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.UNSANCTIONED_CHANGE); + }); + + it("model-registry.ts fixture NOT named by the report + prod fix → UNSANCTIONED_CHANGE (fixtures allowlisted ONLY when report-named)", () => { + // model-registry.ts is a legit fixture target but the report does NOT name it + // (builderFile/typesFile are src/helpers.ts / src/types.ts). It is therefore + // NOT on the allowlist for THIS run and must block. + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts", "src/__tests__/drift/model-registry.ts"], + report: sanctioned(), + ...cleanSignal, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.UNSANCTIONED_CHANGE); + expect(verdict.offendingFiles).toContain("src/__tests__/drift/model-registry.ts"); + }); + + it("FIX #F2 (round-4): a report-NAMED fixture target ALONE (no production change) → NO_PRODUCTION_CHANGE (routed to human, NOT auto-resolved)", () => { + // A canary that names ONLY the fixture as its target and whose diff touches + // ONLY that fixture (model-registry.ts) — with NO production/builder change. + // The module invariant (Signal 2) requires >=1 production mock-builder change + // for RESOLVED: a fixture-target-only change is not independently verifiable + // (the re-collect reads the same fixture), so it must route to human review, + // never auto-resolve. This is the canary-only bypass F2 closes. + const canary = report([ + entry({ + provider: "OpenAI Realtime", + scenario: "known-models canary", + builderFile: "src/__tests__/drift/model-registry.ts", + builderFunctions: ["knownModels"], + typesFile: null, + diffs: [diff({ path: "knownModels", issue: "new model shipped" })], + }), + ]); + const verdict = evaluateDriftResolved({ + changedFiles: ["src/__tests__/drift/model-registry.ts"], + report: canary, + ...cleanSignal, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.NO_PRODUCTION_CHANGE); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(10); + }); + + it("FIX #F2 (round-4): a report-NAMED fixture target ACCOMPANIED BY a production change → RESOLVED (the legit canary shape)", () => { + // The legit canary shape: the report names BOTH a production builder and the + // fixture, and BOTH change. The production change satisfies the >=1 + // production-change invariant, so this auto-resolves (unlike the + // fixture-only case above). + const canary = report([ + entry({ + provider: "OpenAI Realtime", + scenario: "known-models canary", + builderFile: "src/ws-realtime.ts", + builderFunctions: ["buildRealtimeSession"], + typesFile: "src/__tests__/drift/model-registry.ts", + diffs: [diff({ path: "knownModels", issue: "new model shipped" })], + }), + ]); + const verdict = evaluateDriftResolved({ + changedFiles: ["src/ws-realtime.ts", "src/__tests__/drift/model-registry.ts"], + report: canary, + ...cleanSignal, + }); + expect(verdict.resolved).toBe(true); + expect(verdict.reason).toBe(PredicateReason.RESOLVED); + }); + + it("production-source-only fix (no fixtures at all) + clean collector → RESOLVED", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts"], + report: sanctioned(), + ...cleanSignal, + }); + expect(verdict.resolved).toBe(true); + expect(verdict.reason).toBe(PredicateReason.RESOLVED); + }); +}); + +// --------------------------------------------------------------------------- +// PATH CANONICALIZATION (round-2 CR slot-1/slot-2 F1) — a leg edit presented +// under an equivalent-but-non-identical spelling must still be recognized and +// blocked; classification runs on the canonical form. +// --------------------------------------------------------------------------- + +describe("path canonicalization defeats spelling-variant leg sneak-ins", () => { + it("./src/... leading-dot spelling of the SDK leg still blocks", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["./src/__tests__/drift/sdk-shapes.ts"], + report: report(), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.COMPARISON_LEG_ONLY); + }); + + it("doubled-slash spelling of the SDK leg + prod edit still blocks (SUPPRESSION_SUSPECTED)", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src//__tests__/drift/sdk-shapes.ts", "src/helpers.ts"], + report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.SUPPRESSION_SUSPECTED); + }); + + it("trailing-dot-segment spelling of a production target canonicalizes and RESOLVES", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["./src/helpers.ts"], + report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(true); + expect(verdict.reason).toBe(PredicateReason.RESOLVED); + }); + + it("canonicalizePath normalizes ./ , // and . segments", () => { + expect(canonicalizePath("./src/helpers.ts")).toBe("src/helpers.ts"); + expect(canonicalizePath("src//__tests__/drift/sdk-shapes.ts")).toBe( + "src/__tests__/drift/sdk-shapes.ts", + ); + expect(canonicalizePath("src/./helpers.ts")).toBe("src/helpers.ts"); + expect(canonicalizePath("src/helpers.ts")).toBe("src/helpers.ts"); + }); + + it("canonicalizePath rejects a path escaping the repo root (fail-closed)", () => { + expect(() => canonicalizePath("../ag-ui/events.ts")).toThrow(PredicateConfigError); + expect(() => canonicalizePath("src/../../etc/passwd")).toThrow(PredicateConfigError); + }); +}); + +// --------------------------------------------------------------------------- +// F6 — empty / non-integer --post-fix-exit must FAIL CLOSED (Number("")===0 +// must NOT be treated as a clean exit 0). +// --------------------------------------------------------------------------- + +describe("F6 — --post-fix-exit fails closed on empty/whitespace", () => { + it("throws on an empty --post-fix-exit (Number('')===0 must not slip through)", () => { + expect(() => + parseCliArgs(["--report", "a.json", "--post-fix-report", "b.json", "--post-fix-exit", ""]), + ).toThrow(PredicateConfigError); + }); + + it("throws on a whitespace-only --post-fix-exit", () => { + expect(() => + parseCliArgs(["--report", "a.json", "--post-fix-report", "b.json", "--post-fix-exit", " "]), + ).toThrow(PredicateConfigError); + }); +}); + +// --------------------------------------------------------------------------- +// F8 — a malformed post-fix report that crashes countCriticalDiffs must be +// caught and mapped to a NAMED config-error reason, not a bare stacktrace. +// --------------------------------------------------------------------------- + +describe("F8 — malformed post-fix report → named config-error (not a bare crash)", () => { + let dir: string | null = null; + afterEach(() => { + if (dir) rmSync(dir, { recursive: true, force: true }); + dir = null; + }); + + it("runCli exits 2 (CONFIG_ERROR) when the post-fix report has entries with no diffs array", () => { + dir = mkdtempSync(join(tmpdir(), "ws2-f8-")); + const preP = join(dir, "drift-report.json"); + const postP = join(dir, "drift-report.post-fix.json"); + writeFileSync(preP, JSON.stringify(report()), "utf-8"); + // Structurally passes readReport (timestamp + entries array) but each entry + // is missing `diffs`, so countCriticalDiffs would throw. + writeFileSync( + postP, + JSON.stringify({ timestamp: "t", entries: [{ provider: "OpenAI" }] }), + "utf-8", + ); + const code = runCli(["--report", preP, "--post-fix-report", postP, "--post-fix-exit", "0"]); + expect(code).toBe(REASON_EXIT_CODE[PredicateReason.CONFIG_ERROR]); + }); +}); + +// --------------------------------------------------------------------------- +// File-classification unit coverage +// --------------------------------------------------------------------------- + +describe("file classification", () => { + it("isProductionFile matches src/** except src/__tests__/**", () => { + expect(isProductionFile("src/helpers.ts")).toBe(true); + expect(isProductionFile("src/agui-types.ts")).toBe(true); + expect(isProductionFile("src/__tests__/drift/sdk-shapes.ts")).toBe(false); + expect(isProductionFile("scripts/fix-drift.ts")).toBe(false); + }); + + it("isComparisonLeg flags SDK/schema/harness/*.drift.ts (incl dual-classified voice-models) but NOT pure legit targets", () => { + expect(isComparisonLeg("src/__tests__/drift/sdk-shapes.ts")).toBe(true); + expect(isComparisonLeg("src/__tests__/drift/schema.ts")).toBe(true); + expect(isComparisonLeg("src/__tests__/drift/providers.ts")).toBe(true); + expect(isComparisonLeg("src/__tests__/drift/ws-providers.ts")).toBe(true); + expect(isComparisonLeg("src/__tests__/drift/helpers.ts")).toBe(true); + expect(isComparisonLeg("src/__tests__/drift/openai-chat.drift.ts")).toBe(true); + // Dual-classified voice-models.ts blocks (fix #2: harness membership wins). + expect(isComparisonLeg("src/__tests__/drift/voice-models.ts")).toBe(true); + // PURE legit fixture targets (not also a harness leg) are NOT comparison legs. + expect(isComparisonLeg("src/__tests__/drift/model-registry.ts")).toBe(false); + expect(isComparisonLeg("src/__tests__/drift/model-family.ts")).toBe(false); + // Production files are not comparison legs. + expect(isComparisonLeg("src/helpers.ts")).toBe(false); + }); + + it("isSuppressionSurface flags the NARROW always-suppress set (schema + *.drift.ts) only", () => { + // Suppression surface = the actively-silencing subset: schema/allowlist and + // *.drift.ts assertions. These always map to SUPPRESSION_SUSPECTED. The + // broader legs (sdk-shapes / harness) are gameable but are NOT suppression + // surfaces (they map to COMPARISON_LEG_ONLY when standalone). + expect(isSuppressionSurface("src/__tests__/drift/schema.ts")).toBe(true); + expect(isSuppressionSurface("src/__tests__/drift/openai-chat.drift.ts")).toBe(true); + expect(isSuppressionSurface("src/__tests__/drift/sdk-shapes.ts")).toBe(false); + expect(isSuppressionSurface("src/__tests__/drift/providers.ts")).toBe(false); + expect(isSuppressionSurface("src/__tests__/drift/voice-models.ts")).toBe(false); + expect(isSuppressionSurface("src/helpers.ts")).toBe(false); + }); + + it("isGameableLeg flags every leg (incl dual-classified voice-models) but NOT model-registry/model-family/production", () => { + expect(isGameableLeg("src/__tests__/drift/sdk-shapes.ts")).toBe(true); + expect(isGameableLeg("src/__tests__/drift/schema.ts")).toBe(true); + expect(isGameableLeg("src/__tests__/drift/providers.ts")).toBe(true); + expect(isGameableLeg("src/__tests__/drift/ws-providers.ts")).toBe(true); + expect(isGameableLeg("src/__tests__/drift/helpers.ts")).toBe(true); + expect(isGameableLeg("src/__tests__/drift/openai-chat.drift.ts")).toBe(true); + // Dual-classified: harness membership wins over legit-target (fix #2). + expect(isGameableLeg("src/__tests__/drift/voice-models.ts")).toBe(true); + // Pure legit fixture targets are NOT gameable legs. + expect(isGameableLeg("src/__tests__/drift/model-registry.ts")).toBe(false); + expect(isGameableLeg("src/__tests__/drift/model-family.ts")).toBe(false); + expect(isGameableLeg("src/helpers.ts")).toBe(false); + }); + + it("isAllowlisted: production source is allowed; config/manifests/tests are not", () => { + const targets = new Set(["src/helpers.ts", "src/__tests__/drift/model-registry.ts"]); + // Production source (non-test) is always allowlisted. + expect(isAllowlisted("src/helpers.ts", targets)).toBe(true); + expect(isAllowlisted("src/agui-types.ts", targets)).toBe(true); + // A report-named fixture target is allowlisted. + expect(isAllowlisted("src/__tests__/drift/model-registry.ts", targets)).toBe(true); + // A fixture NOT named by the report is not allowlisted. + expect(isAllowlisted("src/__tests__/drift/model-family.ts", targets)).toBe(false); + // Config/manifests/lockfiles/unknown paths are not allowlisted. + expect(isAllowlisted("package.json", targets)).toBe(false); + expect(isAllowlisted("pnpm-lock.yaml", targets)).toBe(false); + expect(isAllowlisted("tsconfig.json", targets)).toBe(false); + expect(isAllowlisted("scripts/fix-drift.ts", targets)).toBe(false); + // A non-production src config-ish manifest is not allowlisted. + expect(isAllowlisted("src/__tests__/server.test.ts", targets)).toBe(false); + }); + + it("sanctionedTargets unions builderFile + non-null typesFile", () => { + const t = sanctionedTargets( + report([ + entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" }), + entry({ builderFile: "src/gemini.ts", typesFile: null }), + ]), + ); + expect(t.has("src/helpers.ts")).toBe(true); + expect(t.has("src/types.ts")).toBe(true); + expect(t.has("src/gemini.ts")).toBe(true); + expect(t.has("src/__tests__/drift/sdk-shapes.ts")).toBe(false); + }); + + it("countCriticalDiffs counts only critical severities", () => { + const r = report([ + entry({ + diffs: [ + diff({ severity: "critical" }), + diff({ severity: "warning" }), + diff({ severity: "critical" }), + ], + }), + ]); + expect(countCriticalDiffs(r)).toBe(2); + }); +}); + +// --------------------------------------------------------------------------- +// CLI arg parsing + config errors (exit 2) +// --------------------------------------------------------------------------- + +describe("parseCliArgs", () => { + it("parses a full valid arg set", () => { + const args = parseCliArgs([ + "--report", + "a.json", + "--post-fix-report", + "b.json", + "--post-fix-exit", + "0", + "--changed-file", + "src/helpers.ts", + "--changed-file", + "src/types.ts", + ]); + expect(args.reportPath).toBe("a.json"); + expect(args.postFixReportPath).toBe("b.json"); + expect(args.postFixExit).toBe(0); + expect(args.changedFiles).toEqual(["src/helpers.ts", "src/types.ts"]); + }); + + it("null changedFiles when no --changed-file flag (derive from git later)", () => { + const args = parseCliArgs([ + "--report", + "a.json", + "--post-fix-report", + "b.json", + "--post-fix-exit", + "2", + ]); + expect(args.changedFiles).toBeNull(); + }); + + it("throws on missing --report", () => { + expect(() => parseCliArgs(["--post-fix-report", "b.json", "--post-fix-exit", "0"])).toThrow( + PredicateConfigError, + ); + }); + + it("throws on non-integer --post-fix-exit", () => { + expect(() => + parseCliArgs(["--report", "a.json", "--post-fix-report", "b.json", "--post-fix-exit", "abc"]), + ).toThrow(PredicateConfigError); + }); + + it("throws on unknown argument", () => { + expect(() => parseCliArgs(["--nope"])).toThrow(PredicateConfigError); + }); +}); + +describe("readReport", () => { + let dir: string | null = null; + afterEach(() => { + if (dir) rmSync(dir, { recursive: true, force: true }); + dir = null; + }); + + it("throws PredicateConfigError on a missing file", () => { + expect(() => readReport("/no/such/drift-report.json")).toThrow(PredicateConfigError); + }); + + // FIX #6 — align the strict/loose validators. readReport must fail-closed on a + // structurally-untrustworthy report the same way fix-drift.ts:readDriftReport + // does: a missing/non-string `timestamp` is a corrupt/truncated collector run + // and must be REJECTED, never silently trusted as a clean signal. + it("rejects a report missing the timestamp field (fail-closed, aligns with readDriftReport)", () => { + dir = mkdtempSync(join(tmpdir(), "ws2-readreport-")); + const p = join(dir, "no-ts.json"); + writeFileSync(p, JSON.stringify({ entries: [] }), "utf-8"); + expect(() => readReport(p)).toThrow(PredicateConfigError); + }); + + it("rejects a non-object / non-{entries:[]} structure", () => { + dir = mkdtempSync(join(tmpdir(), "ws2-readreport-")); + const p = join(dir, "bad.json"); + writeFileSync(p, JSON.stringify({ timestamp: "t", entries: "nope" }), "utf-8"); + expect(() => readReport(p)).toThrow(PredicateConfigError); + }); + + // A legitimately-clean post-fix report IS { entries: [] } — the collector + // emits exactly that when no drift remains. It must be ACCEPTED (the trust + // anchor for "clean" is the collector EXIT CODE, corroborated by fix #1's + // always-block-on-leg-edit rule, not the entries array being non-empty). + it("accepts a well-formed EMPTY report (the genuine clean-collector signal)", () => { + dir = mkdtempSync(join(tmpdir(), "ws2-readreport-")); + const p = join(dir, "clean.json"); + writeFileSync(p, JSON.stringify({ timestamp: "t", entries: [] }), "utf-8"); + expect(() => readReport(p)).not.toThrow(); + }); + + it("accepts a well-formed non-empty report", () => { + dir = mkdtempSync(join(tmpdir(), "ws2-readreport-")); + const p = join(dir, "ok.json"); + writeFileSync(p, JSON.stringify(report()), "utf-8"); + expect(() => readReport(p)).not.toThrow(); + }); +}); + +describe("parsePorcelainLine", () => { + it("strips the 2-char status + space prefix", () => { + expect(parsePorcelainLine(" M src/helpers.ts")).toBe("src/helpers.ts"); + expect(parsePorcelainLine("?? src/new.ts")).toBe("src/new.ts"); + expect(parsePorcelainLine("A src/added.ts")).toBe("src/added.ts"); + }); + + it("takes the NEW path from rename notation (old -> new)", () => { + expect(parsePorcelainLine("R src/old.ts -> src/new.ts")).toBe("src/new.ts"); + }); + + it("unquotes paths with special characters", () => { + expect(parsePorcelainLine('?? "src/weird name.ts"')).toBe("src/weird name.ts"); + expect(parsePorcelainLine('R "src/old x.ts" -> "src/new x.ts"')).toBe("src/new x.ts"); + }); +}); + +// --------------------------------------------------------------------------- +// runCli end-to-end (in-process): exit codes over real temp report files +// --------------------------------------------------------------------------- + +describe("runCli", () => { + let dir: string | null = null; + + afterEach(() => { + if (dir) rmSync(dir, { recursive: true, force: true }); + dir = null; + }); + + function writeReports(pre: DriftReport, post: DriftReport): { pre: string; post: string } { + dir = mkdtempSync(join(tmpdir(), "ws2-predicate-")); + const preP = join(dir, "drift-report.json"); + const postP = join(dir, "drift-report.post-fix.json"); + writeFileSync(preP, JSON.stringify(pre), "utf-8"); + writeFileSync(postP, JSON.stringify(post), "utf-8"); + return { pre: preP, post: postP }; + } + + // NOTE (fix #4): runCli now ALWAYS cross-checks any --changed-file list + // against the real git working tree, so a synthetic list that does not match + // the checkout is rejected (exit 2) BEFORE the predicate runs. The exit-code- + // per-reason mapping for the cheat / real-fix cases is covered directly by the + // evaluateDriftResolved + REASON_EXIT_CODE tests above; here we lock the + // cross-check itself (the fix #4 blinding guard) at the runCli boundary. + it("exits 2 (CONFIG_ERROR) when --changed-file disagrees with git (fix #4 blinding guard)", () => { + const paths = writeReports(report(), report([])); + const code = runCli([ + "--report", + paths.pre, + "--post-fix-report", + paths.post, + "--post-fix-exit", + "0", + "--changed-file", + "src/__tests__/drift/sdk-shapes.ts", + ]); + expect(code).toBe(2); + }); + + it("exits 2 (CONFIG_ERROR) on a missing post-fix report", () => { + const paths = writeReports(report(), report()); + const code = runCli([ + "--report", + paths.pre, + "--post-fix-report", + join(dir!, "does-not-exist.json"), + "--post-fix-exit", + "0", + "--changed-file", + "src/helpers.ts", + ]); + expect(code).toBe(2); + }); + + it("exits 2 (CONFIG_ERROR) on malformed args", () => { + const code = runCli(["--bogus"]); + expect(code).toBe(2); + }); +}); + +// --------------------------------------------------------------------------- +// FIX #4 — authoritative changed-files: crossCheckChangedFiles +// --------------------------------------------------------------------------- + +describe("crossCheckChangedFiles", () => { + it("returns the git set when no explicit list is supplied", () => { + const git = ["src/helpers.ts", "src/types.ts"]; + expect(crossCheckChangedFiles(null, git)).toEqual(git); + }); + + it("accepts an explicit list that matches the git set (order-independent)", () => { + const git = ["src/helpers.ts", "src/types.ts"]; + expect(crossCheckChangedFiles(["src/types.ts", "src/helpers.ts"], git)).toEqual( + expect.arrayContaining(git), + ); + }); + + it("throws when the explicit list OMITS a file git reports (leg-blinding attack)", () => { + const git = ["src/helpers.ts", "src/__tests__/drift/sdk-shapes.ts"]; + // Attacker passes only the benign production file, hiding the relaxed leg. + expect(() => crossCheckChangedFiles(["src/helpers.ts"], git)).toThrow(PredicateConfigError); + }); + + it("throws when the explicit list ADDS a file git does not report", () => { + const git = ["src/helpers.ts"]; + expect(() => crossCheckChangedFiles(["src/helpers.ts", "src/phantom.ts"], git)).toThrow( + PredicateConfigError, + ); + }); +}); + +// --------------------------------------------------------------------------- +// REVERT GUARD + RED-COUNT LOCK +// --------------------------------------------------------------------------- + +describe("revert guard — old always-accept predicate would FAIL these locks", () => { + // The pre-hardening predicate ignored gameable legs once ANY production file + // changed (and had a legacy always-accept fallback). This models that old + // behavior; asserting it DISAGREES with the real predicate on the WS-2b cheat + // proves the hardening is load-bearing. If someone reverts evaluateDriftResolved + // to the old logic, the HEADLINE WS-2b test above flips and the suite breaks. + function oldPredicateWouldAccept(changedFiles: string[]): boolean { + const productionFiles = changedFiles.filter(isProductionFile); + // Old logic: leg files only checked when productionFiles === 0. + return productionFiles.length > 0; + } + + it("WS-2b cheat: old predicate ACCEPTS, real predicate BLOCKS (SUPPRESSION_SUSPECTED)", () => { + const cheat = ["src/__tests__/drift/sdk-shapes.ts", "src/helpers.ts"]; + // Old behavior would have accepted (a production file is present). + expect(oldPredicateWouldAccept(cheat)).toBe(true); + // Real predicate blocks. + const verdict = evaluateDriftResolved({ + changedFiles: cheat, + report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.SUPPRESSION_SUSPECTED); + }); + + // ALLOWLIST-INVERSION revert lock: the OLD denylist only blocked KNOWN gameable + // legs, so an in-diff vector NOT on the denylist (package.json, a lockfile, an + // unknown path) paired with an on-target production edit sailed through to + // RESOLVED. The allowlist inverts that: anything not explicitly allowed blocks. + // Reverting evaluateDriftResolved to a denylist model flips these to RESOLVED + // and breaks the suite. + function denylistWouldAccept(changedFiles: string[]): boolean { + // Old denylist model: block ONLY if a changed file is a known gameable leg; + // otherwise (production + package.json/lockfile/unknown) accept. + return !changedFiles.some(isGameableLeg) && changedFiles.some(isProductionFile); + } + + it("in-diff vector (package.json + prod): old DENYLIST accepts, allowlist BLOCKS (UNSANCTIONED_CHANGE)", () => { + const vector = ["src/helpers.ts", "package.json"]; + // The old denylist would have accepted (no known leg in the set). + expect(denylistWouldAccept(vector)).toBe(true); + const verdict = evaluateDriftResolved({ + changedFiles: vector, + report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.UNSANCTIONED_CHANGE); + }); +}); + +describe("exit-code distinctness lock", () => { + // Structural lock on the reason→exit-code contract: every block reason maps to + // a distinct NON-ZERO exit code and RESOLVED alone is 0. This does NOT count + // RED test cases (it asserts the code table, not the number of scenarios) — it + // guarantees the workflow can route each cause to its own Slack DETAIL without + // two reasons colliding on one exit code. + const BLOCK_REASONS = [ + PredicateReason.NO_PRODUCTION_CHANGE, + PredicateReason.COMPARISON_LEG_ONLY, + PredicateReason.SUPPRESSION_SUSPECTED, + PredicateReason.UNSANCTIONED_CHANGE, + PredicateReason.STILL_DIRTY, + PredicateReason.QUARANTINE_AFTER_FIX, + PredicateReason.COLLECTOR_INFRA, + PredicateReason.PRODUCTION_CHANGE_OFF_TARGET, + PredicateReason.CONFIG_ERROR, + ]; + + it("every block reason has a distinct non-zero exit code and RESOLVED is 0", () => { + expect(REASON_EXIT_CODE[PredicateReason.RESOLVED]).toBe(0); + const codes = BLOCK_REASONS.map((r) => REASON_EXIT_CODE[r]); + for (const c of codes) expect(c).not.toBe(0); + expect(new Set(codes).size).toBe(codes.length); // all distinct + }); +}); + +// --------------------------------------------------------------------------- +// CR round-3 slot-3 LOW gaps — unrecognized collector exit code + `..`/absolute +// canonicalization variants driven end-to-end through the predicate/runCli. +// --------------------------------------------------------------------------- + +describe("unrecognized collector exit code fails closed → COLLECTOR_INFRA", () => { + // The `postFixCollectorExit !== 0` catch-all (COLLECTOR_INFRA / exit 15) had + // no locking test. Tests only fed 0/1/2/5. A future refactor could let an + // unknown exit fall through to a clean accept — this pins it closed. + for (const exit of [3, 7, 99]) { + it(`exit ${exit} (unrecognized) with a production change → COLLECTOR_INFRA / exit 15`, () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts"], + report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), + postFixCollectorExit: exit, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.COLLECTOR_INFRA); + expect(REASON_EXIT_CODE[verdict.reason]).toBe(15); + }); + } +}); + +describe("`..`-segment and absolute path variants block/fail-closed THROUGH the predicate", () => { + it("a `..`-containing spelling of the SDK leg canonicalizes and BLOCKS (COMPARISON_LEG_ONLY)", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/__tests__/drift/foo/../sdk-shapes.ts"], + report: report(), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.COMPARISON_LEG_ONLY); + }); + + it("an absolute changed-file path throws PredicateConfigError from evaluateDriftResolved", () => { + expect(() => + evaluateDriftResolved({ + changedFiles: ["/etc/passwd"], + report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }), + ).toThrow(PredicateConfigError); + }); + + it("a `..`-escaping changed-file path throws PredicateConfigError from evaluateDriftResolved", () => { + expect(() => + evaluateDriftResolved({ + changedFiles: ["src/../../etc/passwd"], + report: report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]), + postFixCollectorExit: 0, + postFixCriticalCount: 0, + }), + ).toThrow(PredicateConfigError); + }); +}); + +describe("runCli maps a repo-escaping/absolute changed-file to CONFIG_ERROR (exit 2)", () => { + let dir: string | null = null; + afterEach(() => { + if (dir) rmSync(dir, { recursive: true, force: true }); + dir = null; + }); + + // Drive an absolute/`..`-escaping path all the way through runCli. A supplied + // --changed-file cross-checks against git first, so an absolute path that git + // never reports would be rejected by the cross-check (still exit 2). To pin the + // fix #8 canonicalize-throw path specifically, we assert the CONFIG_ERROR exit. + it("runCli exits 2 (CONFIG_ERROR) when a --changed-file is an absolute path", () => { + dir = mkdtempSync(join(tmpdir(), "ws2-canon-")); + const preP = join(dir, "drift-report.json"); + const postP = join(dir, "drift-report.post-fix.json"); + writeFileSync(preP, JSON.stringify(report()), "utf-8"); + writeFileSync(postP, JSON.stringify(report([])), "utf-8"); + const code = runCli([ + "--report", + preP, + "--post-fix-report", + postP, + "--post-fix-exit", + "0", + "--changed-file", + "/etc/passwd", + ]); + expect(code).toBe(REASON_EXIT_CODE[PredicateReason.CONFIG_ERROR]); + }); +}); + +// --------------------------------------------------------------------------- +// FIX #F3 (round-4) — a collector-output artifact left in the repo working tree +// (drift-report.post-fix.json / drift-report.json / claude-code-output.log) +// appears as an untracked file in `git status --porcelain` and is scored by the +// predicate as UNSANCTIONED_CHANGE, breaking the happy path. The workflow fix +// writes those artifacts to $RUNNER_TEMP (outside the checkout) so they never +// enter the changed-file set. These pure-function locks prove the predicate's +// behaviour on BOTH sides of that move. +// --------------------------------------------------------------------------- +describe("FIX #F3 — collector-output artifacts must not be in the changed-file set", () => { + const sanctioned = () => + report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]); + const cleanSignal = { postFixCollectorExit: 0, postFixCriticalCount: 0 }; + + it("RED (the bug): a post-fix report artifact in the changed set → UNSANCTIONED_CHANGE (happy path broken)", () => { + // This is exactly what happened when the re-collect wrote into the repo cwd: + // the untracked drift-report.post-fix.json joined the changed set and, being + // neither production source nor a report-named target, fail-closed the run. + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts", "drift-report.post-fix.json"], + report: sanctioned(), + ...cleanSignal, + }); + expect(verdict.resolved).toBe(false); + expect(verdict.reason).toBe(PredicateReason.UNSANCTIONED_CHANGE); + expect(verdict.offendingFiles).toContain("drift-report.post-fix.json"); + }); + + it("GREEN (the fix): the SAME production fix WITHOUT the artifact (now in $RUNNER_TEMP) → RESOLVED", () => { + const verdict = evaluateDriftResolved({ + changedFiles: ["src/helpers.ts"], + report: sanctioned(), + ...cleanSignal, + }); + expect(verdict.resolved).toBe(true); + expect(verdict.reason).toBe(PredicateReason.RESOLVED); + }); +}); + +// --------------------------------------------------------------------------- +// TEST-TIGHTNESS (round-4 slot-3) — lock the runCli behaviour the workflow +// depends on: (F2) the machine-readable `reason=` stdout line the +// "Assert" step greps for Slack routing, and (F1) the CONFIG_ERROR path so +// deleting the parse-error catch fails a test. These drive the REAL runCli in an +// isolated temp git repo so gitChangedFiles() returns a controlled set. +// --------------------------------------------------------------------------- +describe("runCli machine-readable reason= line + CONFIG_ERROR lock (slot-3 F1/F2)", () => { + let repo: string | null = null; + let logSpy: ReturnType | null = null; + let errSpy: ReturnType | null = null; + const origCwd = process.cwd(); + + afterEach(() => { + process.chdir(origCwd); + logSpy?.mockRestore(); + errSpy?.mockRestore(); + logSpy = null; + errSpy = null; + if (repo) rmSync(repo, { recursive: true, force: true }); + repo = null; + }); + + function initRepo(): string { + const dir = mkdtempSync(join(tmpdir(), "ws2-runcli-")); + execFileSync("git", ["init", "-q"], { cwd: dir }); + execFileSync("git", ["config", "user.email", "t@t"], { cwd: dir }); + execFileSync("git", ["config", "user.name", "t"], { cwd: dir }); + return dir; + } + + function captureConsole(): void { + logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + } + + function stdoutLines(): string[] { + return (logSpy?.mock.calls ?? []).map((c) => String(c[0])); + } + + it("prints `reason=resolved` on a genuine RESOLVED verdict (the line the workflow greps)", () => { + repo = initRepo(); + // A real production change in the working tree → gitChangedFiles() reports it. + // `git add -N` (intent-to-add) makes porcelain list the INDIVIDUAL file + // (`A src/helpers.ts`) rather than collapsing an all-untracked dir to `?? src/`. + mkdirSync(join(repo, "src"), { recursive: true }); + writeFileSync(join(repo, "src", "helpers.ts"), "export const x = 1;\n", "utf-8"); + execFileSync("git", ["add", "-N", "src/helpers.ts"], { cwd: repo }); + // Report files live OUTSIDE the repo (mirrors the workflow's $RUNNER_TEMP, + // FIX #F3) so they are NOT untracked entries in `git status --porcelain` and + // do not pollute the changed-file set the predicate scores. + const outDir = mkdtempSync(join(tmpdir(), "ws2-runcli-out-")); + const rep = report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]); + const preP = join(outDir, "pre.json"); + const postP = join(outDir, "post.json"); + writeFileSync(preP, JSON.stringify(rep), "utf-8"); + writeFileSync(postP, JSON.stringify(report([])), "utf-8"); + process.chdir(repo); + captureConsole(); + try { + const code = runCli(["--report", preP, "--post-fix-report", postP, "--post-fix-exit", "0"]); + expect(code).toBe(0); + expect(stdoutLines()).toContain(`reason=${PredicateReason.RESOLVED}`); + } finally { + rmSync(outDir, { recursive: true, force: true }); + } + }); + + it("prints `reason=unsanctioned-change` (non-zero) on a blocked verdict — the Slack routing key", () => { + repo = initRepo(); + mkdirSync(join(repo, "src"), { recursive: true }); + writeFileSync(join(repo, "src", "helpers.ts"), "export const x = 1;\n", "utf-8"); + // package.json IN the repo is the unsanctioned change; reports live OUTSIDE. + writeFileSync(join(repo, "package.json"), '{"name":"x"}\n', "utf-8"); + execFileSync("git", ["add", "-N", "src/helpers.ts", "package.json"], { cwd: repo }); + const outDir = mkdtempSync(join(tmpdir(), "ws2-runcli-out-")); + const rep = report([entry({ builderFile: "src/helpers.ts", typesFile: "src/types.ts" })]); + const preP = join(outDir, "pre.json"); + const postP = join(outDir, "post.json"); + writeFileSync(preP, JSON.stringify(rep), "utf-8"); + writeFileSync(postP, JSON.stringify(report([])), "utf-8"); + process.chdir(repo); + captureConsole(); + try { + const code = runCli(["--report", preP, "--post-fix-report", postP, "--post-fix-exit", "0"]); + expect(code).toBe(REASON_EXIT_CODE[PredicateReason.UNSANCTIONED_CHANGE]); + expect(stdoutLines()).toContain(`reason=${PredicateReason.UNSANCTIONED_CHANGE}`); + } finally { + rmSync(outDir, { recursive: true, force: true }); + } + }); + + it("prints `reason=config-error` and exits 2 when the report is unreadable (CONFIG_ERROR lock)", () => { + repo = initRepo(); + process.chdir(repo); + captureConsole(); + const code = runCli([ + "--report", + join(repo, "does-not-exist.json"), + "--post-fix-report", + join(repo, "also-missing.json"), + "--post-fix-exit", + "0", + ]); + expect(code).toBe(REASON_EXIT_CODE[PredicateReason.CONFIG_ERROR]); + expect(stdoutLines()).toContain(`reason=${PredicateReason.CONFIG_ERROR}`); + }); +}); + +// --------------------------------------------------------------------------- +// CR round-3 F3 — readReport ENTRY-LEVEL validation aligned with +// fix-drift.ts:readDriftReport. A structurally-valid report whose entries are +// malformed at the fields the predicate reads must fail-closed with a DISTINCT, +// NAMED PredicateConfigError (not a bare TypeError caught as an unnamed error). +// --------------------------------------------------------------------------- + +describe("readReport entry-level validation (F3)", () => { + let dir: string | null = null; + afterEach(() => { + if (dir) rmSync(dir, { recursive: true, force: true }); + dir = null; + }); + + function writeAndRead(obj: unknown): void { + dir = mkdtempSync(join(tmpdir(), "ws2-f3-")); + const p = join(dir, "r.json"); + writeFileSync(p, JSON.stringify(obj), "utf-8"); + readReport(p); + } + + it("throws a named PredicateConfigError when an entry is missing its diffs array", () => { + expect(() => + writeAndRead({ + timestamp: "t", + entries: [{ provider: "OpenAI", builderFile: "src/helpers.ts", typesFile: null }], + }), + ).toThrow(PredicateConfigError); + }); + + it('throws a named PredicateConfigError when "diffs" is present via the message', () => { + expect(() => + writeAndRead({ + timestamp: "t", + entries: [{ provider: "OpenAI", builderFile: "src/helpers.ts", typesFile: null }], + }), + ).toThrow(/diffs/); + }); + + it("throws when an entry has a non-string builderFile (cannot derive sanctioned set)", () => { + expect(() => + writeAndRead({ + timestamp: "t", + entries: [{ provider: "OpenAI", builderFile: 123, typesFile: null, diffs: [] }], + }), + ).toThrow(PredicateConfigError); + }); + + it("throws when an entry has an empty builderFile", () => { + expect(() => + writeAndRead({ + timestamp: "t", + entries: [{ provider: "OpenAI", builderFile: "", typesFile: null, diffs: [] }], + }), + ).toThrow(/builderFile/); + }); + + it("throws when an entry has a numeric typesFile (must be string or null)", () => { + expect(() => + writeAndRead({ + timestamp: "t", + entries: [{ provider: "OpenAI", builderFile: "src/helpers.ts", typesFile: 42, diffs: [] }], + }), + ).toThrow(/typesFile/); + }); + + it("throws when an entry is not an object", () => { + expect(() => writeAndRead({ timestamp: "t", entries: [null] })).toThrow(PredicateConfigError); + }); + + it("still ACCEPTS a well-formed report (empty entries + valid entries both OK)", () => { + expect(() => writeAndRead({ timestamp: "t", entries: [] })).not.toThrow(); + expect(() => + writeAndRead({ + timestamp: "t", + entries: [ + { provider: "OpenAI", builderFile: "src/helpers.ts", typesFile: null, diffs: [] }, + ], + }), + ).not.toThrow(); + }); +}); diff --git a/src/__tests__/drift/anthropic.drift.ts b/src/__tests__/drift/anthropic.drift.ts index 8c888599..052acca5 100644 --- a/src/__tests__/drift/anthropic.drift.ts +++ b/src/__tests__/drift/anthropic.drift.ts @@ -58,7 +58,7 @@ describe.skipIf(!ANTHROPIC_API_KEY)("Anthropic Claude Messages drift", () => { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); - const report = formatDriftReport("Anthropic Claude (non-streaming text)", diffs); + const report = formatDriftReport("Anthropic Claude (non-streaming text)", diffs, "anthropic"); expect( diffs.filter((d) => d.severity === "critical"), @@ -90,7 +90,11 @@ describe.skipIf(!ANTHROPIC_API_KEY)("Anthropic Claude Messages drift", () => { })); const diffs = compareSSESequences(sdkEvents, realStream.events, mockSSEShapes); - const report = formatDriftReport("Anthropic Claude (streaming text events)", diffs); + const report = formatDriftReport( + "Anthropic Claude (streaming text events)", + diffs, + "anthropic", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -128,7 +132,11 @@ describe.skipIf(!ANTHROPIC_API_KEY)("Anthropic Claude Messages drift", () => { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); - const report = formatDriftReport("Anthropic Claude (non-streaming tool call)", diffs); + const report = formatDriftReport( + "Anthropic Claude (non-streaming tool call)", + diffs, + "anthropic", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -179,7 +187,11 @@ describe.skipIf(!ANTHROPIC_API_KEY)("Anthropic Claude Messages drift", () => { })); const diffs = compareSSESequences(sdkEvents, realStream.events, mockSSEShapes); - const report = formatDriftReport("Anthropic Claude (streaming tool call events)", diffs); + const report = formatDriftReport( + "Anthropic Claude (streaming tool call events)", + diffs, + "anthropic", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -221,7 +233,11 @@ describe("Anthropic Claude extended thinking shapes", () => { // Shape triangulation (mock-only, no real API call for thinking) const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Anthropic Claude (non-streaming thinking)", diffs); + const report = formatDriftReport( + "Anthropic Claude (non-streaming thinking)", + diffs, + "anthropic", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -283,7 +299,11 @@ describe("Anthropic Claude extended thinking shapes", () => { })); const diffs = compareSSESequences(sdkEvents, sdkEvents, mockSSEShapes); - const report = formatDriftReport("Anthropic Claude (streaming thinking events)", diffs); + const report = formatDriftReport( + "Anthropic Claude (streaming thinking events)", + diffs, + "anthropic", + ); expect( diffs.filter((d) => d.severity === "critical"), diff --git a/src/__tests__/drift/bedrock-stream.drift.ts b/src/__tests__/drift/bedrock-stream.drift.ts index 55cc55f7..b0fc0705 100644 --- a/src/__tests__/drift/bedrock-stream.drift.ts +++ b/src/__tests__/drift/bedrock-stream.drift.ts @@ -201,7 +201,7 @@ describe.skipIf(!HAS_CREDENTIALS)("Bedrock drift", () => { if (mockRes.status === 200) { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Bedrock Invoke", diffs); + const report = formatDriftReport("Bedrock Invoke", diffs, "bedrock-invoke"); expect( diffs.filter((d) => d.severity === "critical"), @@ -261,7 +261,11 @@ describe.skipIf(!HAS_CREDENTIALS)("Bedrock drift", () => { if (!mockEvent) continue; // already asserted presence above const diffs = triangulate(sdkEvent.dataShape, sdkEvent.dataShape, mockEvent.dataShape); - const report = formatDriftReport(`Bedrock InvokeStream:${sdkEvent.type}`, diffs); + const report = formatDriftReport( + `Bedrock InvokeStream:${sdkEvent.type}`, + diffs, + "bedrock-invoke-stream", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -291,7 +295,7 @@ describe.skipIf(!HAS_CREDENTIALS)("Bedrock drift", () => { if (mockRes.status === 200) { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Bedrock Converse", diffs); + const report = formatDriftReport("Bedrock Converse", diffs, "bedrock-converse"); expect( diffs.filter((d) => d.severity === "critical"), @@ -397,7 +401,11 @@ describe.skipIf(!HAS_CREDENTIALS)("Bedrock drift", () => { if (!mockEvent) continue; // already asserted presence above const diffs = triangulate(sdkEvent.dataShape, sdkEvent.dataShape, mockEvent.dataShape); - const report = formatDriftReport(`Bedrock ConverseStream:${sdkEvent.type}`, diffs); + const report = formatDriftReport( + `Bedrock ConverseStream:${sdkEvent.type}`, + diffs, + "bedrock-converse-stream", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -482,7 +490,11 @@ describe.skipIf(!HAS_CREDENTIALS)("Bedrock drift", () => { if (!mockEvent) continue; const diffs = triangulate(sdkEvent.dataShape, sdkEvent.dataShape, mockEvent.dataShape); - const report = formatDriftReport(`Bedrock ConverseStream Tool:${sdkEvent.type}`, diffs); + const report = formatDriftReport( + `Bedrock ConverseStream Tool:${sdkEvent.type}`, + diffs, + "bedrock-converse-stream", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -597,6 +609,7 @@ describe.skipIf(!HAS_CREDENTIALS)("Bedrock drift", () => { const report = formatDriftReport( `Bedrock ConverseStream Reasoning:${sdkEvent.type}`, diffs, + "bedrock-converse-stream", ); expect( diff --git a/src/__tests__/drift/cohere.drift.ts b/src/__tests__/drift/cohere.drift.ts index e5d71f30..3b882d99 100644 --- a/src/__tests__/drift/cohere.drift.ts +++ b/src/__tests__/drift/cohere.drift.ts @@ -156,7 +156,7 @@ describe("Cohere error shapes", () => { const mockShape = extractShape(body); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Cohere /v2/chat malformed JSON error", diffs); + const report = formatDriftReport("Cohere /v2/chat malformed JSON error", diffs, "cohere-chat"); expect( diffs.filter((d) => d.severity === "critical"), @@ -176,7 +176,7 @@ describe("Cohere error shapes", () => { const mockShape = extractShape(body); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Cohere /v2/chat missing model error", diffs); + const report = formatDriftReport("Cohere /v2/chat missing model error", diffs, "cohere-chat"); expect( diffs.filter((d) => d.severity === "critical"), @@ -196,7 +196,11 @@ describe("Cohere error shapes", () => { const mockShape = extractShape(body); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Cohere /v2/chat missing messages error", diffs); + const report = formatDriftReport( + "Cohere /v2/chat missing messages error", + diffs, + "cohere-chat", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -217,7 +221,11 @@ describe("Cohere error shapes", () => { const mockShape = extractShape(body); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Cohere /v2/chat no fixture match error", diffs); + const report = formatDriftReport( + "Cohere /v2/chat no fixture match error", + diffs, + "cohere-chat", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -248,7 +256,7 @@ describe.skipIf(!HAS_CREDENTIALS)("Cohere drift", () => { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); - const report = formatDriftReport("Cohere /v2/chat (non-streaming)", diffs); + const report = formatDriftReport("Cohere /v2/chat (non-streaming)", diffs, "cohere-chat"); expect( diffs.filter((d) => d.severity === "critical"), @@ -284,7 +292,11 @@ describe.skipIf(!HAS_CREDENTIALS)("Cohere drift", () => { const mockChunkShape = extractShape(mockChunks[0]); const diffs = triangulate(sdkChunkShape, realChunkShape, mockChunkShape); - const report = formatDriftReport("Cohere /v2/chat (streaming first chunk)", diffs); + const report = formatDriftReport( + "Cohere /v2/chat (streaming first chunk)", + diffs, + "cohere-chat", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -308,7 +320,11 @@ describe.skipIf(!HAS_CREDENTIALS)("Cohere drift", () => { const mockLastShape = extractShape(mockChunks[mockChunks.length - 1]); const lastDiffs = triangulate(sdkLastChunkShape, realLastShape, mockLastShape); - const lastReport = formatDriftReport("Cohere /v2/chat (streaming last chunk)", lastDiffs); + const lastReport = formatDriftReport( + "Cohere /v2/chat (streaming last chunk)", + lastDiffs, + "cohere-chat", + ); expect( lastDiffs.filter((d) => d.severity === "critical"), diff --git a/src/__tests__/drift/elevenlabs.drift.ts b/src/__tests__/drift/elevenlabs.drift.ts index 17f5f3bf..1da20ae9 100644 --- a/src/__tests__/drift/elevenlabs.drift.ts +++ b/src/__tests__/drift/elevenlabs.drift.ts @@ -174,7 +174,11 @@ describe("ElevenLabs drift — sound generation", () => { const mockShape = extractShape(body); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("ElevenLabs /v1/sound-generation 400 error", diffs); + const report = formatDriftReport( + "ElevenLabs /v1/sound-generation 400 error", + diffs, + "elevenlabs", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -241,7 +245,7 @@ describe("ElevenLabs drift — music endpoints", () => { // Three-way comparison: use SDK shape as both expected and real const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("ElevenLabs /v1/music/plan", diffs); + const report = formatDriftReport("ElevenLabs /v1/music/plan", diffs, "elevenlabs"); expect( diffs.filter((d) => d.severity === "critical"), @@ -265,7 +269,7 @@ describe("ElevenLabs drift — music endpoints", () => { const mockShape = extractShape(body); const diffs = triangulate(expectedShape, expectedShape, mockShape); - const report = formatDriftReport("ElevenLabs /v1/music 400 error", diffs); + const report = formatDriftReport("ElevenLabs /v1/music 400 error", diffs, "elevenlabs"); expect( diffs.filter((d) => d.severity === "critical"), diff --git a/src/__tests__/drift/fal-queue.drift.ts b/src/__tests__/drift/fal-queue.drift.ts index 0394961f..34e92340 100644 --- a/src/__tests__/drift/fal-queue.drift.ts +++ b/src/__tests__/drift/fal-queue.drift.ts @@ -106,7 +106,7 @@ describe("fal.ai queue lifecycle shapes", () => { // Shape comparison const mockShape = extractShape(envelope); const diffs = compareShapes(expectedShape, mockShape); - const report = formatDriftReport("fal.ai queue submit envelope", diffs); + const report = formatDriftReport("fal.ai queue submit envelope", diffs, "fal-queue"); expect( diffs.filter((d) => d.severity === "critical"), @@ -145,7 +145,7 @@ describe("fal.ai queue lifecycle shapes", () => { const mockShape = extractShape(body); const diffs = compareShapes(expectedShape, mockShape); - const report = formatDriftReport("fal.ai queue status", diffs); + const report = formatDriftReport("fal.ai queue status", diffs, "fal-queue"); expect( diffs.filter((d) => d.severity === "critical"), @@ -182,7 +182,7 @@ describe("fal.ai queue lifecycle shapes", () => { const mockShape = extractShape(body); const diffs = compareShapes(expectedShape, mockShape); - const report = formatDriftReport("fal.ai queue result", diffs); + const report = formatDriftReport("fal.ai queue result", diffs, "fal-queue"); expect( diffs.filter((d) => d.severity === "critical"), @@ -219,7 +219,7 @@ describe("fal.ai queue lifecycle shapes", () => { const mockShape = extractShape(body); const diffs = compareShapes(expectedShape, mockShape); - const report = formatDriftReport("fal.ai queue cancel", diffs); + const report = formatDriftReport("fal.ai queue cancel", diffs, "fal-queue"); expect( diffs.filter((d) => d.severity === "critical"), diff --git a/src/__tests__/drift/fal.drift.ts b/src/__tests__/drift/fal.drift.ts index 38fa0b49..5609a088 100644 --- a/src/__tests__/drift/fal.drift.ts +++ b/src/__tests__/drift/fal.drift.ts @@ -142,7 +142,7 @@ describe("fal.ai sync-run response shape", () => { // Two-way triangulation: expected shape is both the "SDK" and "real" reference // since fal passthrough should be identity. const diffs = triangulate(expectedShape, expectedShape, mockShape); - const report = formatDriftReport("fal.ai sync-run (image payload)", diffs); + const report = formatDriftReport("fal.ai sync-run (image payload)", diffs, "fal-sync"); expect( diffs.filter((d) => d.severity === "critical"), @@ -168,7 +168,7 @@ describe("fal.ai sync-run response shape", () => { // Shape match via triangulation const mockShape = extractShape(parsed); const diffs = triangulate(expectedShape, expectedShape, mockShape); - const report = formatDriftReport("fal.ai sync-run (flat payload)", diffs); + const report = formatDriftReport("fal.ai sync-run (flat payload)", diffs, "fal-sync"); expect( diffs.filter((d) => d.severity === "critical"), diff --git a/src/__tests__/drift/gemini-interactions.drift.ts b/src/__tests__/drift/gemini-interactions.drift.ts index f29ab01f..81cfd7ac 100644 --- a/src/__tests__/drift/gemini-interactions.drift.ts +++ b/src/__tests__/drift/gemini-interactions.drift.ts @@ -77,7 +77,11 @@ describe.skipIf(!GOOGLE_API_KEY)("Gemini Interactions API drift", () => { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); - const report = formatDriftReport("Gemini Interactions (non-streaming text)", diffs); + const report = formatDriftReport( + "Gemini Interactions (non-streaming text)", + diffs, + "gemini-interactions", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -120,6 +124,7 @@ describe.skipIf(!GOOGLE_API_KEY)("Gemini Interactions API drift", () => { const report = formatDriftReport( "Gemini Interactions (non-streaming text, Step[] input)", diffs, + "gemini-interactions", ); expect( @@ -162,7 +167,11 @@ describe.skipIf(!GOOGLE_API_KEY)("Gemini Interactions API drift", () => { })); const diffs = compareSSESequences(sdkEvents, realStream.events, mockSSEShapes); - const report = formatDriftReport("Gemini Interactions (streaming text events)", diffs); + const report = formatDriftReport( + "Gemini Interactions (streaming text events)", + diffs, + "gemini-interactions", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -218,7 +227,11 @@ describe.skipIf(!GOOGLE_API_KEY)("Gemini Interactions API drift", () => { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); - const report = formatDriftReport("Gemini Interactions (non-streaming tool call)", diffs); + const report = formatDriftReport( + "Gemini Interactions (non-streaming tool call)", + diffs, + "gemini-interactions", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -276,7 +289,11 @@ describe.skipIf(!GOOGLE_API_KEY)("Gemini Interactions API drift", () => { })); const diffs = compareSSESequences(sdkEvents, realStream.events, mockSSEShapes); - const report = formatDriftReport("Gemini Interactions (streaming tool call events)", diffs); + const report = formatDriftReport( + "Gemini Interactions (streaming tool call events)", + diffs, + "gemini-interactions", + ); expect( diffs.filter((d) => d.severity === "critical"), diff --git a/src/__tests__/drift/gemini.drift.ts b/src/__tests__/drift/gemini.drift.ts index 95bd2686..6f5c7408 100644 --- a/src/__tests__/drift/gemini.drift.ts +++ b/src/__tests__/drift/gemini.drift.ts @@ -56,7 +56,7 @@ describe.skipIf(!GOOGLE_API_KEY)("Google Gemini drift", () => { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); - const report = formatDriftReport("Gemini (non-streaming text)", diffs); + const report = formatDriftReport("Gemini (non-streaming text)", diffs, "gemini"); expect( diffs.filter((d) => d.severity === "critical"), @@ -86,7 +86,7 @@ describe.skipIf(!GOOGLE_API_KEY)("Google Gemini drift", () => { const mockChunkShape = extractShape(mockChunks[0]); const diffs = triangulate(sdkChunkShape, realChunkShape, mockChunkShape); - const report = formatDriftReport("Gemini (streaming intermediate chunk)", diffs); + const report = formatDriftReport("Gemini (streaming intermediate chunk)", diffs, "gemini"); expect( diffs.filter((d) => d.severity === "critical"), @@ -99,7 +99,7 @@ describe.skipIf(!GOOGLE_API_KEY)("Google Gemini drift", () => { const mockLastShape = extractShape(mockChunks[mockChunks.length - 1]); const lastDiffs = triangulate(sdkLastShape, realLastShape, mockLastShape); - const lastReport = formatDriftReport("Gemini (streaming last chunk)", lastDiffs); + const lastReport = formatDriftReport("Gemini (streaming last chunk)", lastDiffs, "gemini"); expect( lastDiffs.filter((d) => d.severity === "critical"), @@ -140,7 +140,7 @@ describe.skipIf(!GOOGLE_API_KEY)("Google Gemini drift", () => { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); - const report = formatDriftReport("Gemini (non-streaming tool call)", diffs); + const report = formatDriftReport("Gemini (non-streaming tool call)", diffs, "gemini"); expect( diffs.filter((d) => d.severity === "critical"), @@ -186,7 +186,7 @@ describe.skipIf(!GOOGLE_API_KEY)("Google Gemini drift", () => { const mockLastShape = extractShape(mockChunks[mockChunks.length - 1]); const diffs = triangulate(sdkLastShape, realLastShape, mockLastShape); - const report = formatDriftReport("Gemini (streaming tool call)", diffs); + const report = formatDriftReport("Gemini (streaming tool call)", diffs, "gemini"); expect( diffs.filter((d) => d.severity === "critical"), @@ -288,7 +288,7 @@ describe("Gemini error shapes", () => { const mockShape = extractShape(body); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Gemini (RESOURCE_EXHAUSTED error)", diffs); + const report = formatDriftReport("Gemini (RESOURCE_EXHAUSTED error)", diffs, "gemini"); expect( diffs.filter((d) => d.severity === "critical"), @@ -318,7 +318,7 @@ describe("Gemini error shapes", () => { const mockShape = extractShape(body); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Gemini (NOT_FOUND error)", diffs); + const report = formatDriftReport("Gemini (NOT_FOUND error)", diffs, "gemini"); expect( diffs.filter((d) => d.severity === "critical"), @@ -344,7 +344,7 @@ describe("Gemini error shapes", () => { const mockShape = extractShape(body); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Gemini (INVALID_ARGUMENT error)", diffs); + const report = formatDriftReport("Gemini (INVALID_ARGUMENT error)", diffs, "gemini"); expect( diffs.filter((d) => d.severity === "critical"), @@ -392,7 +392,7 @@ describe("Gemini error shapes", () => { const mockShape = extractShape(body); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Gemini (no-fixture-match error)", diffs); + const report = formatDriftReport("Gemini (no-fixture-match error)", diffs, "gemini"); expect( diffs.filter((d) => d.severity === "critical"), @@ -433,7 +433,7 @@ describe("Gemini error shapes", () => { const mockShape = extractShape(body); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Gemini (malformed JSON error)", diffs); + const report = formatDriftReport("Gemini (malformed JSON error)", diffs, "gemini"); expect( diffs.filter((d) => d.severity === "critical"), @@ -486,7 +486,7 @@ describe("Gemini thinking token shapes", () => { // Shape comparison: SDK expected vs mock output const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Gemini (non-streaming thinking)", diffs); + const report = formatDriftReport("Gemini (non-streaming thinking)", diffs, "gemini"); expect( diffs.filter((d) => d.severity === "critical"), @@ -562,7 +562,11 @@ describe("Gemini thinking token shapes", () => { sdkThinkingChunkShape, mockThinkingShape, ); - const thinkingReport = formatDriftReport("Gemini (streaming thinking chunk)", thinkingDiffs); + const thinkingReport = formatDriftReport( + "Gemini (streaming thinking chunk)", + thinkingDiffs, + "gemini", + ); expect( thinkingDiffs.filter((d) => d.severity === "critical"), @@ -575,6 +579,7 @@ describe("Gemini thinking token shapes", () => { const contentReport = formatDriftReport( "Gemini (streaming content chunk after thinking)", contentDiffs, + "gemini", ); expect( diff --git a/src/__tests__/drift/images.drift.ts b/src/__tests__/drift/images.drift.ts index 0f44952c..029e9e51 100644 --- a/src/__tests__/drift/images.drift.ts +++ b/src/__tests__/drift/images.drift.ts @@ -116,7 +116,7 @@ describe("OpenAI Images API drift", () => { // Two-way: SDK vs mock (no real API call — DALL-E is expensive) const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("OpenAI Images (url variant)", diffs); + const report = formatDriftReport("OpenAI Images (url variant)", diffs, "images"); expect( diffs.filter((d) => d.severity === "critical"), @@ -138,7 +138,7 @@ describe("OpenAI Images API drift", () => { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("OpenAI Images (b64_json variant)", diffs); + const report = formatDriftReport("OpenAI Images (b64_json variant)", diffs, "images"); expect( diffs.filter((d) => d.severity === "critical"), @@ -163,7 +163,7 @@ describe("OpenAI Images API drift", () => { const mockShape = extractShape(parsed); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("OpenAI Images (multi-image)", diffs); + const report = formatDriftReport("OpenAI Images (multi-image)", diffs, "images"); expect( diffs.filter((d) => d.severity === "critical"), diff --git a/src/__tests__/drift/model-family.test.ts b/src/__tests__/drift/model-family.test.ts new file mode 100644 index 00000000..da230e95 --- /dev/null +++ b/src/__tests__/drift/model-family.test.ts @@ -0,0 +1,35 @@ +/** + * Unit test for the shared `normalizeModelFamily` primitive. + */ +import { describe, it, expect } from "vitest"; +import { normalizeModelFamily } from "./model-family.js"; + +describe("normalizeModelFamily", () => { + it("strips a trailing dated snapshot suffix", () => { + expect(normalizeModelFamily("gpt-audio-2025-08-28", "openai")).toBe("gpt-audio"); + }); + + it("strips a trailing build-tag suffix", () => { + expect(normalizeModelFamily("tts-1-1106", "openai")).toBe("tts-1"); + }); + + it("does not strip a single-digit suffix", () => { + expect(normalizeModelFamily("gpt-live-1", "openai")).toBe("gpt-live-1"); + }); + + it("strips a trailing Anthropic contiguous 8-digit snapshot for anthropic", () => { + expect(normalizeModelFamily("claude-3-5-sonnet-20241022", "anthropic")).toBe( + "claude-3-5-sonnet", + ); + expect(normalizeModelFamily("claude-3-7-sonnet-20250219", "anthropic")).toBe( + "claude-3-7-sonnet", + ); + }); + + it("does NOT strip a contiguous 8-digit tail for openai/gemini", () => { + // The 8-digit rule is Anthropic-only; a non-date 8-digit tail on another + // provider must survive so it is not silently over-stripped. + expect(normalizeModelFamily("gpt-weird-12345678", "openai")).toBe("gpt-weird-12345678"); + expect(normalizeModelFamily("gemini-weird-12345678", "gemini")).toBe("gemini-weird-12345678"); + }); +}); diff --git a/src/__tests__/drift/model-family.ts b/src/__tests__/drift/model-family.ts new file mode 100644 index 00000000..6489f64a --- /dev/null +++ b/src/__tests__/drift/model-family.ts @@ -0,0 +1,59 @@ +/** + * Shared, side-effect-free `normalizeModelFamily` primitive (no + * `describe`/`beforeAll`) reducing a model id to its FAMILY KEY by stripping the + * trailing version/snapshot suffixes that providers append to already-known + * families. + * + * New dated snapshots of an existing family land constantly (`tts-1-1106`, + * `gpt-audio-2025-08-28`, `gpt-4o-mini-tts-2025-12-15`, …); appending every one + * to a known-ID set never converges and turns the daily drift job permanently + * red on false positives. Comparing the NORMALIZED family instead means only a + * genuinely new family (e.g. `gpt-live`) is ever flagged. + * + * Two suffix shapes are stripped, repeatedly, from the END of the id for ALL + * providers (the SHARED CORE): + * - a dated snapshot `-YYYY-MM-DD` (e.g. `-2025-08-28`) + * - a build/version tag `-NNN` or `-NNNN` (3–4 digits, e.g. `-1106`) + * + * Both are anchored to the end and applied in a loop so a trailing dated + * snapshot that itself follows a build tag is fully reduced. A short numeric + * suffix like `gpt-live-1`'s trailing `-1` is a SINGLE digit and is deliberately + * NOT stripped, so `gpt-live-1` normalizes to `gpt-live-1` — an unknown family — + * and stays flagged (the whole point of the canary). + * + * The `provider` argument selects a per-provider EXTRA rule on top of the shared + * core: + * - `anthropic` additionally strips a CONTIGUOUS 8-digit snapshot `-YYYYMMDD` + * (e.g. `claude-3-5-sonnet-20241022` → `claude-3-5-sonnet`). Anthropic dates + * its ids with an undelimited `-YYYYMMDD` suffix rather than the dashed + * `-YYYY-MM-DD` form, so without this every dated Claude id would + * false-positive as drift (the incident-2 class, for Anthropic). + * - `openai` / `gemini` keep ONLY the shared core — the 8-digit strip is NOT + * applied to them, so a non-date 8-digit tail is never over-stripped, and + * `normalizeModelFamily(id, "openai")` stays byte-identical to the historical + * `normalizeVoiceModelFamily(id)`. + * + * The per-provider rule runs inside the SAME reduction loop as the shared core, + * so it stays idempotent: a dated snapshot following a build tag still fully + * reduces. + */ +const DATED_SNAPSHOT_SUFFIX = /-\d{4}-\d{2}-\d{2}$/; +const BUILD_TAG_SUFFIX = /-\d{3,4}$/; +/** Anthropic contiguous `-YYYYMMDD` snapshot suffix (undelimited date). */ +const ANTHROPIC_DATE_SUFFIX = /-\d{8}$/; + +export function normalizeModelFamily( + id: string, + provider: "openai" | "anthropic" | "gemini", +): string { + let family = id; + for (;;) { + let stripped = family.replace(DATED_SNAPSHOT_SUFFIX, "").replace(BUILD_TAG_SUFFIX, ""); + if (provider === "anthropic") { + stripped = stripped.replace(ANTHROPIC_DATE_SUFFIX, ""); + } + if (stripped === family) break; + family = stripped; + } + return family; +} diff --git a/src/__tests__/drift/model-registry.test.ts b/src/__tests__/drift/model-registry.test.ts new file mode 100644 index 00000000..53338cb3 --- /dev/null +++ b/src/__tests__/drift/model-registry.test.ts @@ -0,0 +1,235 @@ +/** + * Unit test for the per-provider model-family registry shape, plus the §6.1c + * builder/fixture cross-check. + * + * Part 1 (registry shape): the minimal invariant probe for model-registry.ts: + * - seeded families are normalization-consistent (membership works), + * - the provider-mode allowlist carries `gemini-interactions`, + * - no family appears in BOTH include and exclude for any provider. + * + * Part 2 (builder/fixture cross-check — §6.1c): every model id that aimock's + * builders and fixtures reference must resolve, via normalizeModelFamily, to a + * family that is either in includeFamilies for its provider (aimock mocks it) + * or in excludeFamilies (retired/non-text/preview). Non-model "provider mode" + * tokens (e.g. `gemini-interactions`) must appear on NON_MODEL_TOKENS. A stray + * builder model id or a misclassified prose token must fail this test — never a + * live crash in the drift job. + * + * The cross-check table is derived from: + * - `DEFAULT_MODELS` in src/server.ts (aimock's advertised /v1/models list) + * - Dated/versioned ids used in conformance and fixture-blocks tests + * - The `gemini-interactions` provider-mode token documented in README + * + * The table is intentionally static (no source scraping) so it exercises the + * EXACT normalizeModelFamily + registry membership path the live drift check + * relies on, with no live-key dependency. + */ +import { describe, it, expect } from "vitest"; +import { normalizeModelFamily } from "./model-family.js"; +import { + includeFamilies, + excludeFamilies, + NON_MODEL_TOKENS, + isClassifiedFamily, + PREVIEW_FAMILY, + GEMMA_FAMILY, +} from "./model-registry.js"; + +// ─── Part 1: registry shape invariants ─────────────────────────────────────── + +describe("model-registry", () => { + it("include contains the normalized family of a mocked model id", () => { + expect(includeFamilies.gemini.has(normalizeModelFamily("gemini-2.5-flash", "gemini"))).toBe( + true, + ); + }); + + it("allowlists the gemini-interactions provider-mode token", () => { + expect(NON_MODEL_TOKENS.has("gemini-interactions")).toBe(true); + }); + + it("no family appears in both include and exclude for any provider", () => { + for (const provider of ["openai", "anthropic", "gemini"] as const) { + const inc = includeFamilies[provider]; + const exc = excludeFamilies[provider]; + const overlap = [...inc].filter((f) => exc.has(f)); + expect( + overlap, + `provider ${provider} has families on both lists: ${overlap.join(", ")}`, + ).toEqual([]); + } + }); + + it("seeds are idempotent under normalization (already family keys)", () => { + for (const provider of ["openai", "anthropic", "gemini"] as const) { + for (const family of includeFamilies[provider]) { + expect(normalizeModelFamily(family, provider)).toBe(family); + } + for (const family of excludeFamilies[provider]) { + expect(normalizeModelFamily(family, provider)).toBe(family); + } + } + }); +}); + +// ─── PREVIEW_FAMILY exclude-by-rule predicate ──────────────────────────────── + +describe("isClassifiedFamily / PREVIEW_FAMILY", () => { + it("classifies a brand-new -preview family by rule (no registry entry)", () => { + const family = normalizeModelFamily("gemini-9-pro-preview", "gemini"); + expect(includeFamilies.gemini.has(family)).toBe(false); + expect(excludeFamilies.gemini.has(family)).toBe(false); + expect(PREVIEW_FAMILY.test(family)).toBe(true); + expect(isClassifiedFamily(family, "gemini")).toBe(true); + }); + + it("matches a trailing short numeric preview build tag (-preview-NN)", () => { + expect(PREVIEW_FAMILY.test("antigravity-preview-05")).toBe(true); + expect(isClassifiedFamily("deep-research-pro-preview-12", "gemini")).toBe(true); + }); + + it("classifies a future Gemma variant by rule (no literal registry entry)", () => { + // A synthetic future Gemma id normalizes to itself (no numeric-only build + // tag to strip) and is on NEITHER include nor exclude — it is classified + // purely by the GEMMA_FAMILY rule. Red against the old literal-names-only + // exclude set; green with the pattern. + const family = normalizeModelFamily("gemma-9-foo", "gemini"); + expect(includeFamilies.gemini.has(family)).toBe(false); + expect(excludeFamilies.gemini.has(family)).toBe(false); + expect(GEMMA_FAMILY.test(family)).toBe(true); + expect(isClassifiedFamily(family, "gemini")).toBe(true); + }); + + it("does NOT match interior -preview- suffixes (stay enumerated)", () => { + expect(PREVIEW_FAMILY.test("gemini-2.5-flash-preview-tts")).toBe(false); + expect(PREVIEW_FAMILY.test("gemini-3.1-pro-preview-customtools")).toBe(false); + // Those two ARE classified — but via explicit excludeFamilies enumeration. + expect(isClassifiedFamily("gemini-2.5-flash-preview-tts", "gemini")).toBe(true); + expect(isClassifiedFamily("gemini-3.1-pro-preview-customtools", "gemini")).toBe(true); + // A NON-enumerated interior-suffix family stays unclassified (still drift). + expect(isClassifiedFamily("gemini-x-flash-preview-widgets", "gemini")).toBe(false); + }); +}); + +// ─── Part 2: §6.1c builder/fixture cross-check ─────────────────────────────── + +type Provider = "openai" | "anthropic" | "gemini"; + +/** + * The static cross-check table. Each entry is a model id that aimock's + * builders, fixtures, or DEFAULT_MODELS reference, tagged with its provider. + * Every entry MUST normalize to a family in includeFamilies[provider] or + * excludeFamilies[provider]. Non-model tokens go in NON_MODEL_TOKENS_UNDER_TEST. + * + * Sources: + * - DEFAULT_MODELS (src/server.ts): the ids aimock advertises at /v1/models + * - Conformance / fixture-blocks tests: dated snapshots, family aliases + * - gemini-interactions: provider-mode token (NON_MODEL_TOKENS, not a real id) + */ +const BUILDER_FIXTURE_MODEL_IDS: Array<{ id: string; provider: Provider }> = [ + // ── DEFAULT_MODELS (src/server.ts) ───────────────────────────────────────── + { id: "gpt-4", provider: "openai" }, + { id: "gpt-4o", provider: "openai" }, + { id: "claude-3-5-sonnet-20241022", provider: "anthropic" }, // dated snapshot + { id: "gemini-2.0-flash", provider: "gemini" }, + // text-embedding-3-small is in excludeFamilies (non-text-generation) — see below + + // ── OpenAI: additional families referenced in conformance/fixture tests ──── + { id: "gpt-3.5-turbo", provider: "openai" }, + { id: "gpt-4-turbo", provider: "openai" }, + { id: "gpt-4-turbo-2024-04-09", provider: "openai" }, // dated → gpt-4-turbo + { id: "gpt-4o-2024-08-06", provider: "openai" }, // dated → gpt-4o + { id: "gpt-4o-mini", provider: "openai" }, + { id: "gpt-4.1", provider: "openai" }, + { id: "gpt-4.1-mini", provider: "openai" }, + { id: "gpt-4.1-nano", provider: "openai" }, + { id: "gpt-5", provider: "openai" }, + { id: "gpt-5-mini", provider: "openai" }, + + // ── OpenAI: excluded families (embeddings, image, voice/audio) ──────────── + { id: "text-embedding-3-small", provider: "openai" }, // exclude: embeddings + { id: "gpt-image-1", provider: "openai" }, // exclude: image + { id: "gpt-audio", provider: "openai" }, // exclude: voice canary + { id: "gpt-audio-mini", provider: "openai" }, // exclude: voice canary + { id: "gpt-audio-2025-08-28", provider: "openai" }, // dated → gpt-audio (exclude) + { id: "tts-1", provider: "openai" }, // exclude: tts + { id: "gpt-4o-mini-tts", provider: "openai" }, // exclude: tts + { id: "gpt-4o-mini-tts-2025-12-15", provider: "openai" }, // dated → gpt-4o-mini-tts (exclude) + { id: "gpt-4o-transcribe", provider: "openai" }, // exclude: transcribe + { id: "gpt-4o-mini-transcribe", provider: "openai" }, // exclude: transcribe + { id: "gpt-realtime", provider: "openai" }, // exclude: realtime canary + { id: "gpt-realtime-mini", provider: "openai" }, // exclude: realtime canary + { id: "gpt-4o-realtime-preview", provider: "openai" }, // exclude: preview realtime + { id: "gpt-4o-mini-realtime-preview", provider: "openai" }, // exclude: preview realtime + + // ── Anthropic ───────────────────────────────────────────────────────────── + { id: "claude-3-opus", provider: "anthropic" }, + { id: "claude-3-opus-20240229", provider: "anthropic" }, // dated → claude-3-opus + { id: "claude-3-sonnet", provider: "anthropic" }, + { id: "claude-3-haiku", provider: "anthropic" }, + { id: "claude-3-5-sonnet", provider: "anthropic" }, + { id: "claude-3-5-haiku", provider: "anthropic" }, + { id: "claude-3-5-haiku-20241022", provider: "anthropic" }, // dated → claude-3-5-haiku + { id: "claude-3-7-sonnet", provider: "anthropic" }, + { id: "claude-3-7-sonnet-20250219", provider: "anthropic" }, // dated → claude-3-7-sonnet + { id: "claude-opus-4", provider: "anthropic" }, + { id: "claude-opus-4-20250514", provider: "anthropic" }, // dated → claude-opus-4 + { id: "claude-sonnet-4", provider: "anthropic" }, + { id: "claude-haiku-4", provider: "anthropic" }, + + // ── Gemini ──────────────────────────────────────────────────────────────── + { id: "gemini-1.5-pro", provider: "gemini" }, + { id: "gemini-1.5-pro-2024-05-14", provider: "gemini" }, // dated → gemini-1.5-pro + { id: "gemini-1.5-flash", provider: "gemini" }, + { id: "gemini-2.5-flash", provider: "gemini" }, + { id: "gemini-2.5-pro", provider: "gemini" }, + { id: "gemini-2.0-flash-exp", provider: "gemini" }, // exclude: experimental + { id: "gemini-2.0-flash-thinking-exp", provider: "gemini" }, // exclude: experimental +]; + +/** + * Provider-mode tokens that aimock uses internally to route to a real upstream + * provider but that are NOT model ids any provider's /models endpoint exposes. + * These MUST be on NON_MODEL_TOKENS — they must never be mistaken for real ids. + */ +const NON_MODEL_TOKENS_UNDER_TEST: string[] = ["gemini-interactions"]; + +describe("§6.1c builder/fixture cross-check (no live keys)", () => { + it("every referenced builder/fixture model id resolves to a classified family", () => { + const failures: string[] = []; + + for (const { id, provider } of BUILDER_FIXTURE_MODEL_IDS) { + const family = normalizeModelFamily(id, provider); + // Use the SAME classification predicate the live drift check uses + // (include ∪ exclude ∪ the PREVIEW_FAMILY / GEMMA_FAMILY exclude-by-rule + // patterns) so the two classification surfaces cannot drift apart. + if (!isClassifiedFamily(family, provider)) { + failures.push( + `${id} (${provider}): normalized to "${family}" which is classified by NEITHER includeFamilies, excludeFamilies, NOR the preview/gemma rules`, + ); + } + } + + expect( + failures, + `Stray builder/fixture model ids not in registry:\n${failures.join("\n")}`, + ).toEqual([]); + }); + + it("every non-model provider-mode token is on NON_MODEL_TOKENS", () => { + const failures: string[] = []; + + for (const token of NON_MODEL_TOKENS_UNDER_TEST) { + if (!NON_MODEL_TOKENS.has(token)) { + failures.push( + `"${token}" is not on NON_MODEL_TOKENS — a greedy scrape would false-positive`, + ); + } + } + + expect( + failures, + `Provider-mode tokens missing from NON_MODEL_TOKENS:\n${failures.join("\n")}`, + ).toEqual([]); + }); +}); diff --git a/src/__tests__/drift/model-registry.ts b/src/__tests__/drift/model-registry.ts new file mode 100644 index 00000000..212f2463 --- /dev/null +++ b/src/__tests__/drift/model-registry.ts @@ -0,0 +1,346 @@ +/** + * Per-provider model-family REGISTRY for text-generation drift detection. + * + * Side-effect-free data module (no `describe`/`beforeAll`, like `voice-models.ts`) + * so both the live drift check (`models.drift.ts`) and its unit test can import + * the same seed data without transitively registering a drift suite. + * + * The drift check works by NORMALIZED FAMILY, not raw id: a provider's live + * `GET /models` list is normalized through `normalizeModelFamily(id, provider)` + * and each resulting family is subtracted against `include ∪ exclude`. Anything + * left over is an UNCLASSIFIED family — the drift signal. (The subtract/delta + * itself is `models.drift.ts`'s job; this module only owns the seed sets.) + * + * Two curated sets per provider, both keyed by the NORMALIZED family: + * + * - `include` — families aimock actually MOCKS. Derived from the model ids + * aimock's conformance tests, README, and fixtures reference (chat / text + * completion families). A live family that normalizes into this set is known + * and generates no drift. Seeds are already family keys, so dated snapshots + * of an included family (`gpt-4o-2024-08-06` → `gpt-4o`) collapse onto them. + * + * - `exclude` — families we deliberately DO NOT treat as text-generation drift: + * retired/legacy ids, preview-only ids, and non-text families (embeddings, + * image, tts/audio/transcribe voice families — the last are the realtime + * canary's responsibility in `voice-models.ts`, not this text check). A live + * family in this set is expected and generates no drift. + * + * A family MUST NOT appear in both `include` and `exclude` for the same provider + * (asserted in the unit test) — the two sets partition the "already classified" + * space; their union is what the drift check subtracts. + * + * `NON_MODEL_TOKENS` is a provider-agnostic allowlist of aimock "provider mode" + * names — internal routing names that reuse a real upstream API key but are NOT + * model ids any provider's `/models` endpoint exposes (e.g. `gemini-interactions` + * reuses the Gemini upstream key). The README documents them, so a greedy source + * scrape or a builder cross-check would otherwise treat them as unknown model + * ids and flag guaranteed false positives. They live here so both the drift + * check and the builder cross-check (B4.3) exclude the exact same tokens. + * + * Every set is built THROUGH `normalizeModelFamily` so membership tests are + * normalization-consistent and the seeds are provably idempotent (a seed carrying + * a stray dated/build suffix would silently normalize to a different key — the + * `.map(normalize)` makes that impossible). + */ + +import { normalizeModelFamily } from "./model-family.js"; + +type Provider = "openai" | "anthropic" | "gemini"; + +/** Build a family Set for a provider, seeding each entry through the normalizer. */ +function familySet(provider: Provider, families: string[]): Set { + return new Set(families.map((f) => normalizeModelFamily(f, provider))); +} + +/** + * Families aimock MOCKS, per provider. These are the canonical text-generation + * family keys referenced by aimock's conformance tests, README, and fixtures. + * Dated/versioned variants normalize onto these (e.g. `gpt-4o-2024-08-06` → + * `gpt-4o`, `claude-3-5-sonnet-20241022` → `claude-3-5-sonnet`). + */ +export const includeFamilies: Record> = { + openai: familySet("openai", [ + // Chat / completion families + "gpt-3.5-turbo", + "gpt-4", + "gpt-4-turbo", + "gpt-4o", + "gpt-4o-mini", + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4.1-nano", + "gpt-5", + "gpt-5-mini", + // gpt-5 tier: additional sizes + chat surfaces + "gpt-5-nano", + "gpt-5-pro", + "gpt-5-chat-latest", + // gpt-5.x point releases (text chat) + "gpt-5.1", + "gpt-5.1-chat-latest", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.2-chat-latest", + "gpt-5.3-chat-latest", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + // gpt-5.6 named variants (text chat) + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + // Codex line (coding chat; text output) + "gpt-5-codex", + "gpt-5.1-codex", + "gpt-5.1-codex-max", + "gpt-5.1-codex-mini", + "gpt-5.2-codex", + "gpt-5.3-codex", + // o-series reasoning (text) + "o1", + "o1-pro", + "o3", + "o3-mini", + "o3-pro", + "o4-mini", + ]), + anthropic: familySet("anthropic", [ + // Claude 3 / 3.5 / 3.7 families + "claude-3-opus", + "claude-3-sonnet", + "claude-3-haiku", + "claude-3-5-sonnet", + "claude-3-5-haiku", + "claude-3-7-sonnet", + // Claude 4 families + "claude-opus-4", + "claude-sonnet-4", + "claude-haiku-4", + // Claude 4.x point releases (text chat) + "claude-haiku-4-5", + "claude-opus-4-1", + "claude-opus-4-5", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-sonnet-4-5", + "claude-sonnet-4-6", + // Claude 5 families (text chat) + "claude-sonnet-5", + // claude-fable-5 is included ahead of a recorded fixture (intended — mirrors + // the forward-looking rationale for the exclude-by-rule patterns above). + "claude-fable-5", + ]), + gemini: familySet("gemini", [ + // Gemini 1.5 / 2.0 / 2.5 text families + "gemini-1.5-pro", + "gemini-1.5-flash", + "gemini-2.0-flash", + "gemini-2.5-flash", + "gemini-2.5-pro", + // Additional stable flash/pro tiers (text) + "gemini-2.0-flash-lite", + "gemini-2.5-flash-lite", + "gemini-3.1-flash-lite", + "gemini-3.5-flash", + ]), +}; + +/** + * Families we deliberately DO NOT count as text-generation drift, per provider: + * retired/legacy ids, preview-only ids, and non-text families (embeddings, + * image, and the voice/audio/tts/transcribe families the realtime canary in + * `voice-models.ts` owns). A live family here is expected, not drift. + */ +export const excludeFamilies: Record> = { + openai: familySet("openai", [ + // Retired / legacy chat + base-completion + "gpt-3", + "gpt-3.5", + "gpt-3.5-turbo-16k", + "gpt-3.5-turbo-instruct", + "babbage", + "davinci", + // Embeddings (non-text-generation) + "text-embedding-3-small", + "text-embedding-3-large", + "text-embedding-ada-002", + // Image models + "dall-e-2", + "dall-e-3", + "gpt-image-1", + "gpt-image-1-mini", + "gpt-image-1.5", + "gpt-image-2", + "chatgpt-image-latest", + // Bare chat alias — a moving alias (not a stable mocked family); matches the + // *-latest exclude policy (chatgpt-image-latest, omni-moderation-latest, etc.) + "chat-latest", + // Video generation (non-text) + "sora-2", + "sora-2-pro", + // Moderation classifier (non-text-generation) + "omni-moderation", + "omni-moderation-latest", + // Deep-research / search specialty surfaces (text output, not plain chat). + // Enumerated because they carry no trailing `-preview` (see PREVIEW_FAMILY). + "o3-deep-research", + "o4-mini-deep-research", + "gpt-5-search-api", + // Voice / audio / tts / transcribe — owned by the realtime canary + "tts-1", + "tts-1-hd", + "whisper-1", + "gpt-audio", + "gpt-audio-mini", + "gpt-audio-1.5", + "gpt-4o-mini-tts", + "gpt-4o-transcribe", + "gpt-4o-mini-transcribe", + "gpt-4o-transcribe-diarize", + "gpt-realtime", + "gpt-realtime-mini", + "gpt-realtime-2", + "gpt-realtime-2.1", + "gpt-realtime-2.1-mini", + "gpt-realtime-1.5", + "gpt-realtime-translate", + "gpt-realtime-whisper", + // NOTE: `-preview` families (gpt-4o-realtime-preview, + // gpt-4o-mini-realtime-preview, gpt-4o-search-preview, + // gpt-4o-mini-search-preview, computer-use-preview, …) are auto-excluded by + // the PREVIEW_FAMILY rule (see isClassifiedFamily below) — no enumeration. + ]), + anthropic: familySet("anthropic", [ + // Retired / legacy Claude ids + "claude-v3", + "claude-2", + "claude-instant-1", + ]), + gemini: familySet("gemini", [ + // Retired / legacy specialty + "gemini-pro", + "aqa", + // Embeddings (non-text-generation) + "text-embedding-004", + "gemini-embedding", + "gemini-embedding-2", + // Image models (non-text) + "gemini-2.5-flash-image", + "gemini-3-pro-image", + "gemini-3.1-flash-image", + "gemini-3.1-flash-lite-image", + "imagen-4.0-fast-generate", + "imagen-4.0-generate", + "imagen-4.0-ultra-generate", + // Audio / native-audio (realtime canary domain) + "gemini-2.5-flash-native-audio-latest", + // NOTE: the open-weight Gemma line (`gemma-4-26b-a4b-it`, `gemma-4-31b-it`, + // and any future variant) is auto-excluded by the GEMMA_FAMILY rule (see + // isClassifiedFamily below) — no enumeration. Gemma is open-weight, not + // mocked on the Gemini surface, so it is out of scope; reversible by future + // explicit handling if a Gemma fixture is ever added. + // Moving aliases (not families aimock commits to as stable text surfaces) + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest", + // Interior `-preview-` specialty suffixes the PREVIEW_FAMILY rule does + // NOT match (it only matches trailing `-preview` / `-preview-`). + // Enumerated so their category (tts / custom-tools) stays documented. + "gemini-2.5-flash-preview-tts", + "gemini-2.5-pro-preview-tts", + "gemini-3.1-pro-preview-customtools", + // Experimental / thinking (kept explicit — historic `-exp`, not `-preview`) + "gemini-2.0-flash-exp", + "gemini-2.0-flash-thinking-exp", + // Live/full-duplex voice — owned by the realtime canary, not this text check + "gemini-live", + // NOTE: every `-preview` family (gemini-3.x preview tiers, deep-research + // previews, antigravity-preview-05, computer-use-preview-10, image/tts/robotics + // previews, lyria/veo/nano-banana previews, gemini-embedding-2-preview, …) is + // auto-excluded by the PREVIEW_FAMILY rule (see isClassifiedFamily below). + ]), +}; + +/** + * aimock "provider mode" names: internal routing names that reuse a real + * upstream provider key but are NOT model ids any provider's `/models` endpoint + * exposes. Provider-agnostic allowlist so both the drift check and the builder + * cross-check exclude the exact same tokens (never false-positive drift). + */ +export const NON_MODEL_TOKENS: Set = new Set(["gemini-interactions"]); + +/** + * Durable EXCLUDE-BY-RULE predicate for preview/experimental families. + * + * Preview tiers (`gemini-3-pro-preview`, `gpt-4o-search-preview`, + * `antigravity-preview-05`, …) are unstable and short-lived: providers ship a + * new one on nearly every release wave, and each one that lands would otherwise + * re-fire the daily drift canary as a brand-new UNCLASSIFIED family, forcing a + * registry edit per preview forever (that is exactly how the Gemini-3.x wave + * surfaced — the exclude set was a hand-maintained list with no preview rule). + * A pattern rule auto-excludes every current AND future `-preview` family with + * zero registry churn, and re-alerting on each new preview tier is precisely the + * whack-a-mole this hardening removes. + * + * The pattern intentionally matches ONLY a trailing `-preview` token, optionally + * followed by a short numeric build tag the normalizer does not strip + * (`-preview-04`, `-preview-05`, `-preview-10`, `-preview-12`). The `-\d+` tail + * is unbounded on purpose — a 1-2 digit build tag survives the normalizer + * (`BUILD_TAG_SUFFIX` only strips 3-4 digit tails), and a longer numeric tail on + * a `-preview` token is still unambiguously a preview surface, so there is no + * upper bound to enforce. It deliberately does NOT match interior + * `-preview-` suffixes like `-preview-tts` / `-preview-customtools`; those + * are non-text specialty surfaces enumerated explicitly in `excludeFamilies` so + * their category (tts / custom-tools) stays documented rather than swept under a + * blanket rule. + * + * TRADEOFF — previews are BLANKET-excluded until GA, per policy. This + * intentionally SILENCES the canary on new text previews (e.g. `o1-preview`, + * `gpt-4.5-preview`, a `gemini-3` chat preview) until they drop the `-preview` + * suffix and go GA. There is no include-side escape hatch: aimock does not mock + * preview surfaces, so re-alerting on each new preview tier is precisely the + * whack-a-mole this rule removes. If aimock ever needs to mock a specific + * preview surface, add explicit handling then. + * + * IMPORTANT — this lives at the CLASSIFICATION layer, NOT the normalizer. The + * normalizer is left untouched so preview family keys stay DISTINCT: a genuinely + * new preview family is still visible as its own key (never collapsed onto its GA + * sibling), which preserves the canary's ability to surface new families while + * making previews self-excluding. + */ +export const PREVIEW_FAMILY = /-preview(-\d+)?$/; + +/** + * Durable EXCLUDE-BY-RULE predicate for the open-weight Gemma line. + * + * Gemma is open-weight and NOT mocked on aimock's Gemini surface (no code / + * fixtures); it only rides the shared Gemini `/models` listing. Every current + * AND future Gemma variant (`gemma-4-26b-a4b-it`, `gemma-4-31b-it`, …) is + * out of scope, so a pattern rule auto-excludes them with zero registry churn — + * mirroring the PREVIEW_FAMILY rationale. Reversible by future explicit handling + * if a Gemma fixture is ever added. + */ +export const GEMMA_FAMILY = /^gemma(-|$)/; + +/** + * A NORMALIZED family key is already classified if it is in `include ∪ exclude`, + * OR it matches an exclude-by-rule pattern (preview, gemma). Both the live drift + * check (`unclassifiedFamilies`) and the builder/fixture cross-check use this + * single predicate so their classification surfaces cannot drift apart. + * + * There is deliberately NO include-side escape hatch: previews and Gemma are + * blanket-excluded (aimock does not mock either surface), so an include entry + * could never legitimately collide with an exclude rule. The intersection + * `include ∩ (preview ∪ gemma)` is empty by construction. + */ +export function isClassifiedFamily(family: string, provider: Provider): boolean { + if (includeFamilies[provider].has(family)) return true; + if (excludeFamilies[provider].has(family)) return true; + if (PREVIEW_FAMILY.test(family)) return true; // exclude-by-rule + if (GEMMA_FAMILY.test(family)) return true; // exclude-by-rule + return false; +} diff --git a/src/__tests__/drift/models.drift.ts b/src/__tests__/drift/models.drift.ts index 73e4f4e0..b3da9f2c 100644 --- a/src/__tests__/drift/models.drift.ts +++ b/src/__tests__/drift/models.drift.ts @@ -1,103 +1,402 @@ /** - * Model deprecation checks — verify that models referenced in aimock's - * tests, docs, and examples still exist at each provider. + * Model-family drift check — verify that each provider's LIVE `GET /models` + * list contains no UNCLASSIFIED text-generation family. + * + * How it works (no source scraping — that path caused incident 5): + * 1. Fetch the provider's live model list (`list*Models`). + * 2. Normalize every id to its FAMILY key (`normalizeModelFamily`), so dated + * snapshots and build tags collapse onto their family + * (`gpt-4o-2024-08-06` → `gpt-4o`, `tts-1-1106` → `tts-1`). + * 3. Subtract the already-classified space: `includeFamilies[provider] ∪ + * excludeFamilies[provider]`, and drop the provider-agnostic + * `NON_MODEL_TOKENS` allowlist. + * 4. Whatever remains is an UNCLASSIFIED family — the drift signal. It is + * surfaced as a FAILING assertion wrapped in a `formatDriftReport` block so + * the collector (`scripts/drift-report-collector.ts`) routes it: the + * `API DRIFT DETECTED` block parses into critical entries (exit-2 auto-fix + * lane); anything it cannot map falls to exit-5 quarantine. + * + * Comparing NORMALIZED families (not raw ids) is what makes this converge: + * appending every new dated snapshot to a known-id set never stabilizes and + * turns the daily job permanently red on false positives (incident 2). Only a + * genuinely NEW family (e.g. `gpt-live`) is ever flagged. + * + * Because nothing is scraped from source, an aimock "provider mode" prose token + * (e.g. `gemini-interactions`) can never enter the pipeline as a candidate id — + * the only inputs are the provider's own `/models` payload. */ import { describe, it, expect } from "vitest"; -import * as fs from "node:fs"; -import * as path from "node:path"; import { listOpenAIModels, listAnthropicModels, listGeminiModels } from "./providers.js"; +import { normalizeModelFamily } from "./model-family.js"; +import { NON_MODEL_TOKENS, isClassifiedFamily } from "./model-registry.js"; +import { formatDriftReport } from "./schema.js"; -// --------------------------------------------------------------------------- -// Scrape referenced models from the codebase -// --------------------------------------------------------------------------- +type Provider = "openai" | "anthropic" | "gemini"; -const PROJECT_ROOT = path.resolve(import.meta.dirname, "..", "..", ".."); - -function scrapeModels(pattern: RegExp, files: string[]): string[] { - const models = new Set(); - for (const file of files) { - const filePath = path.join(PROJECT_ROOT, file); - if (!fs.existsSync(filePath)) continue; - const content = fs.readFileSync(filePath, "utf-8"); - pattern.lastIndex = 0; - let match; - while ((match = pattern.exec(content)) !== null) { - models.add(match[1]); - } +/** + * Reduce a live `/models` list to the UNCLASSIFIED families: normalize each id, + * then drop everything already classified (`include ∪ exclude`, the + * preview/gemma exclude-by-rule patterns — see `isClassifiedFamily` in + * model-registry.ts) or on the non-model allowlist. The returned list (sorted, + * de-duplicated) is the drift signal. + * + * Exported so the co-located regression suite can exercise the EXACT + * enumerate→normalize→subtract pipeline the live check relies on, with an + * injected payload — no reimplementation. + */ +export function unclassifiedFamilies(modelIds: string[], provider: Provider): string[] { + const unclassified = new Set(); + for (const id of modelIds) { + const family = normalizeModelFamily(id, provider); + if (isClassifiedFamily(family, provider)) continue; + if (NON_MODEL_TOKENS.has(family) || NON_MODEL_TOKENS.has(id)) continue; + unclassified.add(family); } - return [...models]; + return [...unclassified].sort(); } -const sourceFiles = [ - "src/__tests__/api-conformance.test.ts", - "src/__tests__/ws-api-conformance.test.ts", - "README.md", - "fixtures/example-greeting.json", - "fixtures/example-multi-turn.json", - "fixtures/example-tool-call.json", -]; +/** + * Assert that a live `/models` list has zero unclassified families. On failure, + * emit one critical drift diff per unclassified family inside a + * `formatDriftReport` block so the collector routes it to the exit-2 auto-fix + * lane (provider names match `PROVIDER_MAP` keys in the collector). + */ +function assertNoUnclassifiedFamilies( + modelIds: string[], + provider: Provider, + context: string, +): void { + const unclassified = unclassifiedFamilies(modelIds, provider); + const report = + unclassified.length > 0 + ? formatDriftReport( + context, + unclassified.map((family) => ({ + path: `models/${family}`, + severity: "critical" as const, + issue: + `Unclassified model family "${family}" in ${provider} /models — ` + + `add it to includeFamilies (aimock mocks it) or excludeFamilies ` + + `(non-text / retired / preview) in model-registry.ts`, + expected: "(family in includeFamilies ∪ excludeFamilies)", + real: family, + mock: "", + })), + ) + : `No drift detected: ${context}`; + expect(unclassified, report).toEqual([]); +} // --------------------------------------------------------------------------- -// OpenAI +// Regression suite (no live keys) — exercises the REAL pipeline with injected +// `/models` payloads. Runs unconditionally so the drift job proves the +// enumerate→normalize→subtract behavior even when live keys are absent. // --------------------------------------------------------------------------- -describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI model availability", () => { - it("models used in aimock tests are still available", async () => { - const models = await listOpenAIModels(process.env.OPENAI_API_KEY!); - const referenced = scrapeModels(/\b(gpt-4o(?:-mini)?|gpt-4|gpt-3\.5-turbo)\b/g, sourceFiles); +describe("model-family pipeline (injected /models)", () => { + it("incident 2: dated snapshots of included families produce ZERO drift", () => { + // Payload of dated/build-tag snapshots whose FAMILIES are all in + // includeFamilies/excludeFamilies. The old scrape+substring path would have + // flagged these dated ids as unknown; the normalize+subtract path collapses + // each onto its known family, so the unclassified delta must be empty. + const openaiPayload = [ + "gpt-4o-2024-08-06", // → gpt-4o (include) + "gpt-4o-mini-2024-07-18", // → gpt-4o-mini (include) + "gpt-4.1-2025-04-14", // → gpt-4.1 (include) + "gpt-audio-2025-08-28", // → gpt-audio (exclude) + "tts-1-1106", // → tts-1 (exclude) + "gpt-4o-mini-tts-2025-12-15", // → gpt-4o-mini-tts (exclude) + ]; + expect(unclassifiedFamilies(openaiPayload, "openai")).toEqual([]); - if (referenced.length === 0) return; // no models found to check + // Gemini dated variants collapse via the same dated-snapshot strip. + const geminiPayload = [ + "gemini-2.5-flash", // include + "gemini-2.0-flash", // include + "gemini-1.5-pro-2024-05-14", // → gemini-1.5-pro (include) + ]; + expect(unclassifiedFamilies(geminiPayload, "gemini")).toEqual([]); - for (const m of referenced) { - // OpenAI model list may include versioned variants — check prefix match - const found = models.some((available) => available === m || available.startsWith(`${m}-`)); - expect(found, `Model ${m} no longer available at OpenAI`).toBe(true); - } + // Anthropic uses a CONTIGUOUS 8-digit snapshot suffix (`-YYYYMMDD`), not the + // dashed `-YYYY-MM-DD` form. These must collapse onto their included family + // via the anthropic-specific strip, or every dated Claude id false-positives + // as drift (the incident-2 class, for Anthropic). + const anthropicPayload = [ + "claude-3-5-sonnet-20241022", // → claude-3-5-sonnet (include) + "claude-3-7-sonnet-20250219", // → claude-3-7-sonnet (include) + "claude-3-5-haiku-20241022", // → claude-3-5-haiku (include) + ]; + expect(unclassifiedFamilies(anthropicPayload, "anthropic")).toEqual([]); + }); + + it("a prose provider-mode token can never enter as a candidate", () => { + // Nothing is scraped from source, so a `gemini-interactions`-style token can + // only appear if a provider's own /models returned it — and even then it is + // on NON_MODEL_TOKENS and never becomes drift. + expect(unclassifiedFamilies(["gemini-interactions"], "gemini")).toEqual([]); + }); + + it("a genuinely new family IS flagged as unclassified drift", () => { + // Guard the other side: the canary must still fire for a real new family. + expect(unclassifiedFamilies(["gpt-live"], "openai")).toEqual(["gpt-live"]); + // Single-digit trailing suffix is NOT a build tag, so it stays unknown. + expect(unclassifiedFamilies(["gpt-live-1"], "openai")).toEqual(["gpt-live-1"]); }); }); // --------------------------------------------------------------------------- -// Anthropic +// PREVIEW_FAMILY exclude-by-rule predicate (the durable hardening). // --------------------------------------------------------------------------- -describe.skipIf(!process.env.ANTHROPIC_API_KEY)("Anthropic model availability", () => { - it("models used in aimock tests are still available", async () => { - const models = await listAnthropicModels(process.env.ANTHROPIC_API_KEY!); - const referenced = scrapeModels( - /\b(claude-3(?:\.\d+)?-(?:opus|sonnet|haiku)(?:-\d{8})?)\b/g, - sourceFiles, - ); +describe("preview families are excluded by rule", () => { + it("a brand-new -preview family auto-excludes with zero registry edits", () => { + // Not in include or exclude — classified purely by the trailing-preview rule. + expect(unclassifiedFamilies(["gemini-9-pro-preview"], "gemini")).toEqual([]); + expect(unclassifiedFamilies(["gpt-9-search-preview"], "openai")).toEqual([]); + }); - if (referenced.length === 0) return; + it("matches a trailing short numeric preview build tag (-preview-NN)", () => { + // The normalizer leaves 2-digit tails intact, so the rule must cover them. + expect(unclassifiedFamilies(["antigravity-preview-05"], "gemini")).toEqual([]); + expect(unclassifiedFamilies(["deep-research-pro-preview-12"], "gemini")).toEqual([]); + }); - for (const m of referenced) { - const found = models.some((available) => available === m || available.startsWith(m)); - expect(found, `Model ${m} no longer available at Anthropic`).toBe(true); - } + it("does NOT match interior -preview- suffixes (they stay enumerated)", () => { + // `-preview-tts` / `-preview-customtools` are non-text specialty surfaces + // enumerated explicitly; a hypothetical UNLISTED interior-suffix family must + // still fire as drift so its category is not silently swallowed. + expect(unclassifiedFamilies(["gemini-x-flash-preview-widgets"], "gemini")).toEqual([ + "gemini-x-flash-preview-widgets", + ]); }); }); // --------------------------------------------------------------------------- -// Gemini +// GEMMA_FAMILY exclude-by-rule predicate (open-weight, out of scope). // --------------------------------------------------------------------------- -describe.skipIf(!process.env.GOOGLE_API_KEY)("Gemini model availability", () => { - it("models used in aimock tests are still available", async () => { - const models = await listGeminiModels(process.env.GOOGLE_API_KEY!); - const referenced = scrapeModels(/\b(gemini-(?:[\w.-]+))\b/g, sourceFiles); +describe("gemma families are excluded by rule", () => { + it("a future Gemma variant auto-excludes with zero registry edits", () => { + // `gemma-9-foo` has no numeric-only build tag to strip, so it normalizes to + // itself and is on NEITHER include nor exclude — classified purely by the + // gemma rule. Red against the old literal-names-only exclude set; green with + // the pattern. + expect(unclassifiedFamilies(["gemma-9-foo"], "gemini")).toEqual([]); + // The two originally-enumerated literals still classify (now via the rule). + expect(unclassifiedFamilies(["gemma-4-26b-a4b-it", "gemma-4-31b-it"], "gemini")).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +// Regression: the FULL live /models family wave (run 29478043559, 2026-07-16). +// These are the exact families the drift job flagged as UNCLASSIFIED (OpenAI 48 +// / Anthropic 10 / Gemini 45). This suite injects representative raw ids for +// every one of them into the REAL enumerate→normalize→subtract pipeline and +// asserts the registry now classifies all of them — zero drift. It exercises +// the same `unclassifiedFamilies` surface the live canary uses, so it is a true +// regression against the classification (not a fake against a private copy). +// --------------------------------------------------------------------------- - if (referenced.length === 0) return; +describe("full live /models wave is fully classified (2026-07-16 drift)", () => { + it("OpenAI: every live family is classified (zero unclassified)", () => { + const openaiLive = [ + // Existing include families (dated snapshots collapse onto them) + "gpt-3.5-turbo", + "gpt-4", + "gpt-4-turbo", + "gpt-4o-2024-08-06", + "gpt-4o-mini", + "gpt-4.1", + "gpt-4.1-mini", + "gpt-4.1-nano", + "gpt-5", + "gpt-5-mini", + // Newly-classified INCLUDE: gpt-5.x tiers + chat surfaces + "gpt-5-chat-latest", + "gpt-5.1-chat-latest", + "gpt-5.2-chat-latest", + "gpt-5.3-chat-latest", + "gpt-5-nano", + "gpt-5-pro", + "gpt-5.1", + "gpt-5.2", + "gpt-5.2-pro", + "gpt-5.4", + "gpt-5.4-mini", + "gpt-5.4-nano", + "gpt-5.4-pro", + "gpt-5.5", + "gpt-5.5-pro", + "gpt-5.6-luna", + "gpt-5.6-sol", + "gpt-5.6-terra", + // o-series reasoning + "o1", + "o1-pro", + "o3", + "o3-mini", + "o3-pro", + "o4-mini", + // Codex line + "gpt-5-codex", + "gpt-5.1-codex", + "gpt-5.1-codex-max", + "gpt-5.1-codex-mini", + "gpt-5.2-codex", + "gpt-5.3-codex", + // Newly-classified EXCLUDE: retired base + "babbage-002", + "davinci-002", + "gpt-3.5-turbo-16k", + "gpt-3.5-turbo-instruct", + // deep-research / search / computer-use surfaces + "o3-deep-research", + "o4-mini-deep-research", + "gpt-5-search-api", + "gpt-4o-search-preview", // preview pattern + "gpt-4o-mini-search-preview", // preview pattern + "computer-use-preview", // preview pattern + // image / video / moderation + "chatgpt-image-latest", + "gpt-image-1-mini", + "gpt-image-1.5", + "gpt-image-2", + "sora-2", + "sora-2-pro", + "omni-moderation", + "omni-moderation-latest", + // bare chat alias (fixture-vs-live gap: present in live /models, was absent + // from the static wave — this is the 2026-07-18 canary residual) + "chat-latest", + ]; + expect(unclassifiedFamilies(openaiLive, "openai")).toEqual([]); + }); + + it("Anthropic: every live family is classified (zero unclassified)", () => { + const anthropicLive = [ + // Existing include (dated snapshots collapse) + "claude-3-opus-20240229", + "claude-3-5-sonnet-20241022", + "claude-3-7-sonnet-20250219", + "claude-opus-4-20250514", + "claude-sonnet-4", + "claude-haiku-4", + // Newly-classified INCLUDE: 4.x/5 point releases + fable + "claude-haiku-4-5", + "claude-opus-4-1", + "claude-opus-4-5", + "claude-opus-4-6", + "claude-opus-4-7", + "claude-opus-4-8", + "claude-sonnet-4-5", + "claude-sonnet-4-6", + "claude-sonnet-5", + "claude-fable-5", + ]; + expect(unclassifiedFamilies(anthropicLive, "anthropic")).toEqual([]); + }); + + it("Gemini: every live family is classified (zero unclassified)", () => { + const geminiLive = [ + // Existing include + "gemini-1.5-pro", + "gemini-1.5-flash", + "gemini-2.0-flash", + "gemini-2.5-flash", + "gemini-2.5-pro", + // Newly-classified INCLUDE: stable flash/pro tiers + "gemini-2.0-flash-lite", + "gemini-2.5-flash-lite", + "gemini-3.1-flash-lite", + "gemini-3.5-flash", + // Preview text tiers — auto-excluded by the -preview pattern rule + "gemini-3-flash-preview", + "gemini-3-pro-preview", + "gemini-3.1-pro-preview", + "gemini-3.1-flash-lite-preview", + // Preview specialty surfaces (pattern OR explicit) + "gemini-3.1-pro-preview-customtools", // explicit exclude (not -preview$) + "antigravity-preview-05", // pattern + "aqa", // explicit exclude (retired specialty) + "deep-research-max-preview-04", // pattern + "deep-research-preview-04", // pattern + "deep-research-pro-preview-12", // pattern + "gemini-2.5-computer-use-preview-10", // pattern + "gemini-omni-flash-preview", // pattern + // image / audio / tts / video / music / robotics / embeddings + "gemini-2.5-flash-image", + "gemini-2.5-flash-native-audio-latest", + "gemini-2.5-flash-preview-tts", // explicit exclude (-preview-tts) + "gemini-2.5-pro-preview-tts", // explicit exclude (-preview-tts) + "gemini-3-pro-image", + "gemini-3-pro-image-preview", // pattern + "gemini-3.1-flash-image", + "gemini-3.1-flash-image-preview", // pattern + "gemini-3.1-flash-lite-image", + "gemini-3.1-flash-tts-preview", // pattern + "gemini-embedding", + "gemini-embedding-2", + "gemini-embedding-2-preview", // pattern + "gemini-robotics-er-1.5-preview", // pattern + "gemini-robotics-er-1.6-preview", // pattern + "imagen-4.0-fast-generate", + "imagen-4.0-generate", + "imagen-4.0-ultra-generate", + "lyria-3-clip-preview", // pattern + "lyria-3-pro-preview", // pattern + "nano-banana-pro-preview", // pattern + "veo-3.1-fast-generate-preview", // pattern + "veo-3.1-generate-preview", // pattern + "veo-3.1-lite-generate-preview", // pattern + // Gemma open-weight — aimock does NOT mock; auto-excluded by GEMMA_FAMILY rule + "gemma-4-26b-a4b-it", + "gemma-4-31b-it", + // Moving aliases + "gemini-flash-latest", + "gemini-flash-lite-latest", + "gemini-pro-latest", + ]; + expect(unclassifiedFamilies(geminiLive, "gemini")).toEqual([]); + }); +}); - // Skip experimental models, live-only models, and anchor-link fragments - // scraped from markdown (e.g., "gemini-live-bidigeneratecontent") - const stable = referenced.filter( - (m) => !m.includes("-exp") && !m.includes("-live") && !m.includes("bidigeneratecontent"), - ); +// --------------------------------------------------------------------------- +// OpenAI +// --------------------------------------------------------------------------- - for (const m of stable) { - const found = models.some((available) => available === m || available.startsWith(m)); - expect(found, `Model ${m} no longer available at Gemini`).toBe(true); - } +describe.skipIf(!process.env.OPENAI_API_KEY)("OpenAI Chat model-family availability", () => { + it("live /models contains no unclassified family", async () => { + const models = await listOpenAIModels(process.env.OPENAI_API_KEY!); + assertNoUnclassifiedFamilies(models, "openai", "OpenAI Chat (live /models family canary)"); + }); +}); + +// --------------------------------------------------------------------------- +// Anthropic +// --------------------------------------------------------------------------- + +describe.skipIf(!process.env.ANTHROPIC_API_KEY)( + "Anthropic Claude model-family availability", + () => { + it("live /models contains no unclassified family", async () => { + const models = await listAnthropicModels(process.env.ANTHROPIC_API_KEY!); + assertNoUnclassifiedFamilies( + models, + "anthropic", + "Anthropic Claude (live /models family canary)", + ); + }); + }, +); + +// --------------------------------------------------------------------------- +// Gemini +// --------------------------------------------------------------------------- + +describe.skipIf(!process.env.GOOGLE_API_KEY)("Google Gemini model-family availability", () => { + it("live /models contains no unclassified family", async () => { + const models = await listGeminiModels(process.env.GOOGLE_API_KEY!); + assertNoUnclassifiedFamilies(models, "gemini", "Google Gemini (live /models family canary)"); }); }); diff --git a/src/__tests__/drift/moderation.drift.ts b/src/__tests__/drift/moderation.drift.ts index d5d1e479..11d603a2 100644 --- a/src/__tests__/drift/moderation.drift.ts +++ b/src/__tests__/drift/moderation.drift.ts @@ -51,7 +51,7 @@ describe("OpenAI Moderations drift", () => { const mockShape = extractShape(mockBody); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("OpenAI Moderations", diffs); + const report = formatDriftReport("OpenAI Moderations", diffs, "moderation"); expect( diffs.filter((d) => d.severity === "critical"), diff --git a/src/__tests__/drift/ollama.drift.ts b/src/__tests__/drift/ollama.drift.ts index becfc051..566107ab 100644 --- a/src/__tests__/drift/ollama.drift.ts +++ b/src/__tests__/drift/ollama.drift.ts @@ -130,7 +130,7 @@ describe.skipIf(!process.env.OLLAMA_HOST)("Ollama drift", () => { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); - const report = formatDriftReport("Ollama /api/chat", diffs); + const report = formatDriftReport("Ollama /api/chat", diffs, "ollama"); expect( diffs.filter((d) => d.severity === "critical"), @@ -168,7 +168,7 @@ describe.skipIf(!process.env.OLLAMA_HOST)("Ollama drift", () => { const mockFirstShape = extractShape(mockChunks[0]); const diffs = triangulate(sdkChunkShape, realFirstShape, mockFirstShape); - const report = formatDriftReport("Ollama /api/chat (streaming chunk)", diffs); + const report = formatDriftReport("Ollama /api/chat (streaming chunk)", diffs, "ollama"); expect( diffs.filter((d) => d.severity === "critical"), @@ -199,7 +199,7 @@ describe.skipIf(!process.env.OLLAMA_HOST)("Ollama drift", () => { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); - const report = formatDriftReport("Ollama /api/generate", diffs); + const report = formatDriftReport("Ollama /api/generate", diffs, "ollama"); expect( diffs.filter((d) => d.severity === "critical"), diff --git a/src/__tests__/drift/openai-chat.drift.ts b/src/__tests__/drift/openai-chat.drift.ts index 3151e6e3..8c08252f 100644 --- a/src/__tests__/drift/openai-chat.drift.ts +++ b/src/__tests__/drift/openai-chat.drift.ts @@ -58,7 +58,7 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Chat Completions drift", () => { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); - const report = formatDriftReport("OpenAI Chat (non-streaming text)", diffs); + const report = formatDriftReport("OpenAI Chat (non-streaming text)", diffs, "openai-chat"); expect( diffs.filter((d) => d.severity === "critical"), @@ -87,7 +87,7 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Chat Completions drift", () => { const mockChunkShape = extractShape(mockChunks[0]); const diffs = triangulate(sdkChunkShape, realChunkShape, mockChunkShape); - const report = formatDriftReport("OpenAI Chat (streaming text chunks)", diffs); + const report = formatDriftReport("OpenAI Chat (streaming text chunks)", diffs, "openai-chat"); expect( diffs.filter((d) => d.severity === "critical"), @@ -127,7 +127,7 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Chat Completions drift", () => { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); - const report = formatDriftReport("OpenAI Chat (non-streaming tool call)", diffs); + const report = formatDriftReport("OpenAI Chat (non-streaming tool call)", diffs, "openai-chat"); expect( diffs.filter((d) => d.severity === "critical"), @@ -172,7 +172,11 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Chat Completions drift", () => { const mockChunkShape = extractShape(mockChunks[0]); const diffs = triangulate(sdkChunkShape, realChunkShape, mockChunkShape); - const report = formatDriftReport("OpenAI Chat (streaming tool call chunks)", diffs); + const report = formatDriftReport( + "OpenAI Chat (streaming tool call chunks)", + diffs, + "openai-chat", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -239,7 +243,7 @@ describe("OpenAI Chat Completions error shapes", () => { const mockShape = extractShape(body); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("OpenAI Chat error fixture (400)", diffs); + const report = formatDriftReport("OpenAI Chat error fixture (400)", diffs, "openai-chat"); expect( diffs.filter((d) => d.severity === "critical"), @@ -274,7 +278,7 @@ describe("OpenAI Chat Completions error shapes", () => { const mockShape = extractShape(body); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("OpenAI Chat no-fixture-match (404)", diffs); + const report = formatDriftReport("OpenAI Chat no-fixture-match (404)", diffs, "openai-chat"); expect( diffs.filter((d) => d.severity === "critical"), @@ -330,7 +334,7 @@ describe("OpenAI Chat Completions error shapes", () => { const mockShape = extractShape(body); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("OpenAI Chat malformed JSON (400)", diffs); + const report = formatDriftReport("OpenAI Chat malformed JSON (400)", diffs, "openai-chat"); expect( diffs.filter((d) => d.severity === "critical"), @@ -387,7 +391,11 @@ describe("OpenAI Chat Completions reasoning shapes", () => { const mockShape = extractShape(body); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("OpenAI Chat (non-streaming reasoning)", diffs); + const report = formatDriftReport( + "OpenAI Chat (non-streaming reasoning)", + diffs, + "openai-chat", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -462,7 +470,11 @@ describe("OpenAI Chat Completions reasoning shapes", () => { const mockChunkShape = extractShape(reasoningChunks[0]); const diffs = triangulate(sdkChunkShape, sdkChunkShape, mockChunkShape); - const report = formatDriftReport("OpenAI Chat (streaming reasoning chunks)", diffs); + const report = formatDriftReport( + "OpenAI Chat (streaming reasoning chunks)", + diffs, + "openai-chat", + ); expect( diffs.filter((d) => d.severity === "critical"), diff --git a/src/__tests__/drift/openai-embeddings.drift.ts b/src/__tests__/drift/openai-embeddings.drift.ts index 3b3bee00..3d745d32 100644 --- a/src/__tests__/drift/openai-embeddings.drift.ts +++ b/src/__tests__/drift/openai-embeddings.drift.ts @@ -48,7 +48,7 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Embeddings drift", () => { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); - const report = formatDriftReport("OpenAI Embeddings", diffs); + const report = formatDriftReport("OpenAI Embeddings", diffs, "openai-embeddings"); expect( diffs.filter((d) => d.severity === "critical"), @@ -71,7 +71,11 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Embeddings drift", () => { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); - const report = formatDriftReport("OpenAI Embeddings (multiple inputs)", diffs); + const report = formatDriftReport( + "OpenAI Embeddings (multiple inputs)", + diffs, + "openai-embeddings", + ); expect( diffs.filter((d) => d.severity === "critical"), diff --git a/src/__tests__/drift/openai-responses.drift.ts b/src/__tests__/drift/openai-responses.drift.ts index 065b5876..a1060266 100644 --- a/src/__tests__/drift/openai-responses.drift.ts +++ b/src/__tests__/drift/openai-responses.drift.ts @@ -61,7 +61,11 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses API drift", () => { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); - const report = formatDriftReport("OpenAI Responses (non-streaming text)", diffs); + const report = formatDriftReport( + "OpenAI Responses (non-streaming text)", + diffs, + "openai-responses", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -92,7 +96,11 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses API drift", () => { })); const diffs = compareSSESequences(sdkEvents, realStream.events, mockSSEShapes); - const report = formatDriftReport("OpenAI Responses (streaming text events)", diffs); + const report = formatDriftReport( + "OpenAI Responses (streaming text events)", + diffs, + "openai-responses", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -130,7 +138,11 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses API drift", () => { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); - const report = formatDriftReport("OpenAI Responses (non-streaming tool call)", diffs); + const report = formatDriftReport( + "OpenAI Responses (non-streaming tool call)", + diffs, + "openai-responses", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -180,7 +192,11 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses API drift", () => { })); const diffs = compareSSESequences(sdkEvents, realStream.events, mockSSEShapes); - const report = formatDriftReport("OpenAI Responses (streaming tool call events)", diffs); + const report = formatDriftReport( + "OpenAI Responses (streaming tool call events)", + diffs, + "openai-responses", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -243,7 +259,11 @@ describe("OpenAI Responses API error shapes", () => { const mockShape = extractShape(body); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("OpenAI Responses (error fixture shape)", diffs); + const report = formatDriftReport( + "OpenAI Responses (error fixture shape)", + diffs, + "openai-responses", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -273,7 +293,11 @@ describe("OpenAI Responses API error shapes", () => { const mockShape = extractShape(body); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("OpenAI Responses (no-fixture-match error shape)", diffs); + const report = formatDriftReport( + "OpenAI Responses (no-fixture-match error shape)", + diffs, + "openai-responses", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -296,7 +320,11 @@ describe("OpenAI Responses API error shapes", () => { const mockShape = extractShape(body); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("OpenAI Responses (malformed request error shape)", diffs); + const report = formatDriftReport( + "OpenAI Responses (malformed request error shape)", + diffs, + "openai-responses", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -459,7 +487,11 @@ describe("OpenAI Responses API reasoning drift", () => { } const diffs = triangulate(sdkEvent.dataShape, sdkEvent.dataShape, mockEvent.dataShape); - const report = formatDriftReport(`OpenAI Responses Reasoning:${sdkEvent.type}`, diffs); + const report = formatDriftReport( + `OpenAI Responses Reasoning:${sdkEvent.type}`, + diffs, + "openai-responses", + ); expect( diffs.filter((d) => d.severity === "critical"), diff --git a/src/__tests__/drift/providers-infra-error.test.ts b/src/__tests__/drift/providers-infra-error.test.ts new file mode 100644 index 00000000..34019683 --- /dev/null +++ b/src/__tests__/drift/providers-infra-error.test.ts @@ -0,0 +1,77 @@ +/** + * Unit test for structured numeric `status` on InfraError (C5.1). + * + * Verifies that errors thrown through the `assertOk` → `withInfraErrorTag` + * pipeline carry a first-class numeric `.status` field, so downstream + * classification (C5.2 Slack, C5.3a preflight) can key off + * `status === 401 || status === 403` → stale-key vs `429`/`5xx` → infra-transient + * WITHOUT parsing prose message strings. + * + * RED captured (pre-change): `e.status` was `undefined` — no numeric status. + * GREEN (post-change): `e.status` is the exact HTTP status code. + */ +import { describe, it, expect } from "vitest"; +import { listOpenAIModels } from "./providers.js"; + +// Helper: build a minimal Response-like object +function fakeResponse(status: number, body = "Unauthorized"): Response { + return new Response(body, { status }); +} + +describe("InfraError structured status (C5.1)", () => { + it("GREEN: caught error exposes status === 401 (stale-key class)", async () => { + const origFetch = globalThis.fetch; + globalThis.fetch = async () => fakeResponse(401, "Unauthorized"); + try { + await listOpenAIModels("fake-key"); + throw new Error("should have thrown"); + } catch (err: unknown) { + expect(err).toBeInstanceOf(Error); + const e = err as Error & { status?: number }; + // Must be numeric 401, not undefined + expect(typeof e.status).toBe("number"); + expect(e.status).toBe(401); + // Human-readable INFRA_ERROR prose must still be present in message + expect((e as Error).message).toMatch(/INFRA_ERROR/); + } finally { + globalThis.fetch = origFetch; + } + }); + + it("GREEN: caught error exposes status === 403 (stale-key class)", async () => { + const origFetch = globalThis.fetch; + globalThis.fetch = async () => fakeResponse(403, "Forbidden"); + try { + await listOpenAIModels("fake-key"); + throw new Error("should have thrown"); + } catch (err: unknown) { + expect(err).toBeInstanceOf(Error); + const e = err as Error & { status?: number }; + expect(typeof e.status).toBe("number"); + expect(e.status).toBe(403); + expect((e as Error).message).toMatch(/INFRA_ERROR/); + } finally { + globalThis.fetch = origFetch; + } + }); + + it("GREEN: caught error exposes status === 503 (infra-transient class)", async () => { + const origFetch = globalThis.fetch; + // 503 is in RETRYABLE_STATUSES; fetchWithRetry retries attempts 0 and 1, + // then returns the response on attempt 2 (the last) without further retry. + // assertOk then fires on the 503 response. + globalThis.fetch = async () => fakeResponse(503, "Service Unavailable"); + try { + await listOpenAIModels("fake-key"); + throw new Error("should have thrown"); + } catch (err: unknown) { + expect(err).toBeInstanceOf(Error); + const e = err as Error & { status?: number }; + expect(typeof e.status).toBe("number"); + expect(e.status).toBe(503); + expect((e as Error).message).toMatch(/INFRA_ERROR/); + } finally { + globalThis.fetch = origFetch; + } + }); +}); diff --git a/src/__tests__/drift/providers.ts b/src/__tests__/drift/providers.ts index 03c80967..c2b2c309 100644 --- a/src/__tests__/drift/providers.ts +++ b/src/__tests__/drift/providers.ts @@ -27,6 +27,27 @@ interface StreamResult { rawEvents: { type: string; data: unknown }[]; } +/** + * Structured error carrying the HTTP status code as a first-class numeric field. + * + * Downstream classifiers (Slack alerts, preflight checks) MUST key off + * `error.status` — never parse the prose message string. + * + * 401 | 403 → stale-key (replace the API key) + * 429 → rate-limited (back off) + * 5xx → infra-transient (retry / alert ops) + * network → no status field (instanceof check first) + */ +export class InfraError extends Error { + readonly status: number; + + constructor(message: string, status: number) { + super(message); + this.name = "InfraError"; + this.status = status; + } +} + // --------------------------------------------------------------------------- // Retry helper // --------------------------------------------------------------------------- @@ -71,7 +92,10 @@ function redactUrl(url: string): string { function assertOk(raw: string, status: number, context: string, url?: string): void { if (status >= 400) { const urlSuffix = url ? ` (${redactUrl(url)})` : ""; - throw new Error(`${context}: API returned ${status}${urlSuffix}: ${raw.slice(0, 300)}`); + throw new InfraError( + `${context}: API returned ${status}${urlSuffix}: ${raw.slice(0, 300)}`, + status, + ); } } @@ -157,7 +181,8 @@ function toSSEEventShapes(events: { type: string; data: unknown }[]): SSEEventSh function withInfraErrorTag(provider: string, fn: () => Promise): Promise { return fn().catch((err: unknown) => { const msg = err instanceof Error ? err.message : String(err); - throw new Error(`INFRA_ERROR: ${provider} — ${msg}`); + const status = err instanceof InfraError ? err.status : 0; + throw new InfraError(`INFRA_ERROR: ${provider} — ${msg}`, status); }); } diff --git a/src/__tests__/drift/rerank.drift.ts b/src/__tests__/drift/rerank.drift.ts index 1a9da5e8..3f5a3125 100644 --- a/src/__tests__/drift/rerank.drift.ts +++ b/src/__tests__/drift/rerank.drift.ts @@ -123,7 +123,7 @@ describe.skipIf(!HAS_CREDENTIALS)("Cohere Rerank drift", () => { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, realShape, mockShape); - const report = formatDriftReport("Cohere /v2/rerank", diffs); + const report = formatDriftReport("Cohere /v2/rerank", diffs, "rerank"); expect( diffs.filter((d) => d.severity === "critical"), @@ -199,7 +199,7 @@ describe("Cohere Rerank mock-only shape validation", () => { // Two-way: SDK vs mock (no real API needed) const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Cohere /v2/rerank (mock vs SDK)", diffs); + const report = formatDriftReport("Cohere /v2/rerank (mock vs SDK)", diffs, "rerank"); expect( diffs.filter((d) => d.severity === "critical"), diff --git a/src/__tests__/drift/schema.ts b/src/__tests__/drift/schema.ts index 1a84a06e..663f4eb4 100644 --- a/src/__tests__/drift/schema.ts +++ b/src/__tests__/drift/schema.ts @@ -3,6 +3,8 @@ * for drift detection between SDK types, real API responses, and aimock output. */ +import { isKnownSurface } from "./surface-registry.js"; + // --------------------------------------------------------------------------- // Shape types // --------------------------------------------------------------------------- @@ -451,11 +453,35 @@ export function compareSSESequences( // Report formatting // --------------------------------------------------------------------------- -export function formatDriftReport(context: string, diffs: ShapeDiff[]): string { +/** + * Format a drift report block for a vitest failure message. + * + * @param context Human-readable prose title (kept for humans + the legacy + * no-marker collector fallback). + * @param diffs The shape diffs to render. + * @param surface Optional stable surface slug (see `surface-registry.ts`). When + * provided it MUST be a registered slug — an unknown slug throws at emit time + * so a typo/forgotten registry entry fails loudly locally. When present, a + * machine-readable `Surface: ` marker line is emitted so the collector + * resolves the block structurally (→ auto-fixable exit 2) instead of + * substring-matching the prose title. + */ +export function formatDriftReport(context: string, diffs: ShapeDiff[], surface?: string): string { + if (surface !== undefined && !isKnownSurface(surface)) { + throw new Error( + `formatDriftReport: unknown drift surface "${surface}" — add it to SURFACE_REGISTRY ` + + `in src/__tests__/drift/surface-registry.ts`, + ); + } + if (diffs.length === 0) return `No drift detected: ${context}`; const lines: string[] = []; - lines.push(`\nAPI DRIFT DETECTED: ${context}\n`); + lines.push(`\nAPI DRIFT DETECTED: ${context}`); + if (surface !== undefined) { + lines.push(` Surface: ${surface}`); + } + lines.push(""); for (let i = 0; i < diffs.length; i++) { const d = diffs[i]; diff --git a/src/__tests__/drift/surface-registry.ts b/src/__tests__/drift/surface-registry.ts new file mode 100644 index 00000000..32d7d89f --- /dev/null +++ b/src/__tests__/drift/surface-registry.ts @@ -0,0 +1,271 @@ +/** + * Single source of truth for drift *surfaces*. + * + * A "surface" is one provider-endpoint shape that emits an `API DRIFT + * DETECTED:` block (e.g. Cohere /v2/chat, Bedrock Invoke, OpenAI Images). Each + * surface is identified by a stable machine slug (e.g. `"cohere-chat"`) that the + * emitter declares via `formatDriftReport(context, diffs, surface)` and the + * drift-report collector resolves back to the source file(s) that must be fixed. + * + * WHY a registry (not title-substring matching): the collector historically + * keyed a drift block by `text.includes()` against ~9 + * hardcoded provider names. ~15 additional surfaces emit valid, fully-parseable + * drift blocks whose PROSE titles are not keys, so they resolved to `null` → + * quarantine (exit 5) → the auto-fix workflow (which gates on exit 2) never ran + * for them. A slug-keyed registry shared by BOTH the emitter (schema.ts) and the + * collector makes adding a surface a single edit that both consumers see, and + * lets an unkeyed-but-emitting surface fail LOUDLY instead of silently + * quarantining. + * + * This module is imported by: + * - `schema.ts` — validates a passed `surface` slug at emit time. + * - `scripts/drift-report-collector.ts` — resolves a slug to a source file. + * - `scripts/drift-report-collector.ts` legacy fallback — matches a no-marker + * block against the `provider` labels below (back-compat). + */ + +/** + * How a surface slug maps onto the source file(s) an auto-fixer must edit. + * + * Mirrors the fields the collector needs to build a `DriftEntry`. `fix-drift.ts` + * hard-requires `builderFile` (non-empty string), `builderFunctions` (non-empty + * string array), and `sdkShapesFile` (non-empty string) — so every entry here + * must resolve to a real file with real, non-invented function names. + */ +export interface SurfaceMapping { + /** Human-readable label used in the report/prompt and the legacy fallback. */ + provider: string; + /** Source file the fixer edits, e.g. `"src/cohere.ts"`. */ + builderFile: string; + /** + * Builder/handler function names in `builderFile` (or across the surface's + * source files) — non-empty. Steer the fixer's Read/Grep; NEVER invented. + */ + builderFunctions: string[]; + /** Types file for the surface, or null if shapes are inline. */ + typesFile: string | null; + /** + * Optional override for the SDK-shapes reference file. Defaults to the + * collector's `SDK_SHAPES_FILE` when omitted. + */ + sdkShapesFile?: string; +} + +const SDK_SHAPES_FILE = "src/__tests__/drift/sdk-shapes.ts"; + +/** + * Slug → surface mapping. ONE table, two consumers. Adding a new provider + * surface is a single edit here plus passing the slug at the emit site. + * + * Keys are stable machine slugs. Open-ended prose title suffixes (e.g. + * `Bedrock ConverseStream:`) do NOT affect the slug — the emitter + * passes the stable slug and keeps the suffix in the prose `context`. + */ +export const SURFACE_REGISTRY: Record = { + // --- Existing PROVIDER_MAP surfaces, migrated to slugs (unchanged mappings) --- + "openai-chat": { + provider: "OpenAI Chat", + builderFile: "src/helpers.ts", + builderFunctions: [ + "buildTextCompletion", + "buildToolCallCompletion", + "buildTextChunks", + "buildToolCallChunks", + ], + typesFile: "src/types.ts", + }, + "openai-responses": { + provider: "OpenAI Responses", + builderFile: "src/responses.ts", + builderFunctions: [ + "buildTextResponse", + "buildToolCallResponse", + "buildTextStreamEvents", + "buildToolCallStreamEvents", + ], + typesFile: null, + }, + anthropic: { + provider: "Anthropic Claude", + builderFile: "src/messages.ts", + builderFunctions: [ + "buildClaudeTextResponse", + "buildClaudeToolCallResponse", + "buildClaudeTextStreamEvents", + "buildClaudeToolCallStreamEvents", + ], + typesFile: null, + }, + gemini: { + provider: "Google Gemini", + builderFile: "src/gemini.ts", + builderFunctions: [ + "buildGeminiTextResponse", + "buildGeminiToolCallResponse", + "buildGeminiTextStreamChunks", + "buildGeminiToolCallStreamChunks", + ], + typesFile: null, + }, + "openai-realtime": { + provider: "OpenAI Realtime", + builderFile: "src/ws-realtime.ts", + builderFunctions: ["handleWebSocketRealtime", "realtimeItemsToMessages"], + typesFile: null, + }, + "openai-responses-ws": { + provider: "OpenAI Responses WS", + builderFile: "src/ws-responses.ts", + builderFunctions: ["handleWebSocketResponses"], + typesFile: null, + }, + "gemini-live": { + provider: "Gemini Live", + builderFile: "src/ws-gemini-live.ts", + builderFunctions: ["handleWebSocketGeminiLive"], + typesFile: null, + }, + "openai-embeddings": { + provider: "OpenAI Embeddings", + builderFile: "src/helpers.ts", + builderFunctions: ["buildEmbeddingResponse", "generateDeterministicEmbedding"], + typesFile: null, + sdkShapesFile: SDK_SHAPES_FILE, + }, + "gemini-interactions": { + provider: "Gemini Interactions", + builderFile: "src/gemini-interactions.ts", + builderFunctions: [ + "buildInteractionsTextResponse", + "buildInteractionsToolCallResponse", + "buildInteractionsContentWithToolCallsResponse", + "buildInteractionsTextSSEEvents", + "buildInteractionsToolCallSSEEvents", + "buildInteractionsContentWithToolCallsSSEEvents", + ], + typesFile: null, + }, + + // --- Previously unmapped surfaces (the hole) — now keyed. ------------------- + "cohere-chat": { + provider: "Cohere Chat", + builderFile: "src/cohere.ts", + builderFunctions: ["handleCohere", "cohereToCompletionRequest"], + typesFile: null, + }, + rerank: { + provider: "Cohere Rerank", + builderFile: "src/rerank.ts", + builderFunctions: ["handleRerank"], + typesFile: null, + }, + "bedrock-invoke": { + provider: "Bedrock Invoke", + builderFile: "src/bedrock.ts", + builderFunctions: [ + "handleBedrock", + "handleBedrockStream", + "bedrockToCompletionRequest", + "buildBedrockStreamTextEvents", + "buildBedrockStreamToolCallEvents", + "buildBedrockStreamContentWithToolCallsEvents", + ], + typesFile: null, + }, + "bedrock-invoke-stream": { + provider: "Bedrock InvokeStream", + builderFile: "src/bedrock.ts", + builderFunctions: [ + "handleBedrockStream", + "buildBedrockStreamTextEvents", + "buildBedrockStreamToolCallEvents", + "buildBedrockStreamContentWithToolCallsEvents", + ], + typesFile: null, + }, + "bedrock-converse": { + provider: "Bedrock Converse", + builderFile: "src/bedrock-converse.ts", + builderFunctions: ["handleConverse", "converseToCompletionRequest"], + typesFile: null, + }, + "bedrock-converse-stream": { + provider: "Bedrock ConverseStream", + builderFile: "src/bedrock-converse.ts", + builderFunctions: ["handleConverseStream", "converseToCompletionRequest"], + typesFile: null, + }, + ollama: { + provider: "Ollama", + builderFile: "src/ollama.ts", + builderFunctions: ["handleOllama", "handleOllamaGenerate", "ollamaToCompletionRequest"], + typesFile: null, + }, + "fal-sync": { + provider: "fal.ai sync-run", + builderFile: "src/fal.ts", + builderFunctions: ["handleFal", "imageResponseToFalJson", "videoResponseToFalJson"], + typesFile: null, + }, + "fal-queue": { + provider: "fal.ai queue", + builderFile: "src/fal.ts", + builderFunctions: ["handleFal", "walkFalQueue", "resolveProgression"], + typesFile: null, + }, + elevenlabs: { + provider: "ElevenLabs", + builderFile: "src/elevenlabs-audio.ts", + builderFunctions: ["handleElevenLabsTTS", "handleElevenLabsAudio"], + typesFile: null, + }, + images: { + provider: "OpenAI Images", + builderFile: "src/images.ts", + builderFunctions: ["handleImages", "handleImageEdit", "handleImageVariations"], + typesFile: null, + }, + video: { + provider: "OpenAI Video", + builderFile: "src/video.ts", + builderFunctions: ["handleVideoCreate", "handleVideoStatus"], + typesFile: null, + }, + moderation: { + provider: "OpenAI Moderations", + builderFile: "src/moderation.ts", + builderFunctions: ["handleModeration"], + typesFile: null, + }, + transcription: { + provider: "Transcription", + builderFile: "src/transcription.ts", + builderFunctions: ["handleTranscription", "extractFormField", "extractBoundary"], + typesFile: null, + }, + "vertex-ai": { + provider: "Vertex AI", + builderFile: "src/gemini.ts", + builderFunctions: [ + "handleGemini", + "buildGeminiTextResponse", + "buildGeminiToolCallResponse", + "buildGeminiTextStreamChunks", + "buildGeminiToolCallStreamChunks", + ], + typesFile: null, + }, +}; + +/** + * Every slug the emitter (`schema.ts` call sites) may pass. Exported so the + * registry-coverage test can assert each is a key of `SURFACE_REGISTRY` — a new + * surface either has an entry or fails the drift run loudly (belt-and-braces + * with the collector's runtime throw on an unknown marker slug). + */ +export const KNOWN_SURFACE_SLUGS: readonly string[] = Object.keys(SURFACE_REGISTRY); + +/** True when `slug` is a registered surface. */ +export function isKnownSurface(slug: string): slug is keyof typeof SURFACE_REGISTRY { + return Object.prototype.hasOwnProperty.call(SURFACE_REGISTRY, slug); +} diff --git a/src/__tests__/drift/transcription.drift.ts b/src/__tests__/drift/transcription.drift.ts index 71296b22..f6e2d3d7 100644 --- a/src/__tests__/drift/transcription.drift.ts +++ b/src/__tests__/drift/transcription.drift.ts @@ -117,7 +117,7 @@ describe("Transcription API shape validation", () => { const mockShape = extractShape(mockRes.body); // Two-way comparison: SDK vs mock (no real API call needed) const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Transcription basic JSON", diffs); + const report = formatDriftReport("Transcription basic JSON", diffs, "transcription"); expect( diffs.filter((d) => d.severity === "critical"), @@ -145,7 +145,7 @@ describe("Transcription API shape validation", () => { const mockShape = extractShape(body); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Transcription verbose JSON", diffs); + const report = formatDriftReport("Transcription verbose JSON", diffs, "transcription"); expect( diffs.filter((d) => d.severity === "critical"), @@ -302,7 +302,7 @@ describe.skipIf(!OPENAI_API_KEY)("Transcription drift (three-way)", () => { const mockShape = extractShape(mockRes.body); const diffs = triangulate(sdkShape, realShape, mockShape); - const report = formatDriftReport("Transcription basic (three-way)", diffs); + const report = formatDriftReport("Transcription basic (three-way)", diffs, "transcription"); expect( diffs.filter((d) => d.severity === "critical"), @@ -327,7 +327,7 @@ describe.skipIf(!OPENAI_API_KEY)("Transcription drift (three-way)", () => { const mockShape = extractShape(mockRes.body); const diffs = triangulate(sdkShape, realShape, mockShape); - const report = formatDriftReport("Transcription verbose (three-way)", diffs); + const report = formatDriftReport("Transcription verbose (three-way)", diffs, "transcription"); expect( diffs.filter((d) => d.severity === "critical"), diff --git a/src/__tests__/drift/vertex-ai.drift.ts b/src/__tests__/drift/vertex-ai.drift.ts index 0a885c82..b3777f40 100644 --- a/src/__tests__/drift/vertex-ai.drift.ts +++ b/src/__tests__/drift/vertex-ai.drift.ts @@ -91,7 +91,7 @@ describe.skipIf(!HAS_CREDENTIALS)("Vertex AI drift", () => { if (mockRes.status === 200) { const mockShape = extractShape(JSON.parse(mockRes.body)); const diffs = triangulate(sdkShape, sdkShape, mockShape); - const report = formatDriftReport("Vertex AI generateContent", diffs); + const report = formatDriftReport("Vertex AI generateContent", diffs, "vertex-ai"); expect( diffs.filter((d) => d.severity === "critical"), @@ -156,7 +156,11 @@ describe.skipIf(!HAS_CREDENTIALS)("Vertex AI drift", () => { const lastChunk = chunks[chunks.length - 1]; const lastShape = extractShape(lastChunk); const diffs = triangulate(sdkChunkShape, sdkChunkShape, lastShape); - const report = formatDriftReport("Vertex AI streamGenerateContent (last chunk)", diffs); + const report = formatDriftReport( + "Vertex AI streamGenerateContent (last chunk)", + diffs, + "vertex-ai", + ); expect( diffs.filter((d) => d.severity === "critical"), diff --git a/src/__tests__/drift/video.drift.ts b/src/__tests__/drift/video.drift.ts index 7ce03524..a6efcddd 100644 --- a/src/__tests__/drift/video.drift.ts +++ b/src/__tests__/drift/video.drift.ts @@ -127,7 +127,7 @@ describe("Video API drift", () => { expect(res.status).toBe(200); const mockShape = extractShape(JSON.parse(res.body)); const diffs = compareShapes(expected, mockShape); - const report = formatDriftReport("Video create (completed)", diffs); + const report = formatDriftReport("Video create (completed)", diffs, "video"); expect( diffs.filter((d) => d.severity === "critical"), @@ -147,7 +147,7 @@ describe("Video API drift", () => { const body = JSON.parse(res.body); const mockShape = extractShape(body); const diffs = compareShapes(expected, mockShape); - const report = formatDriftReport("Video create (processing)", diffs); + const report = formatDriftReport("Video create (processing)", diffs, "video"); // Processing response must NOT include url expect(body.url).toBeUndefined(); @@ -173,7 +173,7 @@ describe("Video API drift", () => { expect(res.status).toBe(200); const mockShape = extractShape(JSON.parse(res.body)); const diffs = compareShapes(expected, mockShape); - const report = formatDriftReport("Video status (completed)", diffs); + const report = formatDriftReport("Video status (completed)", diffs, "video"); expect( diffs.filter((d) => d.severity === "critical"), @@ -195,7 +195,7 @@ describe("Video API drift", () => { const body = JSON.parse(res.body); const mockShape = extractShape(body); const diffs = compareShapes(expected, mockShape); - const report = formatDriftReport("Video status (processing)", diffs); + const report = formatDriftReport("Video status (processing)", diffs, "video"); // Processing status must NOT include url expect(body.url).toBeUndefined(); diff --git a/src/__tests__/drift/voice-models.ts b/src/__tests__/drift/voice-models.ts new file mode 100644 index 00000000..57b24895 --- /dev/null +++ b/src/__tests__/drift/voice-models.ts @@ -0,0 +1,144 @@ +/** + * Known voice/audio model FAMILIES + drift detection for the OpenAI realtime + * canary. Detection compares each candidate's NORMALIZED family (trailing dated + * snapshot / build-tag suffixes stripped) against a known-family set, so dated + * snapshots of a known family don't churn as false-positive drift. + * + * Extracted into its own side-effect-free module (no `describe`/`beforeAll`) so + * both the live canary in ws-realtime.drift.ts AND its unit test can import the + * SAME detection code path without the unit test transitively registering the + * drift suite (which would spin up the drift server). + */ + +import { normalizeModelFamily } from "./model-family.js"; + +/** + * The GA realtime model FAMILIES. At least one voice/audio model whose + * normalized family (see `normalizeVoiceModelFamily`) is one of these MUST + * appear in the account's model list, otherwise the family was renamed/removed + * (NO_GA drift). Entries are already normalized family keys — dated snapshots + * such as `gpt-realtime-2025-08-28` collapse onto `gpt-realtime` and so match + * here without being listed separately. + */ +export const gaRealtimeModels = [ + "gpt-realtime", + "gpt-realtime-2", + "gpt-realtime-2.1", + "gpt-realtime-2.1-mini", + "gpt-realtime-1.5", + "gpt-realtime-mini", +]; + +/** + * Normalize a voice/audio model id to its FAMILY KEY by stripping trailing + * version/snapshot suffixes that OpenAI appends to already-known families. New + * dated snapshots of an existing family land constantly (`tts-1-1106`, + * `gpt-audio-2025-08-28`, `gpt-4o-mini-tts-2025-12-15`, …); appending every one + * to a known-ID set never converges and turns the daily drift job permanently + * red on false positives. Comparing the NORMALIZED family instead means only a + * genuinely new family (e.g. `gpt-live`) is ever flagged. + * + * This is a thin OpenAI-provider wrapper over the shared `normalizeModelFamily` + * primitive (see `model-family.ts`), which owns the dated-snapshot/build-tag + * strip loop. Behavior is byte-identical to the historical inline implementation + * — a short numeric suffix like `gpt-live-1`'s trailing `-1` is a SINGLE digit + * and is deliberately NOT stripped, so `gpt-live-1` normalizes to `gpt-live-1` + * — an unknown family — and stays flagged (the whole point of the canary). + */ +export function normalizeVoiceModelFamily(id: string): string { + return normalizeModelFamily(id, "openai"); +} + +/** + * The set of voice/audio model FAMILIES we already know about, keyed by the + * normalized family (see `normalizeVoiceModelFamily`). A voice/audio model whose + * NORMALIZED family is not in this set is surfaced as new/unknown drift, so a + * newly-shipped family (e.g. `gpt-live`) is flagged the first time it appears — + * while dated snapshots of a known family (e.g. `gpt-audio-2025-08-28`) collapse + * onto their family and stay green. + * + * The listed ids are the family keys; the seed values are already normalized + * (they carry no dated/build suffix), so building the set through + * `normalizeVoiceModelFamily` is idempotent and keeps the two in lockstep. + */ +export const knownVoiceModelFamilies = new Set( + [ + // GA realtime family (dated/versioned variants normalize onto these). + "gpt-realtime", + "gpt-realtime-2", + "gpt-realtime-2.1", + "gpt-realtime-2.1-mini", + "gpt-realtime-1.5", + "gpt-realtime-mini", + // Translate/whisper realtime variants + "gpt-realtime-translate", + "gpt-realtime-whisper", + // Audio models also valid in realtime sessions + "gpt-audio", + "gpt-audio-1.5", + "gpt-audio-mini", + // Transcription/translation models + "gpt-4o-transcribe", + "gpt-4o-mini-transcribe", + "gpt-4o-transcribe-diarize", + "whisper-1", + // Legacy preview models (may still appear) + "gpt-4o-realtime-preview", + "gpt-4o-mini-realtime-preview", + // TTS / speech-out models (voice family, no "realtime" substring) + "gpt-4o-mini-tts", + "tts-1", + "tts-1-hd", + ].map(normalizeVoiceModelFamily), +); + +/** + * Match a model id that belongs to the voice/audio family the realtime canary + * is responsible for. This is DELIBERATELY broader than the old + * `id.includes("realtime")` filter: a new full-duplex voice family whose id + * lacks the "realtime" substring (e.g. OpenAI's `gpt-live-1` / `gpt-live-1-mini`) + * would previously never enter the unknown-model computation and so slip past + * the canary silently. Matching on the broader voice/audio vocabulary closes + * that blind spot generally — the point is "a new audio/voice model family the + * account hasn't seen before gets flagged", not a one-off hardcode of gpt-live. + * + * Chat/text/image/embedding models (gpt-4o, gpt-5, dall-e, text-embedding-*, + * etc.) do NOT match, so they never become false-positive "unknown voice" drift. + */ +export function isVoiceModelId(id: string): boolean { + return /(?:realtime|audio|\blive\b|-live|transcribe|whisper|voice|\btts\b|-tts)/i.test(id); +} + +/** + * Result of running the known-voice-models drift detection over a model list. + */ +export interface VoiceModelDriftResult { + /** Every model id that matched the voice/audio family matcher. */ + candidateModels: string[]; + /** + * Voice/audio ids whose NORMALIZED family is not in knownVoiceModelFamilies — + * new/unknown drift. Dated snapshots of a known family (e.g. + * `gpt-audio-2025-08-28`) collapse onto their family and are NOT listed. + */ + unknown: string[]; + /** Whether at least one GA realtime model (by family) is present. */ + hasGA: boolean; +} + +/** + * The single detection code path shared by the live canary AND its unit test. + * Given a raw `GET /v1/models` id list, compute the voice/audio candidates, the + * unknown (new-family) subset, and GA presence. Keeping this pure lets the unit + * test drive the EXACT logic the live canary runs against a representative + * payload without a network call. + */ +export function detectVoiceModelDrift(models: string[]): VoiceModelDriftResult { + const candidateModels = models.filter(isVoiceModelId); + const unknown = candidateModels.filter( + (m) => !knownVoiceModelFamilies.has(normalizeVoiceModelFamily(m)), + ); + const hasGA = candidateModels.some((m) => + gaRealtimeModels.includes(normalizeVoiceModelFamily(m)), + ); + return { candidateModels, unknown, hasGA }; +} diff --git a/src/__tests__/drift/ws-gemini-live.drift.ts b/src/__tests__/drift/ws-gemini-live.drift.ts index 28ce25b5..0ec45f5b 100644 --- a/src/__tests__/drift/ws-gemini-live.drift.ts +++ b/src/__tests__/drift/ws-gemini-live.drift.ts @@ -141,7 +141,7 @@ describe.skipIf(!GOOGLE_API_KEY)("Gemini Live WS drift", () => { expect(mockEvents.length, "Mock returned no WS messages").toBeGreaterThan(0); const diffs = compareSSESequences(sdkEvents, realResult.events, mockEvents); - const report = formatDriftReport("Gemini Live WS (text events)", diffs); + const report = formatDriftReport("Gemini Live WS (text events)", diffs, "gemini-live"); expect( diffs.filter((d) => d.severity === "critical"), @@ -218,7 +218,7 @@ describe.skipIf(!GOOGLE_API_KEY)("Gemini Live WS drift", () => { expect(mockEvents.length, "Mock returned no WS messages").toBeGreaterThan(0); const diffs = compareSSESequences(sdkEvents, realResult.events, mockEvents); - const report = formatDriftReport("Gemini Live WS (tool call events)", diffs); + const report = formatDriftReport("Gemini Live WS (tool call events)", diffs, "gemini-live"); expect( diffs.filter((d) => d.severity === "critical"), diff --git a/src/__tests__/drift/ws-realtime.drift.ts b/src/__tests__/drift/ws-realtime.drift.ts index 0fd9253f..7450ea6c 100644 --- a/src/__tests__/drift/ws-realtime.drift.ts +++ b/src/__tests__/drift/ws-realtime.drift.ts @@ -11,6 +11,7 @@ import { extractShape, compareSSESequences, formatDriftReport } from "./schema.j import { openaiRealtimeTextEventShapes, openaiRealtimeToolCallEventShapes } from "./sdk-shapes.js"; import { openaiRealtimeWS } from "./ws-providers.js"; import { listOpenAIModels } from "./providers.js"; +import { detectVoiceModelDrift } from "./voice-models.js"; import { startDriftServer, stopDriftServer, collectMockWSMessages } from "./helpers.js"; import { connectWebSocket } from "../ws-test-client.js"; @@ -35,7 +36,18 @@ const BETA_SUPPRESSED_EVENTS = new Set(["conversation.item.done"]); // --------------------------------------------------------------------------- let instance: ServerInstance; +// The known-models canary — the entire reason this suite exists — fetches the +// model list via GET /v1/models, which returns ALL models for ANY valid OpenAI +// key regardless of realtime access. So it uses OPENAI_API_KEY (the credential +// CI actually provides to the drift job) and is gated ONLY on OPENAI_API_KEY, +// guaranteeing it runs in CI and cannot silently skip. +// +// The real realtime WS *session* tests connect a live socket that legitimately +// requires realtime access, so they gate on OPENAI_REALTIME_KEY and pass that +// same credential in their session config — using the chat-only OPENAI_API_KEY +// there could produce a spurious auth-failure "drift" if the two keys differ. const OPENAI_API_KEY = process.env.OPENAI_API_KEY; +const OPENAI_REALTIME_KEY = process.env.OPENAI_REALTIME_KEY; beforeAll(async () => { instance = await startDriftServer(); @@ -50,55 +62,59 @@ afterAll(async () => { // --------------------------------------------------------------------------- describe.skipIf(!OPENAI_API_KEY)("OpenAI Realtime API drift", () => { - const config = { apiKey: OPENAI_API_KEY! }; + // Session config for the real WS realtime tests uses the realtime credential, + // NOT the chat-only OPENAI_API_KEY, so that a differing realtime key can't + // cause a spurious auth-failure "drift". The canary below does NOT use this. + const config = { apiKey: OPENAI_REALTIME_KEY! }; it("canary: GA realtime models available", async () => { - const models = await listOpenAIModels(config.apiKey); - - const gaModels = [ - "gpt-realtime", - "gpt-realtime-2", - "gpt-realtime-2025-08-28", - "gpt-realtime-1.5", - "gpt-realtime-mini", - "gpt-realtime-mini-2025-10-06", - "gpt-realtime-mini-2025-12-15", - ]; - const knownModels = new Set([ - ...gaModels, - // Translate/whisper models (also contain "realtime" in some variants) - "gpt-realtime-translate", - "gpt-realtime-whisper", - // Audio models also valid in realtime sessions - "gpt-audio-1.5", - "gpt-audio-mini", - "gpt-audio-mini-2025-10-06", - "gpt-audio-mini-2025-12-15", - // Transcription/translation models - "gpt-4o-transcribe", - "gpt-4o-mini-transcribe", - "whisper-1", - // Legacy preview models (may still appear) - "gpt-4o-realtime-preview", - "gpt-4o-mini-realtime-preview", - "gpt-4o-realtime-preview-2024-10-01", - "gpt-4o-realtime-preview-2024-12-17", - "gpt-4o-realtime-preview-2025-06-03", - "gpt-4o-mini-realtime-preview-2024-12-17", - ]); - - const realtimeModels = models.filter((m) => m.includes("realtime")); - - // At least one GA model should exist - const hasGA = realtimeModels.some((m) => gaModels.includes(m)); - expect(hasGA).toBe(true); - - // Flag unknown realtime models - const unknown = realtimeModels.filter((m) => !knownModels.has(m)); + // Fetch via GET /v1/models, which lists ALL models for ANY valid key. Use + // OPENAI_API_KEY (guaranteed present by the describe.skipIf above and the + // credential CI provides) so the canary ALWAYS runs in CI and never skips — + // gating this on OPENAI_REALTIME_KEY (which CI does NOT provide) would have + // silently skipped the one check this whole suite exists to run. + const models = await listOpenAIModels(OPENAI_API_KEY!); + + // Run the SHARED detection code path (also driven directly by the unit test + // in ws-realtime-canary.test.ts). The voice/audio family matcher is broader + // than the old `includes("realtime")` filter so a NEW voice family whose id + // lacks the "realtime" substring (e.g. gpt-live-1) is still flagged. + const { candidateModels: realtimeModels, unknown, hasGA } = detectVoiceModelDrift(models); + + // Compute the unknown-model list BEFORE the hasGA assertion. A run can be + // BOTH GA-family-gone AND carry new unknown models; because the hasGA + // assertion below throws first, the later unknown-models assertion would + // never run and its list would be lost from the NO_GA failure message (and + // therefore from the auto-fix prompt). So we carry the unknown list into the + // NO_GA marker too — no information is lost in the combined case. if (unknown.length > 0) { - console.warn(`[DRIFT] Unknown realtime models detected: ${unknown.join(", ")}`); + console.warn(`[DRIFT] Unknown voice/audio models detected: ${unknown.join(", ")}`); } - expect(unknown).toEqual([]); + + // At least one GA model should exist. Carry the OBSERVED realtime models in + // a stable custom assertion message (symmetric to UNKNOWN_REALTIME_MODELS= + // below) so that when the GA family is renamed/removed — or the credential + // cannot see any realtime models — the drift collector recognizes the + // NO_GA_REALTIME_MODELS= marker and emits a CRITICAL OpenAI-Realtime entry + // (exit 2, auto-remediated) instead of crashing to exit 1. Without the + // marker, "expected false to be true" is an unrecognized shape that the + // collector would treat as unparseable and throw on. + // + // The message ALSO carries the unknown list after a ` | UNKNOWN_REALTIME_MODELS=` + // segment so the combined (no-GA AND unknown-models-present) case does not + // lose the unknown list when this assertion short-circuits the one below. + // The collector splits the two markers apart; each list stays clean. + expect( + hasGA, + `NO_GA_REALTIME_MODELS=${realtimeModels.join(",")} | UNKNOWN_REALTIME_MODELS=${unknown.join(",")}`, + ).toBe(true); + + // Carry the FULL unknown-model list in a stable custom assertion message. + // vitest truncates the printed array in failureMessages (`…(N)`), so the + // drift collector cannot recover ids beyond the first from the array. The + // custom message is emitted verbatim and is NOT truncated, so the collector + // parses the UNKNOWN_REALTIME_MODELS= marker as its source of truth. + expect(unknown, `UNKNOWN_REALTIME_MODELS=${unknown.join(",")}`).toEqual([]); }); it.skipIf(!process.env.OPENAI_REALTIME_KEY)( @@ -166,7 +182,11 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Realtime API drift", () => { expect(mockEvents.length, "Mock returned no WS messages").toBeGreaterThan(0); const diffs = compareSSESequences(sdkEvents, realResult.events, mockEvents); - const report = formatDriftReport("OpenAI Realtime WS (GA text events)", diffs); + const report = formatDriftReport( + "OpenAI Realtime WS (GA text events)", + diffs, + "openai-realtime", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -263,7 +283,11 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Realtime API drift", () => { expect(mockEvents.length, "Mock returned no WS messages").toBeGreaterThan(0); const diffs = compareSSESequences(sdkEvents, realResult.events, mockEvents); - const report = formatDriftReport("OpenAI Realtime WS (GA tool call events)", diffs); + const report = formatDriftReport( + "OpenAI Realtime WS (GA tool call events)", + diffs, + "openai-realtime", + ); expect( diffs.filter((d) => d.severity === "critical"), diff --git a/src/__tests__/drift/ws-responses.drift.ts b/src/__tests__/drift/ws-responses.drift.ts index e8697a66..4dc99232 100644 --- a/src/__tests__/drift/ws-responses.drift.ts +++ b/src/__tests__/drift/ws-responses.drift.ts @@ -63,7 +63,11 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses WS drift", () => { expect(mockResult.events.length, "Mock returned no WS messages").toBeGreaterThan(0); const diffs = compareSSESequences(sdkEvents, realResult.events, mockResult.events); - const report = formatDriftReport("OpenAI Responses WS (text events)", diffs); + const report = formatDriftReport( + "OpenAI Responses WS (text events)", + diffs, + "openai-responses-ws", + ); expect( diffs.filter((d) => d.severity === "critical"), @@ -119,7 +123,11 @@ describe.skipIf(!OPENAI_API_KEY)("OpenAI Responses WS drift", () => { expect(mockResult.events.length, "Mock returned no WS messages").toBeGreaterThan(0); const diffs = compareSSESequences(sdkEvents, realResult.events, mockResult.events); - const report = formatDriftReport("OpenAI Responses WS (tool call events)", diffs); + const report = formatDriftReport( + "OpenAI Responses WS (tool call events)", + diffs, + "openai-responses-ws", + ); expect( diffs.filter((d) => d.severity === "critical"), diff --git a/src/__tests__/fix-drift-invoke.test.ts b/src/__tests__/fix-drift-invoke.test.ts new file mode 100644 index 00000000..e000c2ce --- /dev/null +++ b/src/__tests__/fix-drift-invoke.test.ts @@ -0,0 +1,77 @@ +/** + * WS-4 residual lock (slot2-F1) — a SYNCHRONOUS throw during the Promise + * executor setup of `invokeClaudeCode` must NOT strand the already-armed 30-min + * `killTimer`. + * + * The executor arms `killTimer` (a 30-minute setTimeout that would later + * group-kill `child.pid`) BEFORE it wires the stdout/stderr handlers. If + * `child.stdout` is null (a spawn edge case), `child.stdout.on(...)` throws a + * TypeError synchronously inside the executor. Without the fix, that throw + * rejects the Promise but leaves `killTimer` LIVE — a leaked 30-min timer that + * could later SIGKILL a reused PID. The fix clears the timer before rejecting. + * + * We mock `spawn` to return a fake child with NULL streams, drive + * `invokeClaudeCode`, and assert (a) the Promise REJECTS (not hangs) and (b) no + * timer is left pending (fake timers report zero pending after the reject). + */ +import { EventEmitter } from "node:events"; + +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +vi.mock("node:child_process", async () => { + const actual = await vi.importActual("node:child_process"); + return { ...actual, spawn: vi.fn() }; +}); + +import { spawn } from "node:child_process"; + +import { invokeClaudeCode } from "../../scripts/fix-drift.js"; + +const mockedSpawn = vi.mocked(spawn); + +/** A fake detached child whose stdout/stderr are NULL (the edge case). */ +function makeNullStreamChild(pid = 4242): EventEmitter & { + pid: number; + stdout: null; + stderr: null; +} { + const child = new EventEmitter() as EventEmitter & { + pid: number; + stdout: null; + stderr: null; + }; + child.pid = pid; + child.stdout = null; + child.stderr = null; + return child; +} + +describe("invokeClaudeCode — executor-setup throw must not strand killTimer (WS-4/slot2-F1)", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.useFakeTimers(); + vi.spyOn(console, "error").mockImplementation(() => {}); + }); + + afterEach(() => { + vi.clearAllTimers(); + vi.useRealTimers(); + vi.restoreAllMocks(); + }); + + it("REJECTS when the child has null streams (no stdout/stderr to attach handlers to)", async () => { + mockedSpawn.mockReturnValue(makeNullStreamChild() as never); + await expect(invokeClaudeCode("prompt")).rejects.toThrow(/no stdout\/stderr pipe/i); + }); + + it("clears the 30-min killTimer on the setup-throw path — NO timer is left pending", async () => { + mockedSpawn.mockReturnValue(makeNullStreamChild() as never); + + // The executor arms killTimer, then throws attaching the null-stream + // handler. The fix must clearTimeout(killTimer) before rejecting, so after + // the reject settles there must be ZERO pending fake timers. Without the + // fix, the 30-min killTimer would still be pending here. + await expect(invokeClaudeCode("prompt")).rejects.toThrow(); + expect(vi.getTimerCount()).toBe(0); + }); +}); diff --git a/src/__tests__/fix-drift-kill.test.ts b/src/__tests__/fix-drift-kill.test.ts new file mode 100644 index 00000000..0f0e62d8 --- /dev/null +++ b/src/__tests__/fix-drift-kill.test.ts @@ -0,0 +1,235 @@ +/** + * WS-4 regression locks — real process-group subprocess control. + * + * The original `invokeClaudeCode` had two live defects: + * 1. `spawn("npx", …)` had NO `detached: true`, so signalling the child pid + * reached only the `npx` wrapper, never the `@anthropic-ai/claude-code` + * grandchild — a wedged fixer survived and burned the 30-min job budget. + * 2. The SIGKILL escalation was gated on `if (!child.killed)`, but Node sets + * `child.killed = true` the instant SIGTERM is DELIVERED (not when the + * process exits), so `!child.killed` was ~always false and SIGKILL NEVER + * fired against a process that ignored SIGTERM. + * + * These tests exercise a REAL controlled subprocess that traps SIGTERM and + * sleeps, proving the OLD logic leaves it alive and the NEW logic + * (`killProcessGroup` + `scheduleEscalatingKill`, gated on a real has-exited + * flag) kills it and its whole group within the grace window. + */ +import { spawn } from "node:child_process"; + +import { describe, it, expect, vi } from "vitest"; + +/** Real subprocess spin-up + grace windows need more than the default budget. */ +const SUBPROC_TIMEOUT = 15000; + +import { killProcessGroup, scheduleEscalatingKill } from "../../scripts/fix-drift.js"; + +/** A child that TRAPS SIGTERM and keeps sleeping — models a wedged fixer. */ +const WEDGED_CHILD = ` +process.on("SIGTERM", () => { /* ignore — wedged, refuse to die on SIGTERM */ }); +setTimeout(() => process.exit(0), 60000); +`; + +/** A child that exits cleanly on SIGTERM — models a well-behaved fixer. */ +const OBEDIENT_CHILD = ` +process.on("SIGTERM", () => process.exit(0)); +setTimeout(() => process.exit(0), 60000); +`; + +function isAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch { + return false; + } +} + +function sleep(ms: number): Promise { + return new Promise((r) => setTimeout(r, ms)); +} + +describe("killProcessGroup", () => { + it( + "delivers a signal to the whole GROUP of a detached child (kills a SIGTERM-trapping grandchild-style process)", + { timeout: SUBPROC_TIMEOUT }, + async () => { + const child = spawn("node", ["-e", WEDGED_CHILD], { stdio: "ignore", detached: true }); + const pid = child.pid!; + await sleep(300); + expect(isAlive(pid)).toBe(true); + + // SIGTERM to the group is IGNORED by the wedged child (it traps it). + expect(killProcessGroup(pid, "SIGTERM")).toBe(true); + await sleep(200); + expect(isAlive(pid)).toBe(true); // still alive — it trapped SIGTERM + + // SIGKILL to the GROUP cannot be trapped — it dies. + expect(killProcessGroup(pid, "SIGKILL")).toBe(true); + await sleep(400); + expect(isAlive(pid)).toBe(false); + }, + ); + + it("tolerates ESRCH (group already gone) and returns false, never throwing", () => { + // A pid that is essentially certain not to exist as a group leader. + const missing = 2 ** 30; + expect(() => killProcessGroup(missing, "SIGTERM")).not.toThrow(); + expect(killProcessGroup(missing, "SIGTERM")).toBe(false); + }); + + it("does NOT silently treat EPERM as success — logs a visible warning and attempts a single-PID fallback (slot2-F5)", () => { + // EPERM means the group EXISTS but is unkillable by us (re-credentialed / + // re-parented child) — it may still be ALIVE burning the budget. It must + // NOT be swallowed as a benign "nothing to kill" like ESRCH. Assert we (a) + // log a distinct WARNING and (b) attempt the single-PID fallback. + const errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + let call = 0; + const killSpy = vi.spyOn(process, "kill").mockImplementation(((pidArg: number) => { + call += 1; + if (call === 1) { + // First call: group signal (negative pid) → EPERM. + expect(pidArg).toBeLessThan(0); + const e = new Error("EPERM") as NodeJS.ErrnoException; + e.code = "EPERM"; + throw e; + } + // Second call: the single-PID fallback (positive pid) succeeds. + expect(pidArg).toBeGreaterThan(0); + return true; + }) as never); + try { + expect(killProcessGroup(12345, "SIGKILL")).toBe(true); // fallback delivered + expect(call).toBe(2); // group attempt + single-PID fallback + const warned = errSpy.mock.calls.some((c) => String(c[0]).includes("EPERM")); + expect(warned).toBe(true); // distinct, visible EPERM warning — not silent + } finally { + killSpy.mockRestore(); + errSpy.mockRestore(); + } + }); + + it("re-throws unexpected errors (e.g. EINVAL from a bad signal)", () => { + const child = spawn("node", ["-e", OBEDIENT_CHILD], { stdio: "ignore", detached: true }); + const pid = child.pid!; + try { + // An invalid signal name yields a non-ESRCH/EPERM error, which must + // propagate rather than be swallowed as "nothing to kill". + expect(() => killProcessGroup(pid, "SIGNOTAREALSIGNAL" as NodeJS.Signals)).toThrow(); + } finally { + try { + process.kill(-pid, "SIGKILL"); + } catch { + /* best effort */ + } + } + }); +}); + +describe("scheduleEscalatingKill — SIGKILL escalation gated on a REAL exit flag", () => { + it( + "SIGKILLs a wedged (SIGTERM-trapping) subprocess group within the grace window (GREEN)", + { timeout: SUBPROC_TIMEOUT }, + async () => { + const child = spawn("node", ["-e", WEDGED_CHILD], { stdio: "ignore", detached: true }); + const pid = child.pid!; + let exited = false; + child.on("close", () => { + exited = true; + }); + await sleep(300); + expect(isAlive(pid)).toBe(true); + + // Short grace so the test is fast. hasExited() is backed by the real + // `close` event, NOT child.killed. + const timer = scheduleEscalatingKill(pid, () => exited, 150); + // Before the grace elapses, SIGTERM has been delivered but the wedged child + // is still alive (it trapped SIGTERM). + await sleep(80); + expect(isAlive(pid)).toBe(true); + // After the grace, the escalation SIGKILLs the group. + await sleep(400); + expect(isAlive(pid)).toBe(false); + clearTimeout(timer); + }, + ); + + it( + "does NOT SIGKILL when hasExited() is true — the gate genuinely SKIPS escalation (a SIGTERM-trapping child that WOULD have died to a stray SIGKILL stays alive)", + { timeout: SUBPROC_TIMEOUT }, + async () => { + // The previous version of this test used an OBEDIENT child and asserted + // it was dead — but an obedient child dies from the unconditional SIGTERM + // that scheduleEscalatingKill sends FIRST, regardless of whether the + // SIGKILL escalation is skipped, so it passed for the WRONG reason (it + // could not distinguish "escalation skipped" from "escalation fired"). + // + // Use a WEDGED child that TRAPS SIGTERM instead: the first SIGTERM leaves + // it alive, so the ONLY thing that could kill it within the window is the + // SIGKILL escalation. With hasExited() forced true, that escalation MUST + // be skipped — so the child must remain ALIVE after the grace elapses. If + // the has-exited gate regressed to always-escalate (the WS-4 defect), the + // group SIGKILL would fire and the child would be DEAD — turning this RED. + const child = spawn("node", ["-e", WEDGED_CHILD], { stdio: "ignore", detached: true }); + const pid = child.pid!; + await sleep(300); + expect(isAlive(pid)).toBe(true); + + const timer = scheduleEscalatingKill(pid, () => true, 100); + // Wait well past the grace window: the escalation callback has run (and, + // gated on hasExited()===true, SKIPPED the SIGKILL). + await sleep(400); + // Wedged child trapped the SIGTERM and no SIGKILL was sent → STILL ALIVE. + expect(isAlive(pid)).toBe(true); + + // The grace timer must have already fired-and-skipped (not still pending): + // clearing it now is a no-op, and no LATE SIGKILL can arrive after this. + clearTimeout(timer); + await sleep(200); + expect(isAlive(pid)).toBe(true); // no late kill + + // Clean up the still-alive wedged group. + try { + process.kill(-pid, "SIGKILL"); + } catch { + /* best effort */ + } + await sleep(300); + expect(isAlive(pid)).toBe(false); + }, + ); + + it( + "the returned grace timer, once cleared before it fires, delivers NO late SIGKILL to a still-running group", + { timeout: SUBPROC_TIMEOUT }, + async () => { + // Locks the caller-side lifecycle used by invokeClaudeCode's `close` + // handler: on a clean early exit the caller clears the returned timer, and + // a pending SIGKILL escalation must NOT fire late against a (possibly + // reused) PID. Here the child is still running and hasExited() would be + // false, so ONLY a fired escalation could kill it — we clear the timer + // BEFORE the grace elapses and assert the child survives. + const child = spawn("node", ["-e", WEDGED_CHILD], { stdio: "ignore", detached: true }); + const pid = child.pid!; + await sleep(300); + expect(isAlive(pid)).toBe(true); + + // Long grace so we can cancel before it fires. hasExited stays false. + const timer = scheduleEscalatingKill(pid, () => false, 5000); + // The initial SIGTERM is trapped; child alive. Cancel before the grace. + await sleep(100); + clearTimeout(timer); + // Well past what the grace would have been — no SIGKILL arrives. + await sleep(300); + expect(isAlive(pid)).toBe(true); + + try { + process.kill(-pid, "SIGKILL"); + } catch { + /* best effort */ + } + await sleep(300); + expect(isAlive(pid)).toBe(false); + }, + ); +}); diff --git a/src/__tests__/fix-drift-straggler.test.ts b/src/__tests__/fix-drift-straggler.test.ts new file mode 100644 index 00000000..4bd78a49 --- /dev/null +++ b/src/__tests__/fix-drift-straggler.test.ts @@ -0,0 +1,122 @@ +/** + * FIX #F5 (round-4) — createPr's STRAGGLER fail-closed guard, in isolation. + * + * On a RESOLVED verdict every changed file MUST fall into a gated commit group + * (production builder, report-named fixture target, or skills/). A "straggler" + * (a changed file in none of those groups) means the predicate verdict and the + * staging partition have DIVERGED — staging would silently drop it and ship an + * incomplete fix behind a green verdict. createPr must exit UNSANCTIONED_CHANGE + * and stage NOTHING. + * + * By construction with the CURRENT predicate + gatedCommitFiles there is no file + * that both PASSES the predicate and becomes a straggler (the allowlist blocks + * everything gatedCommitFiles would not classify). This guard is therefore + * defense-in-depth against a FUTURE predicate change. To lock it load-bearingly + * we mock `evaluateDriftResolved` to force a RESOLVED verdict while git reports a + * straggler in the working tree, then assert createPr fail-closes before any + * `git add`. This is a pure-function/mock test — it never runs the autofix + * subprocess. + */ +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +import type { DriftReport } from "../../scripts/drift-types.js"; + +// Mock git so gitChangedFiles()/exec() are controllable, and force the predicate +// to RESOLVED so the straggler guard (which runs AFTER the verdict) is reached. +vi.mock("node:fs", async () => { + const actual = await vi.importActual("node:fs"); + return { ...actual, readFileSync: vi.fn(actual.readFileSync), writeFileSync: vi.fn() }; +}); + +vi.mock("node:child_process", async () => { + const actual = await vi.importActual("node:child_process"); + return { ...actual, execFileSync: vi.fn(), execSync: vi.fn() }; +}); + +vi.mock("../../scripts/drift-success-predicate.js", async () => { + const actual = await vi.importActual( + "../../scripts/drift-success-predicate.js", + ); + return { + ...actual, + // Force RESOLVED regardless of inputs so the guard downstream is exercised. + evaluateDriftResolved: vi.fn(() => ({ + resolved: true, + reason: actual.PredicateReason.RESOLVED, + detail: "forced resolved for the straggler-guard test", + offendingFiles: [], + })), + // gitChangedFiles is used by createPr — return a production file PLUS a + // straggler (a root file gatedCommitFiles cannot classify). + gitChangedFiles: vi.fn(() => ["src/helpers.ts", "weird-root-file.txt"]), + }; +}); + +import { execSync, execFileSync } from "node:child_process"; + +import { createPr } from "../../scripts/fix-drift.js"; + +const mockedExecSync = vi.mocked(execSync); +const mockedExecFileSync = vi.mocked(execFileSync); + +describe("createPr straggler guard fail-closed (fix #F5, isolated)", () => { + let logSpy: ReturnType; + let errSpy: ReturnType; + let exitSpy: ReturnType; + + beforeEach(() => { + vi.clearAllMocks(); + logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`__exit__${code}`); + }) as never); + // Default git calls return empty (branch lookups etc.). + mockedExecSync.mockReturnValue("fix/drift-2026-07-16\n" as unknown as string); + }); + + afterEach(() => { + logSpy.mockRestore(); + errSpy.mockRestore(); + exitSpy.mockRestore(); + }); + + const rep: DriftReport = { + timestamp: "2026-07-16T00:00:00.000Z", + entries: [ + { + provider: "OpenAI", + scenario: "chat completion", + builderFile: "src/helpers.ts", + builderFunctions: ["buildChatCompletion"], + typesFile: null, + sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", + diffs: [ + { + path: "x", + severity: "critical", + issue: "missing", + expected: "string", + real: "string", + mock: "", + }, + ], + }, + ], + }; + + it("exits UNSANCTIONED_CHANGE (17) and NEVER stages the straggler even on a RESOLVED verdict", () => { + expect(() => createPr(rep, { report: { timestamp: "t", entries: [] }, exitCode: 0 })).toThrow( + /__exit__17/, + ); + + const staged = mockedExecFileSync.mock.calls.some( + (c) => + c[0] === "git" && Array.isArray(c[1]) && (c[1] as string[]).includes("weird-root-file.txt"), + ); + expect(staged).toBe(false); + + const lines = logSpy.mock.calls.map((c) => String(c[0])); + expect(lines).toContain("reason=unsanctioned-change"); + }); +}); diff --git a/src/__tests__/fix-drift-workflow.test.ts b/src/__tests__/fix-drift-workflow.test.ts new file mode 100644 index 00000000..e48a5263 --- /dev/null +++ b/src/__tests__/fix-drift-workflow.test.ts @@ -0,0 +1,285 @@ +/** + * Static (text-level) assertions on .github/workflows/fix-drift.yml. + * + * These pin the LOAD-BEARING wiring that the drift-success predicate/guard now + * REQUIRE (CR round-3): + * + * F1 — the workflow must (a) re-collect drift AUTHORITATIVELY after the autofix + * to a distinct post-fix path, capturing its exit code, and (b) pass BOTH + * --post-fix-report and --post-fix-exit into the `--create-pr` invocation. + * Without these, the mandatory-post-fix guard fails closed and NO PR is + * ever opened (the gate would be inert). + * + * F-A — the PRE-fix report the allowlist's sanctioned-target set is derived + * from must be PINNED outside the LLM-writable repo checkout BEFORE the + * autofix runs, and both the Assert and Create-PR steps must read + * --report from that pinned copy — NEVER the in-repo drift-report.json + * the autofix LLM could overwrite. + * + * No YAML dependency is added; the repo ships none. These are deliberately + * text-shape assertions on the committed workflow — an actionlint run in CI + * covers structural validity separately. + */ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +import { describe, it, expect } from "vitest"; + +const WORKFLOW_PATH = resolve(__dirname, "../../.github/workflows/fix-drift.yml"); +const wf = readFileSync(WORKFLOW_PATH, "utf-8"); + +/** Collapse runs of whitespace so multi-line YAML `run:` blocks match linearly. */ +const wfFlat = wf.replace(/\s+/g, " "); + +describe("fix-drift.yml — F1: post-fix re-collect + args wired into --create-pr", () => { + it("has an authoritative post-fix re-collect step writing a DISTINCT report path OUTSIDE the repo (FIX #F3)", () => { + expect(wf).toContain("Re-collect drift (authoritative)"); + // FIX #F3 — the re-collect writes to $RUNNER_TEMP (via the POST_FIX_REPORT + // env), NOT the repo cwd, so it is never scored by the predicate's git scan. + expect(wf).toContain('npx tsx scripts/drift-report-collector.ts --out "$POST_FIX_REPORT"'); + expect(wf).toContain("POST_FIX_REPORT: ${{ runner.temp }}/drift-report.post-fix.json"); + }); + + it("captures the post-fix collector exit code as a step output", () => { + expect(wfFlat).toContain('POST_FIX_EXIT=$? set -e echo "post_fix_exit=$POST_FIX_EXIT"'); + }); + + it("passes BOTH --post-fix-report and --post-fix-exit into `fix-drift.ts --create-pr`", () => { + expect(wfFlat).toContain("npx tsx scripts/fix-drift.ts --create-pr"); + expect(wfFlat).toMatch( + /fix-drift\.ts --create-pr[^]*?--post-fix-report "\$\{POST_FIX_REPORT\}"[^]*?--post-fix-exit "\$\{POST_FIX_EXIT\}"/, + ); + }); + + it("the Assert step runs the predicate with post-fix args (the happy-path gate)", () => { + expect(wf).toContain("Assert drift truly resolved"); + expect(wfFlat).toMatch( + /drift-success-predicate\.ts[^]*?--post-fix-report "\$\{POST_FIX_REPORT\}"[^]*?--post-fix-exit "\$\{POST_FIX_EXIT\}"/, + ); + }); +}); + +describe("fix-drift.yml — F-A: PRE-fix report pinned outside the LLM-writable checkout", () => { + it("has a pin step that copies the pre-fix report into runner.temp before autofix", () => { + expect(wf).toContain("Pin pre-fix drift report (integrity)"); + // FIX #F3 — the pre-fix report is itself collected into $RUNNER_TEMP + // (PRE_FIX_REPORT), so the pin copies from that out-of-repo path, never the + // repo cwd. Both source and destination are outside the LLM-writable checkout. + expect(wf).toContain('cp "$PRE_FIX_REPORT" "$PINNED_REPORT"'); + expect(wf).toContain("PINNED_REPORT: ${{ runner.temp }}/drift-report.pinned.json"); + }); + + it("the pin step runs BEFORE the Auto-fix step (so the LLM cannot pre-tamper the pin)", () => { + const pinIdx = wf.indexOf("Pin pre-fix drift report"); + const autofixIdx = wf.indexOf("name: Auto-fix drift"); + expect(pinIdx).toBeGreaterThan(-1); + expect(autofixIdx).toBeGreaterThan(-1); + expect(pinIdx).toBeLessThan(autofixIdx); + }); + + it("the Assert step reads --report from the PINNED copy, not the in-repo file", () => { + // The YAML line-continuation `\` survives whitespace-flattening, so match + // tolerantly across it. + expect(wfFlat).toMatch(/drift-success-predicate\.ts \\? *--report "\$\{PINNED_REPORT\}"/); + }); + + it("the Create PR step reads --report from the PINNED copy, not the in-repo file", () => { + expect(wfFlat).toMatch( + /scripts\/fix-drift\.ts --create-pr \\? *--report "\$\{PINNED_REPORT\}"/, + ); + }); + + it("neither the Assert nor Create-PR predicate invocation reads --report drift-report.json (the LLM-writable file)", () => { + // The in-repo drift-report.json is still uploaded as an artifact + copied by + // the pin step, but must NEVER be the --report source for the gate. + expect(wfFlat).not.toMatch(/drift-success-predicate\.ts \\? *--report drift-report\.json/); + expect(wfFlat).not.toMatch(/fix-drift\.ts --create-pr \\? *--report drift-report\.json/); + }); +}); + +// --------------------------------------------------------------------------- +// report-path — the Auto-fix step must pass --report from the PINNED copy. +// +// FIX #F3 moved the collector's report to $RUNNER_TEMP (outside the repo +// checkout), so the repo-root drift-report.json that fix-drift.ts defaults to no +// longer exists. The Assert and Create-PR steps were updated to read --report +// from the pinned copy, but the Auto-fix step still ran `fix-drift.ts` with NO +// --report, so it fell back to the missing repo-root path and died with "Drift +// report not found" (readDriftReport) — the recurring drift-job failure. These +// lock that the Auto-fix step reads --report from the pinned copy, matching the +// downstream integrity gate. +// --------------------------------------------------------------------------- +describe("fix-drift.yml — report-path: Auto-fix step reads --report from the pinned copy", () => { + it("passes --report from the PINNED copy into the Auto-fix `fix-drift.ts` invocation", () => { + expect(wfFlat).toMatch(/scripts\/fix-drift\.ts --report "\$\{PINNED_REPORT\}"/); + }); + + it("does NOT run the Auto-fix step against the default (missing) repo-root drift-report.json", () => { + // The bare `npx tsx scripts/fix-drift.ts` (no --report) is the regression: + // it defaults to the repo-root drift-report.json which FIX #F3 no longer + // writes. Ensure the autofix invocation always carries a --report flag. + expect(wfFlat).not.toMatch(/npx tsx scripts\/fix-drift\.ts(?! --)/); + }); + + it("exposes PINNED_REPORT as an env in the Auto-fix step", () => { + const idx = wf.indexOf("name: Auto-fix drift"); + expect(idx).toBeGreaterThan(-1); + const nextStep = wf.indexOf("\n - name:", idx + 1); + const stepBlock = wf.slice(idx, nextStep === -1 ? undefined : nextStep); + expect(stepBlock).toContain("PINNED_REPORT: ${{ runner.temp }}/drift-report.pinned.json"); + }); +}); + +// --------------------------------------------------------------------------- +// FIX (round-4, user-approved) — HUMAN-APPROVAL BACKSTOP. The drift path opens a +// PR but must NEVER auto-merge: the predicate is a strong AUTO-FILTER, not a +// provable merge gate (the re-collect is not independent of the fix — WS-2b), so +// a human reviews CI + the diff + the verdict and merges. These lock that the +// unattended in-workflow merge is GONE and the Slack copy no longer claims +// "merged to main". +// --------------------------------------------------------------------------- +describe("fix-drift.yml — human-approval backstop: no unattended auto-merge", () => { + it("has NO auto-merge step and never runs `gh pr merge`", () => { + expect(wf).not.toContain("Auto-merge PR"); + expect(wf).not.toMatch(/gh pr merge/); + }); + + it("documents WHY the drift path is human-gated (predicate is a filter, not a merge gate)", () => { + expect(wf).toContain("NO AUTO-MERGE"); + // The rationale wraps across comment lines (a `#` marker survives flattening), + // so match tolerantly across the wrap. + expect(wfFlat).toMatch(/AUTO-FILTER, NOT a provable merge (# )?gate/i); + }); + + it("the success Slack message says the PR needs human review + merge, NOT merged to main", () => { + expect(wf).not.toContain("Drift auto-fix merged to main"); + expect(wf).toContain("Drift-fix PR opened — needs human review + merge"); + }); + + it("the fix-failure Slack step no longer references the removed merge step outputs", () => { + expect(wf).not.toContain("steps.merge.outputs"); + expect(wf).not.toContain("MERGE_REASON"); + }); +}); + +// --------------------------------------------------------------------------- +// WS-6 — end-of-job CATCH-ALL alert for the EARLY-infra window. The four +// specific alerts (collector-crash / autofix-step-fail / quarantine / +// fix-failure) are all gated on step outputs that only exist AFTER the +// collector ran. An EARLY failure (checkout / mint-app-token / pnpm install / +// clone ag-ui / git config) leaves those outputs empty, so without a catch-all +// the job dies red with ZERO Slack signal. These lock the catch-all's presence, +// its UNCONDITIONAL `if: failure()` gating, the anti-double-alert guard, and the +// infra-vs-drift-fix distinction. +// --------------------------------------------------------------------------- +describe("fix-drift.yml — WS-6: early-infra catch-all failure alert", () => { + it("has an end-of-job catch-all alert step", () => { + expect(wf).toContain("Alert on early-infra failure (catch-all)"); + }); + + it("the catch-all is gated on failure() and is UNCONDITIONAL on the earlier step OUTCOMES", () => { + // Isolate the catch-all step's `if:` expression. + const idx = wf.indexOf("Alert on early-infra failure (catch-all)"); + expect(idx).toBeGreaterThan(-1); + const stepBlock = wf.slice(idx, idx + 600); + const ifMatch = stepBlock.match(/if:\s*>-([\s\S]*?)\n\s{8}env:/); + expect(ifMatch).not.toBeNull(); + const ifExpr = (ifMatch?.[1] ?? "").replace(/\s+/g, " ").trim(); + + // MUST fire on any job failure. + expect(ifExpr).toContain("failure()"); + // MUST NOT gate on the autofix step OUTCOME (that would re-open the + // early-infra silence: autofix.outcome is empty pre-detect). + expect(ifExpr).not.toContain("steps.autofix.outcome"); + // Anti-double-alert guard: only fires when NONE of the specific alerts did + // (collector_crashed unset, quarantine unset, and check never ran so its + // skip output is empty). + expect(ifExpr).toContain("steps.detect.outputs.collector_crashed != 'true'"); + expect(ifExpr).toContain("steps.check.outputs.quarantine != 'true'"); + expect(ifExpr).toContain("steps.check.outputs.skip == ''"); + }); + + it("distinguishes an INFRA/SETUP failure from a drift-fix failure in its message", () => { + const idx = wf.indexOf("Alert on early-infra failure (catch-all)"); + const stepBlock = wf.slice(idx, idx + 1400); + expect(stepBlock).toMatch(/INFRA\/SETUP failure/); + // Missing-webhook must be a VISIBLE ::error:: + step failure, never silent. + expect(stepBlock).toContain("SLACK_WEBHOOK is not set"); + expect(stepBlock).toMatch(/::error::/); + }); + + it("the autofix-step-failure alert requires `check` to have RUN (skip == 'false'), so it never misfires on the early-infra window", () => { + // The autofix-failure alert must NOT fire before the collector ran — that + // window belongs to the catch-all. Gating on skip == 'false' (never empty + // once `check` executed) ensures the two are mutually exclusive. + const idx = wf.indexOf("Alert on autofix step failure"); + expect(idx).toBeGreaterThan(-1); + const stepBlock = wf.slice(idx, idx + 400); + expect(stepBlock).toContain("steps.check.outputs.skip == 'false'"); + }); +}); + +// --------------------------------------------------------------------------- +// WS-8 — the version-bump fail-closed reason must be NAMED in the failure +// alert, and the Create-PR step must surface the script's `reason=` on a +// non-zero exit so a fail-closed exit is not reported blank. +// --------------------------------------------------------------------------- +describe("fix-drift.yml — WS-8: version-bump-failed reason wiring", () => { + it("the fix-failure Slack alert names the version-bump-failed reason", () => { + expect(wf).toContain("version-bump-failed)"); + expect(wf).toMatch(/version-bump-failed\)\s+DETAIL=.*UNVERSIONED PR/); + }); + + it("the Create-PR step captures the script exit code + reason and surfaces it as a step output on failure", () => { + expect(wfFlat).toContain("PR_EXIT=${PIPESTATUS[0]}"); + expect(wfFlat).toContain("reason=${PR_REASON}"); + }); +}); + +// --------------------------------------------------------------------------- +// slot2-F3 — QUARANTINE must FAIL THE JOB (non-green), not just Slack-ping. +// A human watching CI status (not Slack) must see quarantine as a failure, like +// the collector-crash / autofix-failure alerts. Because quarantine sets +// check.outputs.skip == 'true', the fix-failure alert (needs skip != 'true') +// and the catch-all (needs skip == '') are both disjoint from it, so making the +// quarantine step exit 1 does NOT double-alert. +// --------------------------------------------------------------------------- +describe("fix-drift.yml — slot2-F3: quarantine fails the job (non-green)", () => { + it("the quarantine alert step exits non-zero on the happy (webhook-sent) path too", () => { + const idx = wf.indexOf("Alert on drift quarantine"); + expect(idx).toBeGreaterThan(-1); + // The step body runs until the next `- name:` step. + const nextStep = wf.indexOf("\n - name:", idx + 1); + const stepBlock = wf.slice(idx, nextStep === -1 ? undefined : nextStep); + // The curl (happy path) must be FOLLOWED by an `exit 1` — the step is not + // allowed to end green after sending the Slack ping. + const curlIdx = stepBlock.lastIndexOf("curl -fsS"); + expect(curlIdx).toBeGreaterThan(-1); + expect(stepBlock.slice(curlIdx)).toMatch(/\n\s*exit 1\b/); + }); + + it("does not overlap the fix-failure alert or the catch-all (both disjoint from quarantine's skip=='true')", () => { + // fix-failure requires skip != 'true'; catch-all requires skip == ''; + // quarantine sets skip == 'true' — so neither fires alongside it. + expect(wf).toContain("steps.check.outputs.skip != 'true'"); // fix-failure guard + const catchAllIdx = wf.indexOf("Alert on early-infra failure (catch-all)"); + expect(wf.slice(catchAllIdx, catchAllIdx + 600)).toContain("steps.check.outputs.skip == ''"); + }); +}); + +// --------------------------------------------------------------------------- +// slot2-F7/F12 — the fail-closed parse/git paths must be NAMED in the failure +// alert (post-fix-parse-error / git-push-failed), not blank. The code emits the +// reason; the workflow's case block must translate it to a human DETAIL. +// --------------------------------------------------------------------------- +describe("fix-drift.yml — slot2-F7/F12: fail-closed reasons are named in the alert", () => { + it("the fix-failure alert names the post-fix-parse-error reason", () => { + expect(wf).toContain("post-fix-parse-error)"); + expect(wfFlat).toMatch(/post-fix-parse-error\) DETAIL=.*Failed closed/); + }); + + it("the fix-failure alert names the git-push-failed reason", () => { + expect(wf).toContain("git-push-failed)"); + expect(wfFlat).toMatch(/git-push-failed\) DETAIL=.*no PR opened/); + }); +}); diff --git a/src/__tests__/fix-drift.test.ts b/src/__tests__/fix-drift.test.ts index 5cf3c157..24bee46c 100644 --- a/src/__tests__/fix-drift.test.ts +++ b/src/__tests__/fix-drift.test.ts @@ -1,4 +1,4 @@ -import { describe, it, expect, vi, beforeEach } from "vitest"; +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; import { resolve } from "node:path"; import type { @@ -39,10 +39,18 @@ import { readFileIfExists, execFileSafe, parseMode, + hasPostFixArgs, + parsePostFixExit, getChangedFiles, + gatedCommitFiles, + createPr, affectedSkillSections, BUILDER_TO_SKILL_SECTION, + truncateBody, + GH_BODY_MAX, + GH_BODY_SAFE_MAX, } from "../../scripts/fix-drift.js"; +import { sanctionedTargets } from "../../scripts/drift-success-predicate.js"; import { readFileSync, writeFileSync, existsSync } from "node:fs"; import { execFileSync, execSync } from "node:child_process"; @@ -603,6 +611,66 @@ describe("buildPrBody", () => { }); }); +// --------------------------------------------------------------------------- +// truncateBody +// --------------------------------------------------------------------------- + +describe("truncateBody", () => { + it("exports GH_BODY_MAX and GH_BODY_SAFE_MAX with sensible values", () => { + expect(GH_BODY_MAX).toBe(65536); + expect(GH_BODY_SAFE_MAX).toBeLessThan(GH_BODY_MAX); + }); + + it("passes through when under max", () => { + expect(truncateBody("hello", 60000)).toBe("hello"); + }); + + it("truncates and appends marker when over max", () => { + const out = truncateBody("a".repeat(70000)); + expect(out.length).toBeLessThanOrEqual(60000); + expect(out).toContain("Body truncated"); + }); + + it("never exceeds max when max is smaller than the marker", () => { + expect(truncateBody("x".repeat(100), 10).length).toBeLessThanOrEqual(10); + }); + + it("never exceeds the hard GH_BODY_MAX even when caller passes a larger max", () => { + expect(truncateBody("x".repeat(70000), 100000).length).toBeLessThanOrEqual(GH_BODY_MAX); + }); + + it("returns under-limit input unchanged", () => { + expect(truncateBody("hello", 10)).toBe("hello"); + }); + + it("appends marker and stays within effectiveMax when over-limit", () => { + const out = truncateBody("x".repeat(70000), 100000); + expect(out).toContain("Body truncated"); + expect(out.length).toBeLessThanOrEqual(GH_BODY_MAX); + }); +}); + +// --------------------------------------------------------------------------- +// buildPrBody — body length cap +// --------------------------------------------------------------------------- + +describe("buildPrBody — body length cap", () => { + it("caps the PR body under GitHub's 65536-char limit for huge reports", () => { + const entries = Array.from({ length: 400 }, (_, i) => + makeEntry({ scenario: "x".repeat(500) + String(i) }), + ); + const body = buildPrBody(makeReport({ entries })); + expect(body.length).toBeLessThanOrEqual(65536); + expect(body).toContain("## Summary"); + expect(body).toContain("Body truncated"); + }); + + it("leaves small bodies unchanged (no marker)", () => { + const body = buildPrBody(makeReport()); + expect(body).not.toContain("Body truncated"); + }); +}); + // --------------------------------------------------------------------------- // parsePorcelainLine // --------------------------------------------------------------------------- @@ -715,6 +783,64 @@ describe("parseMode", () => { }); }); +// --------------------------------------------------------------------------- +// FIX #5 — hasPostFixArgs: PR mode REQUIRES both post-fix flags. The old legacy +// no-post-fix fallback re-opened the fixture-only cheat (a test-file-only change +// satisfied it and opened a PR), so main() throws unless BOTH are present. A +// live subprocess red-green confirmed the OLD code proceeded to `gh pr create` +// with no post-fix args while the NEW code fails-closed BEFORE any git op. +// --------------------------------------------------------------------------- +describe("hasPostFixArgs (fix #5 legacy-fallback closure)", () => { + it("false when NO post-fix flags (the legacy cheat path — must be rejected)", () => { + expect(hasPostFixArgs(["--create-pr", "--report", "drift-report.json"])).toBe(false); + }); + + it("false when only --post-fix-report is present", () => { + expect(hasPostFixArgs(["--create-pr", "--post-fix-report", "post.json"])).toBe(false); + }); + + it("false when only --post-fix-exit is present", () => { + expect(hasPostFixArgs(["--create-pr", "--post-fix-exit", "0"])).toBe(false); + }); + + it("false when a flag is present but its value is missing (trailing flag)", () => { + expect(hasPostFixArgs(["--post-fix-report", "post.json", "--post-fix-exit"])).toBe(false); + }); + + it("true only when BOTH flags carry values", () => { + expect( + hasPostFixArgs(["--create-pr", "--post-fix-report", "post.json", "--post-fix-exit", "0"]), + ).toBe(true); + }); +}); + +// --------------------------------------------------------------------------- +// FIX #F7 (round-4) — parsePostFixExit: the PR path must fail CLOSED on an +// empty/whitespace --post-fix-exit rather than accept Number("")===0 as a clean +// collector exit 0. Mirrors the predicate CLI's guard so a missing recollect +// output never masquerades as clean and opens a PR on an unverified fix. +// --------------------------------------------------------------------------- +describe("parsePostFixExit (fix #F7 empty-exit fail-closed)", () => { + it("throws on an empty string (Number('')===0 must NOT slip through as clean)", () => { + expect(() => parsePostFixExit("")).toThrow(/empty\/whitespace/); + }); + + it("throws on a whitespace-only value", () => { + expect(() => parsePostFixExit(" ")).toThrow(/empty\/whitespace/); + }); + + it("throws on a non-integer value", () => { + expect(() => parsePostFixExit("abc")).toThrow(/integer/); + expect(() => parsePostFixExit("1.5")).toThrow(/integer/); + }); + + it("returns the integer for a valid value", () => { + expect(parsePostFixExit("0")).toBe(0); + expect(parsePostFixExit("2")).toBe(2); + expect(parsePostFixExit("-1")).toBe(-1); + }); +}); + // --------------------------------------------------------------------------- // getChangedFiles // --------------------------------------------------------------------------- @@ -751,6 +877,328 @@ describe("getChangedFiles", () => { // affectedSkillSections // --------------------------------------------------------------------------- +// --------------------------------------------------------------------------- +// CR round-3 F-C / F2 — gatedCommitFiles: createPr stages ONLY the allowlisted +// set (production source + report-named fixture targets), NEVER a straggler +// catch-all. A file the predicate would have blocked (config/manifest/unnamed +// fixture) must land in `stragglers`, which createPr never `git add`s. +// --------------------------------------------------------------------------- + +describe("gatedCommitFiles (F-C: no straggler catch-all)", () => { + const sanctioned = new Set([ + "src/helpers.ts", + "src/__tests__/drift/model-registry.ts", + "src/types.ts", + ]); + + it("groups production source into builderFiles", () => { + const g = gatedCommitFiles(["src/helpers.ts", "src/types.ts"], sanctioned); + expect(g.builderFiles).toEqual(["src/helpers.ts", "src/types.ts"]); + expect(g.stragglers).toEqual([]); + }); + + it("stages a report-named fixture target as a testFile", () => { + const g = gatedCommitFiles(["src/__tests__/drift/model-registry.ts"], sanctioned); + expect(g.testFiles).toEqual(["src/__tests__/drift/model-registry.ts"]); + expect(g.stragglers).toEqual([]); + }); + + it("EXCLUDES an UNSANCTIONED src/__tests__ file — it lands in stragglers, never testFiles", () => { + const g = gatedCommitFiles(["src/__tests__/drift/sdk-shapes.ts"], sanctioned); + expect(g.testFiles).toEqual([]); + expect(g.stragglers).toEqual(["src/__tests__/drift/sdk-shapes.ts"]); + }); + + it("puts package.json / lockfiles / config in stragglers (never staged)", () => { + const g = gatedCommitFiles( + ["src/helpers.ts", "package.json", "pnpm-lock.yaml", "tsconfig.json"], + sanctioned, + ); + expect(g.builderFiles).toEqual(["src/helpers.ts"]); + expect(g.stragglers).toEqual(["package.json", "pnpm-lock.yaml", "tsconfig.json"]); + }); + + it("stragglers is empty for a clean allowlisted set (the RESOLVED-verdict invariant)", () => { + const g = gatedCommitFiles( + ["src/helpers.ts", "src/__tests__/drift/model-registry.ts"], + sanctioned, + ); + expect(g.stragglers).toEqual([]); + }); +}); + +// --------------------------------------------------------------------------- +// FIX #F5 (round-4) — createPr MUST fail-closed if any changed file falls +// outside every gated commit group (a straggler): staging it as part of an +// allowlisted group would silently drop or mis-stage it. Because the predicate +// allowlist already blocks any non-production, non-report-named file BEFORE +// createPr stages, an unclassified working-tree file causes createPr to exit +// with UNSANCTIONED_CHANGE and stage NOTHING — the straggler is never `git add`ed. +// +// Driven against the REAL createPr with git mocked (no autofix subprocess). +// --------------------------------------------------------------------------- +describe("createPr straggler / unsanctioned fail-closed (fix #F5)", () => { + const mockedExecSync = vi.mocked(execSync); + let logSpy: ReturnType | null = null; + let errSpy: ReturnType | null = null; + let exitSpy: ReturnType | null = null; + + beforeEach(() => { + vi.clearAllMocks(); + logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`__exit__${code}`); + }) as never); + void errSpy; + void exitSpy; + }); + + afterEach(() => { + logSpy?.mockRestore(); + errSpy?.mockRestore(); + exitSpy?.mockRestore(); + }); + + function stdoutLines(): string[] { + return (logSpy?.mock.calls ?? []).map((c) => String(c[0])); + } + + const rep: DriftReport = { + timestamp: "2026-07-16T00:00:00.000Z", + entries: [ + { + provider: "OpenAI", + scenario: "chat completion", + builderFile: "src/helpers.ts", + builderFunctions: ["buildChatCompletion"], + typesFile: null, + sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", + diffs: [ + { + path: "x", + severity: "critical", + issue: "missing", + expected: "string", + real: "string", + mock: "", + }, + ], + }, + ], + }; + + it("exits UNSANCTIONED_CHANGE (17) and stages NOTHING when an unclassified file is in the tree", () => { + // A production fix (helpers.ts, allowlisted) PLUS an unclassified root file. + // createPr fail-closes (the root file is neither allowlisted nor a gated + // group) BEFORE any staging — the straggler is never git-added. + mockedExecSync.mockImplementation((cmd: unknown) => { + if (typeof cmd === "string" && cmd.includes("status --porcelain")) { + return "M src/helpers.ts\n?? weird-root-file.txt\n" as unknown as string; + } + return "" as unknown as string; + }); + + expect(() => createPr(rep, { report: { timestamp: "t", entries: [] }, exitCode: 0 })).toThrow( + /__exit__17/, + ); + + const addedStraggler = mockedExecFileSync.mock.calls.some( + (c) => + c[0] === "git" && Array.isArray(c[1]) && (c[1] as string[]).includes("weird-root-file.txt"), + ); + expect(addedStraggler).toBe(false); + expect(stdoutLines()).toContain("reason=unsanctioned-change"); + }); + + it("gatedCommitFiles never leaves a production or report-named file as a straggler (the invariant createPr asserts)", () => { + // The straggler guard in createPr is defense-in-depth: prove the partition + // it relies on classifies every allowlisted file into a gated group so + // stragglers is empty on any RESOLVED-shaped set. + const s = sanctionedTargets(rep); + const g = gatedCommitFiles(["src/helpers.ts", "src/foo.ts"], s); + expect(g.stragglers).toEqual([]); + expect(g.builderFiles).toEqual(["src/helpers.ts", "src/foo.ts"]); + }); +}); + +// --------------------------------------------------------------------------- +// CR round-3 F-A — the sanctioned/allowlist set is derived SOLELY from the +// (pinned) report object passed to createPr, NOT from any on-disk file. The +// workflow pins the pre-fix report OUTSIDE the LLM-writable repo checkout and +// passes THAT copy via --report; here we prove that a DIFFERENT on-disk report +// cannot expand the allowlist, because the set is a pure function of the passed +// report — the file the LLM could overwrite has no bearing on the sanctioned set. +// --------------------------------------------------------------------------- + +describe("F-A: sanctioned set comes from the passed (pinned) report, not on-disk", () => { + it("sanctionedTargets is a pure function of the report — a forged on-disk file cannot widen it", () => { + // The PINNED report names only the production builder as a target. + const pinned = makeReport({ + entries: [makeEntry({ builderFile: "src/helpers.ts", typesFile: null })], + }); + // A FORGED report (what an autofix LLM might write to drift-report.json in the + // repo) tries to sanction the SDK-shape fixture as a target. + const forged = makeReport({ + entries: [makeEntry({ builderFile: "src/__tests__/drift/sdk-shapes.ts", typesFile: null })], + }); + + const pinnedSet = sanctionedTargets(pinned); + const forgedSet = sanctionedTargets(forged); + + // The sanctioned set is derived purely from the object passed in. + expect(pinnedSet.has("src/helpers.ts")).toBe(true); + expect(pinnedSet.has("src/__tests__/drift/sdk-shapes.ts")).toBe(false); + // The forged set (only relevant if createPr were fed the LLM-writable file) + // would sanction the fixture — which is EXACTLY why the workflow must pin the + // pre-fix report and pass THAT copy, never the in-repo drift-report.json. + expect(forgedSet.has("src/__tests__/drift/sdk-shapes.ts")).toBe(true); + // The two sets are disjoint on the fixture: pinning the report is what keeps + // the fixture OUT of the allowlist. + expect(pinnedSet.has("src/__tests__/drift/sdk-shapes.ts")).not.toBe( + forgedSet.has("src/__tests__/drift/sdk-shapes.ts"), + ); + }); +}); + +// --------------------------------------------------------------------------- +// WS-8 — version-bump failure must FAIL CLOSED, not warn-and-continue. +// +// The original createPr wrapped patchBumpVersion()+changelog+version-commit in +// a try/catch that did `console.warn("Version bump failed, skipping")` and +// CONTINUED to push + open the PR — shipping an UNVERSIONED "fix" that a human +// might merge but which never publishes a release (silent value loss). The fix +// makes a bump failure a HARD, fail-closed error: exit VERSION_BUMP_FAILED (18) +// with a named reason and NO push / NO PR. +// +// Driven against the REAL createPr with git mocked so the version-bump commit +// throws (no autofix subprocess). +// --------------------------------------------------------------------------- +describe("createPr version-bump fail-closed (WS-8)", () => { + const mockedExecSync = vi.mocked(execSync); + let logSpy: ReturnType | null = null; + let errSpy: ReturnType | null = null; + let warnSpy: ReturnType | null = null; + let exitSpy: ReturnType | null = null; + + beforeEach(() => { + vi.clearAllMocks(); + logSpy = vi.spyOn(console, "log").mockImplementation(() => {}); + errSpy = vi.spyOn(console, "error").mockImplementation(() => {}); + warnSpy = vi.spyOn(console, "warn").mockImplementation(() => {}); + exitSpy = vi.spyOn(process, "exit").mockImplementation(((code?: number) => { + throw new Error(`__exit__${code}`); + }) as never); + void errSpy; + void exitSpy; + }); + + afterEach(() => { + logSpy?.mockRestore(); + errSpy?.mockRestore(); + warnSpy?.mockRestore(); + exitSpy?.mockRestore(); + }); + + function stdoutLines(): string[] { + return (logSpy?.mock.calls ?? []).map((c) => String(c[0])); + } + + // A clean RESOLVED-shaped report: ONE production builder changed (helpers.ts, + // sanctioned), no stragglers — so createPr proceeds PAST the straggler guard + // into the version-bump step. + const rep: DriftReport = { + timestamp: "2026-07-16T00:00:00.000Z", + entries: [ + { + provider: "OpenAI", + scenario: "chat completion", + builderFile: "src/helpers.ts", + builderFunctions: ["buildChatCompletion"], + typesFile: null, + sdkShapesFile: "src/__tests__/drift/sdk-shapes.ts", + diffs: [ + { + path: "x", + severity: "critical", + issue: "missing", + expected: "string", + real: "string", + mock: "", + }, + ], + }, + ], + }; + + function onlyHelpersChanged(cmd: unknown): string { + if (typeof cmd === "string" && cmd.includes("status --porcelain")) { + return "M src/helpers.ts\n" as unknown as string; + } + return "" as unknown as string; + } + + it("exits VERSION_BUMP_FAILED (18) with a named reason and opens NO PR when the version-bump commit throws", () => { + mockedExecSync.mockImplementation(onlyHelpersChanged as never); + // Make the version-bump git commit throw (mock git so the bump step fails). + mockedExecFileSync.mockImplementation((file: unknown, args: unknown) => { + if ( + file === "git" && + Array.isArray(args) && + (args as string[]).includes("commit") && + (args as string[]).some((a) => typeof a === "string" && a.includes("bump version")) + ) { + throw new Error("simulated git failure during version-bump commit"); + } + return Buffer.from("") as unknown as void; + }); + + expect(() => createPr(rep, { report: { timestamp: "t", entries: [] }, exitCode: 0 })).toThrow( + /__exit__18/, + ); + + // Named, fail-closed reason emitted for the workflow's failure alert. + expect(stdoutLines()).toContain("reason=version-bump-failed"); + + // FAIL CLOSED: it must NOT warn-and-continue, and must NEVER push or open a PR. + expect(warnSpy?.mock.calls ?? []).toEqual([]); + const pushed = mockedExecFileSync.mock.calls.some( + (c) => c[0] === "git" && Array.isArray(c[1]) && (c[1] as string[]).includes("push"), + ); + const openedPr = mockedExecFileSync.mock.calls.some( + (c) => c[0] === "gh" && Array.isArray(c[1]) && (c[1] as string[]).includes("pr"), + ); + expect(pushed).toBe(false); + expect(openedPr).toBe(false); + }); + + it("exits GIT_PUSH_FAILED (20) with a named reason and opens NO PR when `git push` throws", () => { + // slot2-F12 lock: a git push failure after the local commits must fail + // CLOSED (no PR) AND emit a NAMED reason — historically it reached the + // top-level catch as a blank-reason exit 3. + mockedExecSync.mockImplementation(onlyHelpersChanged as never); + mockedExecFileSync.mockImplementation((file: unknown, args: unknown) => { + if (file === "git" && Array.isArray(args) && (args as string[]).includes("push")) { + throw new Error("simulated git push failure (auth/network)"); + } + return Buffer.from("") as unknown as void; + }); + + expect(() => createPr(rep, { report: { timestamp: "t", entries: [] }, exitCode: 0 })).toThrow( + /__exit__20/, + ); + + // Named, fail-closed reason (not blank) for the workflow's failure alert. + expect(stdoutLines()).toContain("reason=git-push-failed"); + + // No PR opened — push failed before `gh pr create`. + const openedPr = mockedExecFileSync.mock.calls.some( + (c) => c[0] === "gh" && Array.isArray(c[1]) && (c[1] as string[]).includes("pr"), + ); + expect(openedPr).toBe(false); + }); +}); + describe("affectedSkillSections", () => { it("returns empty array when no builder files are present", () => { expect(affectedSkillSections(["src/__tests__/foo.test.ts", "package.json"])).toEqual([]); @@ -841,15 +1289,3 @@ describe("buildPrBody — skill sections", () => { expect(body).toContain("- Responses API"); }); }); - -// --------------------------------------------------------------------------- -// buildPrompt — skill file reference -// --------------------------------------------------------------------------- - -describe("buildPrompt — skill file", () => { - it("includes skill file update instructions", () => { - const prompt = buildPrompt(makeReport()); - expect(prompt).toContain("## Skill file update"); - expect(prompt).toContain("skills/write-fixtures/SKILL.md"); - }); -}); diff --git a/src/__tests__/fixture-loader.test.ts b/src/__tests__/fixture-loader.test.ts index ca4719c1..1617eda2 100644 --- a/src/__tests__/fixture-loader.test.ts +++ b/src/__tests__/fixture-loader.test.ts @@ -316,6 +316,32 @@ describe("loadFixtureFile", () => { expect(fixtures[0].disconnectAfterMs).toBeUndefined(); }); + it("loads sibling fixtures when one entry has a non-array interChunkDelaysMs", () => { + // Untrusted fixture JSON: interChunkDelaysMs is typed number[] but may be + // null/missing/non-array. Without an Array.isArray guard, .filter throws a + // TypeError inside entryToFixture, aborting the entire file's load and + // silently dropping every fixture in it. + const filePath = writeJson(tmpDir, "bad-timings.json", { + fixtures: [ + { + match: { userMessage: "malformed" }, + response: { content: "bad" }, + recordedTimings: { ttftMs: 0, interChunkDelaysMs: null, totalDurationMs: 0 }, + }, + { + match: { userMessage: "valid" }, + response: { content: "good" }, + }, + ], + }); + + const fixtures = loadFixtureFile(filePath); + expect(fixtures).toHaveLength(2); + expect(fixtures[1].match.userMessage).toBe("valid"); + // The malformed entry is sanitized: non-array interChunkDelaysMs -> []. + expect(fixtures[0].recordedTimings?.interChunkDelaysMs).toEqual([]); + }); + it("warns and returns empty array for invalid JSON", () => { const filePath = join(tmpDir, "bad.json"); writeFileSync(filePath, "{ not valid json", "utf-8"); @@ -1233,6 +1259,34 @@ describe("validateFixtures", () => { expect(results.filter((r) => r.message.includes("hasToolResult"))).toHaveLength(0); }); + // --- match.toolResultContains type checks --- + + it("error: toolResultContains is a number", () => { + const fixtures = [ + makeFixture({ match: { userMessage: "test", toolResultContains: 42 as never } }), + ]; + const results = validateFixtures(fixtures); + expect( + results.some((r) => r.severity === "error" && r.message.includes("toolResultContains")), + ).toBe(true); + }); + + it("error: toolResultContains is an empty string", () => { + const fixtures = [makeFixture({ match: { userMessage: "test", toolResultContains: "" } })]; + const results = validateFixtures(fixtures); + expect( + results.some((r) => r.severity === "error" && r.message.includes("toolResultContains")), + ).toBe(true); + }); + + it("no error: toolResultContains is a non-empty string", () => { + const fixtures = [ + makeFixture({ match: { userMessage: "test", toolResultContains: "cancelled" } }), + ]; + const results = validateFixtures(fixtures); + expect(results.filter((r) => r.message.includes("toolResultContains"))).toHaveLength(0); + }); + // --- match.systemMessage type checks --- it("error: systemMessage is a number", () => { @@ -1348,6 +1402,18 @@ describe("validateFixtures", () => { expect(duplicateWarnings).toHaveLength(0); }); + it("no warning: same userMessage but different toolResultContains", () => { + const fixtures = [ + makeFixture({ match: { userMessage: "hello", toolResultContains: "cancelled" } }), + makeFixture({ match: { userMessage: "hello", toolResultContains: "chosen_" } }), + ]; + const results = validateFixtures(fixtures); + const duplicateWarnings = results.filter( + (r) => r.severity === "warning" && r.message.includes("duplicate"), + ); + expect(duplicateWarnings).toHaveLength(0); + }); + it("no warning: same userMessage but different sequenceIndex", () => { const fixtures = [ makeFixture({ match: { userMessage: "hello", sequenceIndex: 0 } }), @@ -1360,6 +1426,90 @@ describe("validateFixtures", () => { expect(duplicateWarnings).toHaveLength(0); }); + it("no warning: same userMessage but different toolCallId", () => { + const fixtures = [ + makeFixture({ match: { userMessage: "hello", toolCallId: "call_a" } }), + makeFixture({ match: { userMessage: "hello", toolCallId: "call_b" } }), + ]; + const results = validateFixtures(fixtures); + const duplicateWarnings = results.filter( + (r) => r.severity === "warning" && r.message.includes("duplicate"), + ); + expect(duplicateWarnings).toHaveLength(0); + }); + + it("no warning: same userMessage but different systemMessage", () => { + const fixtures = [ + makeFixture({ match: { userMessage: "hello", systemMessage: "persona A" } }), + makeFixture({ match: { userMessage: "hello", systemMessage: "persona B" } }), + ]; + const results = validateFixtures(fixtures); + const duplicateWarnings = results.filter( + (r) => r.severity === "warning" && r.message.includes("duplicate"), + ); + expect(duplicateWarnings).toHaveLength(0); + }); + + it("no warning: same userMessage but different model", () => { + const fixtures = [ + makeFixture({ match: { userMessage: "hello", model: "gpt-4o" } }), + makeFixture({ match: { userMessage: "hello", model: "claude-opus-4" } }), + ]; + const results = validateFixtures(fixtures); + const duplicateWarnings = results.filter( + (r) => r.severity === "warning" && r.message.includes("duplicate"), + ); + expect(duplicateWarnings).toHaveLength(0); + }); + + it("no warning: same userMessage but different toolName", () => { + const fixtures = [ + makeFixture({ match: { userMessage: "hello", toolName: "search" } }), + makeFixture({ match: { userMessage: "hello", toolName: "fetch" } }), + ]; + const results = validateFixtures(fixtures); + const duplicateWarnings = results.filter( + (r) => r.severity === "warning" && r.message.includes("duplicate"), + ); + expect(duplicateWarnings).toHaveLength(0); + }); + + it("no warning: same userMessage but different responseFormat", () => { + const fixtures = [ + makeFixture({ match: { userMessage: "hello", responseFormat: "text" } }), + makeFixture({ match: { userMessage: "hello", responseFormat: "json_object" } }), + ]; + const results = validateFixtures(fixtures); + const duplicateWarnings = results.filter( + (r) => r.severity === "warning" && r.message.includes("duplicate"), + ); + expect(duplicateWarnings).toHaveLength(0); + }); + + it("no warning: same userMessage but different endpoint", () => { + const fixtures = [ + makeFixture({ match: { userMessage: "hello", endpoint: "chat" } }), + makeFixture({ match: { userMessage: "hello", endpoint: "video" } }), + ]; + const results = validateFixtures(fixtures); + const duplicateWarnings = results.filter( + (r) => r.severity === "warning" && r.message.includes("duplicate"), + ); + expect(duplicateWarnings).toHaveLength(0); + }); + + it("no warning: same userMessage but different inputText", () => { + const fixtures = [ + makeFixture({ match: { userMessage: "hello", inputText: "embed A" } }), + makeFixture({ match: { userMessage: "hello", inputText: "embed B" } }), + ]; + const results = validateFixtures(fixtures); + const duplicateWarnings = results.filter( + (r) => r.severity === "warning" && r.message.includes("duplicate"), + ); + expect(duplicateWarnings).toHaveLength(0); + }); + it("warning: same userMessage with identical turnIndex/hasToolResult/sequenceIndex", () => { const fixtures = [ makeFixture({ match: { userMessage: "hello", turnIndex: 1, hasToolResult: true } }), @@ -1868,6 +2018,15 @@ describe("auto-stringify JSON objects in fixture entries", () => { expect(fixture.match.systemMessage).toBe("name=Atai"); }); + it("passes toolResultContains through entryToFixture", () => { + const entry: FixtureFileEntry = { + match: { toolCallId: "call_x", toolResultContains: '"cancelled"' }, + response: { content: "ok" }, + }; + const fixture = entryToFixture(entry); + expect(fixture.match.toolResultContains).toBe('"cancelled"'); + }); + it("stringifies nested objects in arguments", () => { const entry: FixtureFileEntry = { match: { userMessage: "test" }, diff --git a/src/__tests__/journal.test.ts b/src/__tests__/journal.test.ts index 4209aa17..6a92b163 100644 --- a/src/__tests__/journal.test.ts +++ b/src/__tests__/journal.test.ts @@ -457,4 +457,99 @@ describe("Journal", () => { expect(journal.getFixtureMatchCount(fixture, "t-9999")).toBe(1); }); }); + + describe("journal body cap (ISL regression)", () => { + const BODY_CAP = 64 * 1024; // 64 KB — must match journal.ts constant + + it("replaces oversized bodies with a truncation marker", () => { + const journal = new Journal(); + // Build a body that serializes to well over 64 KB (100 KB of content) + const bigContent = "A".repeat(100 * 1024); + const entry = journal.add( + makeEntry({ + body: { + model: "gpt-4o", + messages: [{ role: "user", content: bigContent }], + }, + }), + ); + + // The stored body must be the truncation marker, not the original + const body = entry.body as Record; + expect(body.__aimock_truncated).toBe(true); + expect(typeof body.originalByteSize).toBe("number"); + expect(body.originalByteSize as number).toBeGreaterThan(BODY_CAP); + expect(typeof body.note).toBe("string"); + }); + + it("retains bodies that are within the 64 KB cap", () => { + const journal = new Journal(); + const smallBody = { + model: "gpt-4o", + messages: [{ role: "user" as const, content: "hello" }], + }; + const entry = journal.add(makeEntry({ body: smallBody })); + + // Small body must be stored as-is (no truncation marker) + expect((entry.body as Record).__aimock_truncated).toBeUndefined(); + expect(entry.body).toEqual(smallBody); + }); + + it("truncates multibyte (CJK) bodies whose UTF-8 byte size exceeds 64 KB even when code-unit length does not", () => { + // Regression: the original gate used serialized.length (UTF-16 code units). + // CJK characters are 3 bytes in UTF-8 but 1 JS code unit, so a body whose + // byte size > 64 KB can have code-unit length << 64 K. + // This test proves the byte-based gate catches what the code-unit gate missed. + // + // 22,000 CJK chars × 3 UTF-8 bytes = 66,000 bytes, plus ~60 bytes of JSON + // envelope → ~66,060 bytes total (> 65,536 cap). + // Code-unit length: 22,000 + ~60 = ~22,060 (well under 65,536). + const journal = new Journal(); + // U+4E2D (中) is a 3-byte UTF-8 character; repeat 22,000 times. + const cjkContent = "中".repeat(22_000); + const body = { + model: "gpt-4o", + messages: [{ role: "user" as const, content: cjkContent }], + }; + // Verify our test invariant: byte size > cap, code-unit length < cap. + const serialized = JSON.stringify(body); + expect(serialized.length).toBeLessThan(BODY_CAP); // code-unit gate would MISS this + expect(Buffer.byteLength(serialized, "utf8")).toBeGreaterThan(BODY_CAP); // byte gate catches it + + const entry = journal.add(makeEntry({ body })); + + // With a byte-based gate, the body MUST be truncated. + const stored = entry.body as Record; + expect(stored.__aimock_truncated).toBe(true); + expect(stored.originalByteSize as number).toBeGreaterThan(BODY_CAP); + }); + + it("JSON.stringify(journal.getAll()) does not throw with 1000 large-body entries", () => { + // Regression anchor for RangeError: Invalid string length (ISL). + // 1000 entries × 100 KB body each = 100 MB without cap → exceeds V8 limit. + // With cap, each entry stores only the ~200-byte marker → <1 MB total. + const journal = new Journal({ maxEntries: 1000 }); + const bigContent = "B".repeat(100 * 1024); + for (let i = 0; i < 1000; i++) { + journal.add( + makeEntry({ + body: { + model: "gpt-4o", + messages: [{ role: "user", content: bigContent }], + }, + }), + ); + } + + expect(journal.size).toBe(1000); + // This must not throw RangeError: Invalid string length + let serialized: string; + expect(() => { + serialized = JSON.stringify(journal.getAll()); + }).not.toThrow(); + // Sanity-check: valid JSON, 1000 entries + const parsed = JSON.parse(serialized!) as unknown[]; + expect(parsed).toHaveLength(1000); + }); + }); }); diff --git a/src/__tests__/provider-auth.test.ts b/src/__tests__/provider-auth.test.ts new file mode 100644 index 00000000..b93755ca --- /dev/null +++ b/src/__tests__/provider-auth.test.ts @@ -0,0 +1,435 @@ +import { describe, it, expect, afterEach, beforeEach } from "vitest"; +import * as http from "node:http"; +import type { RecordProviderKey } from "../types.js"; +import { createServer, type ServerInstance } from "../server.js"; + +// --------------------------------------------------------------------------- +// This suite exercises the REAL forwarded-header surface: a fake upstream HTTP +// server records the auth headers aimock forwards to it on a fixture-miss +// passthrough (proxy-only). We assert on what the fake upstream actually saw — +// not on a mock of aimock's internals. +// --------------------------------------------------------------------------- + +function post( + url: string, + body: unknown, + headers?: Record, +): Promise<{ status: number; body: string }> { + return new Promise((resolve, reject) => { + const data = JSON.stringify(body); + const parsed = new URL(url); + const req = http.request( + { + hostname: parsed.hostname, + port: parsed.port, + path: parsed.pathname, + method: "POST", + headers: { + "Content-Type": "application/json", + "Content-Length": Buffer.byteLength(data), + ...headers, + }, + }, + (res) => { + const chunks: Buffer[] = []; + res.on("data", (c: Buffer) => chunks.push(c)); + res.on("end", () => + resolve({ status: res.statusCode ?? 0, body: Buffer.concat(chunks).toString() }), + ); + }, + ); + req.on("error", reject); + req.write(data); + req.end(); + }); +} + +interface FakeUpstream { + server: http.Server; + url: string; + /** Auth-relevant headers observed on the most recent request. */ + last: () => http.IncomingHttpHeaders; + requestCount: () => number; +} + +/** + * A fake upstream that records the headers it receives and returns a minimal + * OpenAI-shaped chat completion so aimock's collapse/record path is happy. + */ +function createFakeUpstream(): Promise { + return new Promise((resolve) => { + let seen: http.IncomingHttpHeaders = {}; + let count = 0; + const server = http.createServer((req, res) => { + count++; + seen = req.headers; + // Drain the request body before responding. + req.on("data", () => {}); + req.on("end", () => { + res.writeHead(200, { "Content-Type": "application/json" }); + res.end( + JSON.stringify({ + id: "chatcmpl-fake", + object: "chat.completion", + created: Date.now(), + model: "gpt-4", + choices: [ + { + index: 0, + message: { role: "assistant", content: "hi" }, + finish_reason: "stop", + }, + ], + usage: { prompt_tokens: 1, completion_tokens: 1, total_tokens: 2 }, + }), + ); + }); + }); + server.listen(0, "127.0.0.1", () => { + const addr = server.address(); + const port = typeof addr === "object" && addr ? addr.port : 0; + resolve({ + server, + url: `http://127.0.0.1:${port}`, + last: () => seen, + requestCount: () => count, + }); + }); + }); +} + +const DUMMY = "sk-aimock-dev-ci-only"; +const REAL = "sk-real-test-123"; + +let upstream: FakeUpstream | undefined; +let recorder: ServerInstance | undefined; + +afterEach(async () => { + if (recorder) { + await new Promise((r) => recorder!.server.close(() => r())); + recorder = undefined; + } + if (upstream) { + await new Promise((r) => upstream!.server.close(() => r())); + upstream = undefined; + } +}); + +/** + * Stand up the fake upstream + an aimock proxy-only recorder pointed at it for a + * single provider, optionally with a built-in key configured for that provider. + */ +async function setup( + provider: RecordProviderKey, + providerKey?: string, +): Promise<{ recorderUrl: string }> { + upstream = await createFakeUpstream(); + recorder = await createServer([], { + port: 0, + record: { + providers: { [provider]: upstream.url }, + providerKeys: providerKey ? { [provider]: providerKey } : undefined, + proxyOnly: true, + }, + }); + return { recorderUrl: recorder.url }; +} + +const CHAT_PATH = "/v1/chat/completions"; +const CHAT_BODY = { model: "gpt-4", messages: [{ role: "user", content: "unmatched-miss" }] }; + +describe("applyProviderAuth — aimock owns the upstream key", () => { + describe("OpenAI (bearer)", () => { + it("RED baseline: with NO built-in key, a dummy caller key is forwarded verbatim", async () => { + const { recorderUrl } = await setup("openai"); // feature OFF + const res = await post(recorderUrl + CHAT_PATH, CHAT_BODY, { + Authorization: `Bearer ${DUMMY}`, + }); + expect(res.status).toBe(200); + // Today's behavior (and case (a)): dummy forwarded unchanged. + expect(upstream!.last().authorization).toBe(`Bearer ${DUMMY}`); + }); + + it("GREEN: built-in key set + caller sends dummy → aimock injects its own key", async () => { + const { recorderUrl } = await setup("openai", REAL); + const res = await post(recorderUrl + CHAT_PATH, CHAT_BODY, { + Authorization: `Bearer ${DUMMY}`, + }); + expect(res.status).toBe(200); + expect(upstream!.last().authorization).toBe(`Bearer ${REAL}`); + }); + + it("case (b): built-in key set + caller sends a REAL key → caller overrides", async () => { + const callerReal = "sk-caller-owns-this-999"; + const { recorderUrl } = await setup("openai", REAL); + await post(recorderUrl + CHAT_PATH, CHAT_BODY, { + Authorization: `Bearer ${callerReal}`, + }); + expect(upstream!.last().authorization).toBe(`Bearer ${callerReal}`); + }); + + it("no caller credential + built-in key set → aimock injects its own key", async () => { + const { recorderUrl } = await setup("openai", REAL); + await post(recorderUrl + CHAT_PATH, CHAT_BODY); + expect(upstream!.last().authorization).toBe(`Bearer ${REAL}`); + }); + }); + + describe("Anthropic (x-api-key)", () => { + it("built-in key set + caller sends dummy → injects x-api-key", async () => { + const { recorderUrl } = await setup("anthropic", REAL); + await post(recorderUrl + "/v1/messages", CHAT_BODY, { "x-api-key": DUMMY }); + expect(upstream!.last()["x-api-key"]).toBe(REAL); + }); + + it("caller sends a real x-api-key → forwarded unchanged", async () => { + const { recorderUrl } = await setup("anthropic", REAL); + await post(recorderUrl + "/v1/messages", CHAT_BODY, { "x-api-key": "real-anthropic-key" }); + expect(upstream!.last()["x-api-key"]).toBe("real-anthropic-key"); + }); + }); + + describe("Gemini (x-goog-api-key)", () => { + it("built-in key set + caller sends dummy → injects x-goog-api-key", async () => { + const { recorderUrl } = await setup("gemini", REAL); + await post(recorderUrl + "/v1beta/models/gemini-pro:generateContent", CHAT_BODY, { + "x-goog-api-key": DUMMY, + }); + expect(upstream!.last()["x-goog-api-key"]).toBe(REAL); + }); + }); + + describe("Cohere (bearer, via /v2/chat)", () => { + const COHERE_PATH = "/v2/chat"; + const COHERE_BODY = { + model: "command-r", + messages: [{ role: "user", content: "unmatched-miss" }], + }; + + it("built-in key set + caller sends dummy → injects Bearer", async () => { + const { recorderUrl } = await setup("cohere", REAL); + await post(recorderUrl + COHERE_PATH, COHERE_BODY, { + Authorization: `Bearer ${DUMMY}`, + }); + expect(upstream!.last().authorization).toBe(`Bearer ${REAL}`); + }); + + it("caller sends a real Bearer key → forwarded unchanged", async () => { + const callerReal = "co-caller-real-abc"; + const { recorderUrl } = await setup("cohere", REAL); + await post(recorderUrl + COHERE_PATH, COHERE_BODY, { + Authorization: `Bearer ${callerReal}`, + }); + expect(upstream!.last().authorization).toBe(`Bearer ${callerReal}`); + }); + + it("inert when no built-in key set → dummy forwarded verbatim", async () => { + const { recorderUrl } = await setup("cohere"); // feature OFF + await post(recorderUrl + COHERE_PATH, COHERE_BODY, { + Authorization: `Bearer ${DUMMY}`, + }); + expect(upstream!.last().authorization).toBe(`Bearer ${DUMMY}`); + }); + }); + + describe("Ollama (bearer, via /api/chat)", () => { + const OLLAMA_PATH = "/api/chat"; + const OLLAMA_BODY = { + model: "llama3", + messages: [{ role: "user", content: "unmatched-miss" }], + }; + + it("built-in key set + caller sends dummy → injects Bearer", async () => { + const { recorderUrl } = await setup("ollama", REAL); + await post(recorderUrl + OLLAMA_PATH, OLLAMA_BODY, { + Authorization: `Bearer ${DUMMY}`, + }); + expect(upstream!.last().authorization).toBe(`Bearer ${REAL}`); + }); + + it("inert when no built-in key set → dummy forwarded verbatim", async () => { + const { recorderUrl } = await setup("ollama"); // feature OFF + await post(recorderUrl + OLLAMA_PATH, OLLAMA_BODY, { + Authorization: `Bearer ${DUMMY}`, + }); + expect(upstream!.last().authorization).toBe(`Bearer ${DUMMY}`); + }); + }); + + describe("Azure OpenAI (api-key, via /openai/deployments/{id}/chat/completions)", () => { + const AZURE_PATH = "/openai/deployments/my-deploy/chat/completions"; + + it("built-in key set + caller sends dummy → injects api-key", async () => { + const { recorderUrl } = await setup("azure", REAL); + await post(recorderUrl + AZURE_PATH, CHAT_BODY, { "api-key": DUMMY }); + expect(upstream!.last()["api-key"]).toBe(REAL); + }); + + it("caller sends a real api-key → forwarded unchanged", async () => { + const callerReal = "azure-caller-real-xyz"; + const { recorderUrl } = await setup("azure", REAL); + await post(recorderUrl + AZURE_PATH, CHAT_BODY, { "api-key": callerReal }); + expect(upstream!.last()["api-key"]).toBe(callerReal); + }); + + it("inert when no built-in key set → dummy forwarded verbatim", async () => { + const { recorderUrl } = await setup("azure"); // feature OFF + await post(recorderUrl + AZURE_PATH, CHAT_BODY, { "api-key": DUMMY }); + expect(upstream!.last()["api-key"]).toBe(DUMMY); + }); + }); + + describe("ElevenLabs (xi-api-key, via /v1/text-to-speech/{voice})", () => { + const EL_PATH = "/v1/text-to-speech/voice-123"; + const EL_BODY = { text: "unmatched-miss speak this" }; + + it("built-in key set + caller sends dummy → injects xi-api-key", async () => { + const { recorderUrl } = await setup("elevenlabs", REAL); + await post(recorderUrl + EL_PATH, EL_BODY, { "xi-api-key": DUMMY }); + expect(upstream!.last()["xi-api-key"]).toBe(REAL); + }); + + it("caller sends a real xi-api-key → forwarded unchanged", async () => { + const callerReal = "el-caller-real-777"; + const { recorderUrl } = await setup("elevenlabs", REAL); + await post(recorderUrl + EL_PATH, EL_BODY, { "xi-api-key": callerReal }); + expect(upstream!.last()["xi-api-key"]).toBe(callerReal); + }); + + it("inert when no built-in key set → dummy forwarded verbatim", async () => { + const { recorderUrl } = await setup("elevenlabs"); // feature OFF + await post(recorderUrl + EL_PATH, EL_BODY, { "xi-api-key": DUMMY }); + expect(upstream!.last()["xi-api-key"]).toBe(DUMMY); + }); + }); + + describe("fal.ai (Authorization: Key, via /fal/queue/submit)", () => { + const FAL_PATH = "/fal/queue/submit/fal-ai/some-model"; + const FAL_BODY = { prompt: "unmatched-miss render this" }; + + it("built-in key set + caller sends dummy → injects Authorization: Key", async () => { + const { recorderUrl } = await setup("fal", REAL); + await post(recorderUrl + FAL_PATH, FAL_BODY, { + Authorization: `Key ${DUMMY}`, + }); + expect(upstream!.last().authorization).toBe(`Key ${REAL}`); + }); + + it("caller sends a real Key credential → forwarded unchanged", async () => { + const callerReal = "fal-caller-real-555"; + const { recorderUrl } = await setup("fal", REAL); + await post(recorderUrl + FAL_PATH, FAL_BODY, { + Authorization: `Key ${callerReal}`, + }); + expect(upstream!.last().authorization).toBe(`Key ${callerReal}`); + }); + + it("inert when no built-in key set → dummy forwarded verbatim", async () => { + const { recorderUrl } = await setup("fal"); // feature OFF + await post(recorderUrl + FAL_PATH, FAL_BODY, { + Authorization: `Key ${DUMMY}`, + }); + expect(upstream!.last().authorization).toBe(`Key ${DUMMY}`); + }); + }); + + describe("fixture MATCH short-circuits injection", () => { + it("a matched request never reaches the proxy, so no injection happens", async () => { + upstream = await createFakeUpstream(); + // Serve a fixture that matches the request so proxyAndRecord never runs. + recorder = await createServer( + [ + { + match: { userMessage: "matched-hit" }, + response: { content: "served from fixture" }, + }, + ], + { + port: 0, + record: { + providers: { openai: upstream.url }, + providerKeys: { openai: REAL }, + proxyOnly: true, + }, + }, + ); + const res = await post( + recorder.url + CHAT_PATH, + { model: "gpt-4", messages: [{ role: "user", content: "matched-hit" }] }, + { Authorization: `Bearer ${DUMMY}` }, + ); + expect(res.status).toBe(200); + expect(res.body).toContain("served from fixture"); + // The fake upstream was never contacted → injection path never ran. + expect(upstream!.requestCount()).toBe(0); + }); + }); +}); + +describe("readProviderKeysFromEnv", () => { + const ALL_ENV_VARS = [ + "AIMOCK_PROVIDER_OPENAI_KEY", + "AIMOCK_PROVIDER_ANTHROPIC_KEY", + "AIMOCK_PROVIDER_GEMINI_KEY", + "AIMOCK_PROVIDER_OPENROUTER_KEY", + "AIMOCK_PROVIDER_COHERE_KEY", + "AIMOCK_PROVIDER_GROK_KEY", + "AIMOCK_PROVIDER_OLLAMA_KEY", + "AIMOCK_PROVIDER_VEO_KEY", + "AIMOCK_PROVIDER_AZURE_KEY", + "AIMOCK_PROVIDER_ELEVENLABS_KEY", + "AIMOCK_PROVIDER_FAL_KEY", + ] as const; + const saved = { ...process.env }; + beforeEach(() => { + for (const name of ALL_ENV_VARS) delete process.env[name]; + }); + afterEach(() => { + process.env = { ...saved }; + }); + + it("returns undefined when no provider key env vars are set", async () => { + const { readProviderKeysFromEnv } = await import("../provider-auth.js"); + expect(readProviderKeysFromEnv({})).toBeUndefined(); + }); + + it("reads every wired provider key from its env var", async () => { + const { readProviderKeysFromEnv } = await import("../provider-auth.js"); + const keys = readProviderKeysFromEnv({ + AIMOCK_PROVIDER_OPENAI_KEY: "o", + AIMOCK_PROVIDER_ANTHROPIC_KEY: "a", + AIMOCK_PROVIDER_GEMINI_KEY: "g", + AIMOCK_PROVIDER_OPENROUTER_KEY: "or", + AIMOCK_PROVIDER_COHERE_KEY: "co", + AIMOCK_PROVIDER_GROK_KEY: "gr", + AIMOCK_PROVIDER_OLLAMA_KEY: "ol", + AIMOCK_PROVIDER_VEO_KEY: "ve", + AIMOCK_PROVIDER_AZURE_KEY: "az", + AIMOCK_PROVIDER_ELEVENLABS_KEY: "el", + AIMOCK_PROVIDER_FAL_KEY: "fa", + } as NodeJS.ProcessEnv); + expect(keys).toEqual({ + openai: "o", + anthropic: "a", + gemini: "g", + openrouter: "or", + cohere: "co", + grok: "gr", + ollama: "ol", + veo: "ve", + azure: "az", + elevenlabs: "el", + fal: "fa", + }); + }); + + it("treats an empty-string env var as unset (feature stays inert)", async () => { + const { readProviderKeysFromEnv } = await import("../provider-auth.js"); + const keys = readProviderKeysFromEnv({ + AIMOCK_PROVIDER_OPENAI_KEY: "o", + AIMOCK_PROVIDER_COHERE_KEY: "", + AIMOCK_PROVIDER_FAL_KEY: "", + } as NodeJS.ProcessEnv); + expect(keys).toEqual({ openai: "o" }); + }); +}); diff --git a/src/__tests__/router.test.ts b/src/__tests__/router.test.ts index 50885730..a8190b09 100644 --- a/src/__tests__/router.test.ts +++ b/src/__tests__/router.test.ts @@ -1,5 +1,11 @@ import { describe, it, expect } from "vitest"; -import { matchFixture, getLastMessageByRole, getSystemText, getTextContent } from "../router.js"; +import { + matchFixture, + getLastMessageByRole, + getSystemText, + getTextContent, + getLastUserText, +} from "../router.js"; import type { ChatCompletionRequest, ChatMessage, ContentPart, Fixture } from "../types.js"; // --------------------------------------------------------------------------- @@ -213,6 +219,42 @@ describe("matchFixture — userMessage (array content)", () => { }); expect(matchFixture([fixture], req)).toBeNull(); }); + + it("matches the text prompt when a trailing user message is attachment-only (multimodal image split)", () => { + // Some SDKs (e.g. Microsoft Agent Framework's agent_framework_openai image + // path) serialise a single multimodal turn into a text-only user message + // FOLLOWED by a separate attachment-only user message. The trailing + // image-only message must not shadow the real prompt. + const fixture = makeFixture({ userMessage: "describe this image" }); + const req = makeReq({ + messages: [ + { role: "user", content: "please describe this image" }, + { + role: "user", + content: [{ type: "image_url", image_url: { url: "data:image/png;base64,AAAA" } }], + }, + ], + }); + expect(matchFixture([fixture], req)).toBe(fixture); + }); + + it("keeps matching a trailing user message that HAS text (does not skip a flattened attachment)", () => { + // Contrast with the image split above: when the trailing user message + // carries text (e.g. a pdf flattened to `[Attached document]\n…` by the + // agent) it is NOT skipped — it is the match target. Fixtures for such a + // turn therefore key on the flattened body, not the original prompt. + const fixture = makeFixture({ userMessage: "CopilotKit Quickstart" }); + const req = makeReq({ + messages: [ + { role: "user", content: "can you tell me what is in this demo pdf I just attached" }, + { + role: "user", + content: "[Attached document]\nCopilotKit Quickstart\nAdd AI copilots to your app.", + }, + ], + }); + expect(matchFixture([fixture], req)).toBe(fixture); + }); }); describe("matchFixture — userMessage (RegExp)", () => { @@ -585,6 +627,127 @@ describe("matchFixture — toolCallId", () => { }); }); +// --------------------------------------------------------------------------- +// matchFixture — toolResultContains +// --------------------------------------------------------------------------- + +describe("matchFixture — toolResultContains", () => { + it("discriminates approve vs cancel legs that share a toolCallId", () => { + // The motivating case: a human-in-the-loop suspend tool resumes with the + // SAME tool_call_id for both outcomes; only the tool-result JSON differs. + const cancelled = makeFixture( + { toolCallId: "call_schedule_001", toolResultContains: '"cancelled"' }, + { content: "No problem — nothing was booked." }, + ); + const confirmed = makeFixture( + { toolCallId: "call_schedule_001", toolResultContains: '"chosen_' }, + { content: "Booked: Monday 9:00 AM confirmed." }, + ); + const base = [ + { role: "user" as const, content: "schedule a meeting" }, + { + role: "assistant" as const, + content: null, + tool_calls: [ + { + id: "call_schedule_001", + type: "function" as const, + function: { name: "schedule_meeting", arguments: "{}" }, + }, + ], + }, + ]; + const cancelReq = makeReq({ + messages: [ + ...base, + { role: "tool", content: '{"cancelled": true}', tool_call_id: "call_schedule_001" }, + ], + }); + const pickReq = makeReq({ + messages: [ + ...base, + { + role: "tool", + content: '{"chosen_time": "2026-07-20T09:00:00Z", "chosen_label": "Monday 9:00 AM"}', + tool_call_id: "call_schedule_001", + }, + ], + }); + const fixtures = [cancelled, confirmed]; + expect(matchFixture(fixtures, cancelReq)).toBe(cancelled); + expect(matchFixture(fixtures, pickReq)).toBe(confirmed); + }); + + it("matches when the last tool message contains the substring", () => { + const fixture = makeFixture({ toolResultContains: "cancelled" }); + const req = makeReq({ + messages: [ + { role: "user", content: "do thing" }, + { role: "tool", content: '{"cancelled": true}', tool_call_id: "call_x" }, + ], + }); + expect(matchFixture([fixture], req)).toBe(fixture); + }); + + it("does not match when the substring is absent", () => { + const fixture = makeFixture({ toolResultContains: "cancelled" }); + const req = makeReq({ + messages: [ + { role: "user", content: "do thing" }, + { role: "tool", content: '{"chosen_time": "9am"}', tool_call_id: "call_x" }, + ], + }); + expect(matchFixture([fixture], req)).toBeNull(); + }); + + it("does not match a request with no tool message", () => { + const fixture = makeFixture({ toolResultContains: "cancelled" }); + const req = makeReq({ messages: [{ role: "user", content: "cancelled my plans" }] }); + expect(matchFixture([fixture], req)).toBeNull(); + }); + + it("does not match when a newer user turn follows the tool message", () => { + // Same last-message rule as toolCallId: a stale tool result in history + // must not shadow matchers for the new turn. + const fixture = makeFixture({ toolResultContains: "cancelled" }); + const req = makeReq({ + messages: [ + { role: "user", content: "do thing" }, + { role: "tool", content: '{"cancelled": true}', tool_call_id: "call_x" }, + { role: "assistant", content: "Cancelled." }, + { role: "user", content: "something else" }, + ], + }); + expect(matchFixture([fixture], req)).toBeNull(); + }); + + it("matches array-of-parts tool content", () => { + const fixture = makeFixture({ toolResultContains: "cancelled" }); + const req = makeReq({ + messages: [ + { role: "user", content: "do thing" }, + { + role: "tool", + content: [{ type: "text", text: '{"cancelled": true}' }], + tool_call_id: "call_x", + }, + ], + }); + expect(matchFixture([fixture], req)).toBe(fixture); + }); + + it("does not match tool content with no extractable text", () => { + const fixture = makeFixture({ toolResultContains: "cancelled" }); + const req = makeReq({ + messages: [ + { role: "user", content: "do thing" }, + { role: "tool", content: null, tool_call_id: "call_x" }, + ], + }); + expect(matchFixture([fixture], req)).toBeNull(); + }); +}); + // --------------------------------------------------------------------------- // matchFixture — toolName // --------------------------------------------------------------------------- @@ -1188,3 +1351,144 @@ describe("matchFixture — first-match-wins", () => { expect(matchFixture([noMatch, match], req)).toBe(match); }); }); + +// --------------------------------------------------------------------------- +// Item 1 — null-vs-"" empty-body matching +// --------------------------------------------------------------------------- + +describe("getLastUserText — empty-string user message", () => { + it("returns '' for an explicit empty-text user message", () => { + expect(getLastUserText([{ role: "user", content: "" }])).toBe(""); + }); + it("still skips a trailing attachment-only (null-text) user message", () => { + const msgs: ChatMessage[] = [ + { role: "user", content: "describe this" }, + { role: "user", content: [{ type: "image_url", image_url: { url: "x" } }] as ContentPart[] }, + ]; + expect(getLastUserText(msgs)).toBe("describe this"); + }); + it("returns null when no user message has any text part", () => { + const msgs: ChatMessage[] = [ + { role: "user", content: [{ type: "image_url", image_url: { url: "x" } }] as ContentPart[] }, + ]; + expect(getLastUserText(msgs)).toBeNull(); + }); +}); + +describe("matchFixture — empty userMessage", () => { + it("matches userMessage:'' against an empty user message (exact)", () => { + const fx = makeFixture({ userMessage: "" }); + const req = makeReq({ messages: [{ role: "user", content: "" }] }); + expect(matchFixture([fx], req, undefined, (r) => r)).toBe(fx); + }); + it("matches userMessage:/^$/ against an empty user message", () => { + const fx = makeFixture({ userMessage: /^$/ }); + const req = makeReq({ messages: [{ role: "user", content: "" }] }); + expect(matchFixture([fx], req)).toBe(fx); + }); + it("does NOT match empty userMessage against a non-empty message", () => { + const fx = makeFixture({ userMessage: "" }); + const req = makeReq({ messages: [{ role: "user", content: "hello" }] }); + expect(matchFixture([fx], req, undefined, (r) => r)).toBeNull(); + }); + it("truly-absent user text still skips (attachment-only turn, no fixture match)", () => { + const fx = makeFixture({ userMessage: "" }); + const req = makeReq({ + messages: [ + { + role: "user", + content: [{ type: "image_url", image_url: { url: "x" } }] as ContentPart[], + }, + ], + }); + expect(matchFixture([fx], req, undefined, (r) => r)).toBeNull(); + }); +}); + +describe("matchFixture — empty inputText", () => { + it("matches inputText:'' against embeddingInput:'' (exact)", () => { + const fx = makeFixture({ inputText: "" }, { embedding: [0.1] }); + const req = makeReq({ embeddingInput: "" }); + expect(matchFixture([fx], req, undefined, (r) => r)).toBe(fx); + }); + it("matches inputText:/^$/ against embeddingInput:''", () => { + const fx = makeFixture({ inputText: /^$/ }, { embedding: [0.1] }); + const req = makeReq({ embeddingInput: "" }); + expect(matchFixture([fx], req)).toBe(fx); + }); + it("skips when embeddingInput is undefined (absent)", () => { + const fx = makeFixture({ inputText: "" }, { embedding: [0.1] }); + const req = makeReq({}); + expect(matchFixture([fx], req, undefined, (r) => r)).toBeNull(); + }); +}); + +describe("matchFixture — systemMessage empty behavior unchanged", () => { + it("systemMessage:'' does NOT match a request with no system message (documented catch-all avoidance)", () => { + const fx = makeFixture({ systemMessage: "" }); + const req = makeReq({ messages: [{ role: "user", content: "hi" }] }); + expect(matchFixture([fx], req, undefined, (r) => r)).toBeNull(); + }); +}); + +// --------------------------------------------------------------------------- +// Item 3 — matcher must not mutate caller-supplied RegExp lastIndex +// --------------------------------------------------------------------------- + +describe("matchFixture — does not mutate caller's RegExp lastIndex", () => { + it("leaves a /g userMessage regex lastIndex at 0 after a match", () => { + const re = /hello/g; + const fx = makeFixture({ userMessage: re }); + matchFixture([fx], makeReq({ messages: [{ role: "user", content: "hello" }] })); + expect(re.lastIndex).toBe(0); + }); + it("a /g regex reused across TWO match calls matches BOTH times (no leaked lastIndex)", () => { + const re = /world/g; + const fx = makeFixture({ userMessage: re }); + const req = makeReq({ messages: [{ role: "user", content: "world" }] }); + // Advance the caller's own lastIndex the way an external re.exec would: + re.exec("world world"); // lastIndex now > 0 + expect(matchFixture([fx], req)).toBe(fx); + re.exec("world world"); + expect(matchFixture([fx], req)).toBe(fx); + }); + it("does not clobber the caller's mid-scan lastIndex (userMessage /g)", () => { + const re = /a/g; + re.exec("aaa"); // caller mid-scan, lastIndex === 1 + const fx = makeFixture({ userMessage: re }); + matchFixture([fx], makeReq({ messages: [{ role: "user", content: "a" }] })); + expect(re.lastIndex).toBe(1); + }); + it("systemMessage /g regex lastIndex preserved", () => { + const re = /ctx/g; + re.exec("ctx ctx"); + const before = re.lastIndex; + matchFixture( + [makeFixture({ systemMessage: re })], + makeReq({ + messages: [ + { role: "system", content: "ctx" }, + { role: "user", content: "x" }, + ], + }), + ); + expect(re.lastIndex).toBe(before); + }); + it("inputText /g regex lastIndex preserved", () => { + const re = /q/g; + re.exec("q q"); + const before = re.lastIndex; + matchFixture( + [makeFixture({ inputText: re }, { embedding: [0.1] })], + makeReq({ embeddingInput: "q" }), + ); + expect(re.lastIndex).toBe(before); + }); + it("model /g regex lastIndex preserved", () => { + const re = /gpt/g; + re.exec("gpt gpt"); + const before = re.lastIndex; + matchFixture([makeFixture({ model: re })], makeReq({ model: "gpt-4o" })); + expect(re.lastIndex).toBe(before); + }); +}); diff --git a/src/__tests__/stream-collapse-string-cap.test.ts b/src/__tests__/stream-collapse-string-cap.test.ts new file mode 100644 index 00000000..292b8e5f --- /dev/null +++ b/src/__tests__/stream-collapse-string-cap.test.ts @@ -0,0 +1,160 @@ +import { describe, it, expect, afterEach } from "vitest"; +import { + collapseOpenAISSE, + collapseAnthropicSSE, + collapseGeminiSSE, + collapseCohereSSE, + collapseOllamaNDJSON, + collapseGeminiInteractionsSSE, + collapseBedrockEventStream, + setCollapseStringLimitForTests, + MAX_COLLAPSE_STRING_LENGTH, +} from "../stream-collapse.js"; + +// --------------------------------------------------------------------------- +// Accumulated-string cap on the stream collapsers. +// +// Each collapser accumulates per-channel strings (`content`, `reasoning`, a +// tool call's `arguments`, `audioB64`, ...) from stream fragments with `+=`. +// With no cap a large upstream response builds a string past V8's ~512 MiB max +// string length and throws `RangeError: Invalid string length` — the ~1/sec +// prod crash, caught at server.ts:1230. These tests use a tiny test-only limit +// so they exercise the REAL accumulator/guard code without allocating 512 MiB: +// the collapser must NOT throw, must clamp `content` at the ceiling, and must +// stamp `truncated: true` so the recorder skips journaling a partial fixture. +// --------------------------------------------------------------------------- + +afterEach(() => { + setCollapseStringLimitForTests(undefined); +}); + +/** Build an OpenAI SSE body whose accumulated `content` is `totalChars` long. */ +function openAiSseBody(totalChars: number, perFrame: number): string { + const frames: string[] = []; + let emitted = 0; + while (emitted < totalChars) { + const n = Math.min(perFrame, totalChars - emitted); + frames.push( + `data: ${JSON.stringify({ choices: [{ delta: { content: "a".repeat(n) } }] })}\n\n`, + ); + emitted += n; + } + frames.push("data: [DONE]\n\n"); + return frames.join(""); +} + +describe("stream-collapse accumulated-string cap", () => { + it("has a real ceiling comfortably below V8's max string length", () => { + // V8 max string length on 64-bit is 2^29 - 1 (~536.87M). The ceiling must + // be strictly below it so an accumulator clamped at the ceiling can never + // reach the throwing boundary. + expect(MAX_COLLAPSE_STRING_LENGTH).toBeLessThan(2 ** 29 - 1); + expect(MAX_COLLAPSE_STRING_LENGTH).toBeGreaterThan(0); + }); + + it("clamps OpenAI content at the ceiling, marks truncated, and never throws", () => { + setCollapseStringLimitForTests(1000); + // 5000 chars of content across 100-char frames — 5x over the 1000 ceiling. + const body = openAiSseBody(5000, 100); + + let result: ReturnType | undefined; + expect(() => { + result = collapseOpenAISSE(body); + }).not.toThrow(); + + // content clamped at (or below) the ceiling — never past it. + expect(result!.content!.length).toBeLessThanOrEqual(1000); + // The over-ceiling input is flagged so the recorder skips journaling. + expect(result!.truncated).toBe(true); + }); + + it("does not truncate when content stays under the ceiling", () => { + setCollapseStringLimitForTests(1_000_000); + const body = openAiSseBody(500, 100); + const result = collapseOpenAISSE(body); + expect(result.content).toBe("a".repeat(500)); + expect(result.truncated).toBeUndefined(); + }); + + it("caps Anthropic content and marks truncated without throwing", () => { + setCollapseStringLimitForTests(1000); + const frames: string[] = []; + for (let i = 0; i < 50; i++) { + frames.push( + `event: content_block_delta\ndata: ${JSON.stringify({ + delta: { type: "text_delta", text: "b".repeat(100) }, + })}\n\n`, + ); + } + const body = frames.join(""); + let result: ReturnType | undefined; + expect(() => { + result = collapseAnthropicSSE(body); + }).not.toThrow(); + expect(result!.content!.length).toBeLessThanOrEqual(1000); + expect(result!.truncated).toBe(true); + }); + + it("caps Gemini, Cohere, Ollama, and Gemini-Interactions without throwing", () => { + setCollapseStringLimitForTests(500); + + const gemini = collapseGeminiSSE( + Array.from( + { length: 20 }, + () => + `data: ${JSON.stringify({ + candidates: [{ content: { parts: [{ text: "g".repeat(100) }] } }], + })}\n\n`, + ).join(""), + ); + expect(gemini.content!.length).toBeLessThanOrEqual(500); + expect(gemini.truncated).toBe(true); + + const cohere = collapseCohereSSE( + Array.from( + { length: 20 }, + () => + `event: content-delta\ndata: ${JSON.stringify({ + delta: { message: { content: { text: "c".repeat(100) } } }, + })}\n\n`, + ).join(""), + ); + expect(cohere.content!.length).toBeLessThanOrEqual(500); + expect(cohere.truncated).toBe(true); + + const ollama = collapseOllamaNDJSON( + Array.from( + { length: 20 }, + () => `${JSON.stringify({ message: { content: "o".repeat(100) } })}\n`, + ).join(""), + ); + expect(ollama.content!.length).toBeLessThanOrEqual(500); + expect(ollama.truncated).toBe(true); + + const geminiInteractions = collapseGeminiInteractionsSSE( + Array.from( + { length: 20 }, + () => + `data: ${JSON.stringify({ + event_type: "step.delta", + index: 0, + delta: { type: "text", text: "i".repeat(100) }, + })}\n\n`, + ).join(""), + ); + expect(geminiInteractions.content!.length).toBeLessThanOrEqual(500); + expect(geminiInteractions.truncated).toBe(true); + }); + + it("caps Bedrock EventStream input by byte length and marks truncated", () => { + setCollapseStringLimitForTests(64); + // A raw buffer over the (byte) ceiling — even garbage bytes must be bounded + // and flagged truncated rather than fed unbounded into the accumulators. + const big = Buffer.alloc(4096, 0x41); // 4 KiB of 'A' + let result: ReturnType | undefined; + expect(() => { + result = collapseBedrockEventStream(big); + }).not.toThrow(); + expect(result!.truncated).toBe(true); + }); +}); diff --git a/src/__tests__/ws-realtime-canary.test.ts b/src/__tests__/ws-realtime-canary.test.ts new file mode 100644 index 00000000..ac61ee1a --- /dev/null +++ b/src/__tests__/ws-realtime-canary.test.ts @@ -0,0 +1,177 @@ +/** + * Unit test for the ws-realtime known-voice-models drift canary. + * + * This drives the SAME detection code path the live canary runs + * (`detectVoiceModelDrift` from ws-realtime.drift.ts) against a representative + * `GET /v1/models` id list — no network call, but NOT a reimplemented fake: the + * exported function under test IS the one the live canary invokes. + * + * Regression guard for the voice-model-family blind spot: the canary used to + * filter the model list with `models.filter((m) => m.includes("realtime"))`, so + * a NEW full-duplex voice family whose id lacks the "realtime" substring (e.g. + * OpenAI's gpt-live-1 / gpt-live-1-mini) never entered the unknown-model + * computation and slipped past the canary silently. The broadened voice/audio + * matcher closes that blind spot generally. + */ + +import { describe, it, expect } from "vitest"; +import { + detectVoiceModelDrift, + isVoiceModelId, + knownVoiceModelFamilies, + normalizeVoiceModelFamily, +} from "./drift/voice-models.js"; +import { normalizeModelFamily } from "./drift/model-family.js"; + +// Dated-snapshot / build-tag variants of families that are ALREADY known. These +// are the exact ids the live Drift Tests run (28968203340) flagged as false +// positives before family normalization: each normalizes onto a known family +// (tts-1, tts-1-hd, gpt-audio, gpt-4o-mini-transcribe, gpt-4o-mini-tts) and so +// MUST NOT be flagged. +const KNOWN_FAMILY_SNAPSHOTS = [ + "tts-1-1106", + "tts-1-hd-1106", + "gpt-audio-2025-08-28", + "gpt-4o-mini-transcribe-2025-12-15", + "gpt-4o-mini-transcribe-2025-03-20", + "gpt-4o-mini-tts-2025-03-20", + "gpt-4o-mini-tts-2025-12-15", +]; + +// A representative GET /v1/models id list: the known GA realtime family, some +// known audio/transcribe/tts models (incl. dated snapshots of known families +// that MUST normalize onto their family), a batch of non-voice +// chat/image/embedding models that MUST NOT be flagged, and the newly-shipped +// gpt-live-* full-duplex voice family that is a genuinely new family. +const REPRESENTATIVE_MODELS = [ + // --- known voice/audio family (should stay green) --- + "gpt-realtime", + "gpt-realtime-2.1", + "gpt-realtime-mini", + "gpt-audio-mini", + "gpt-4o-transcribe", + "whisper-1", + "gpt-4o-realtime-preview", + "tts-1", + // --- dated-snapshot / build-tag variants of KNOWN families (stay green) --- + ...KNOWN_FAMILY_SNAPSHOTS, + // --- non-voice models (must NOT be flagged as voice drift) --- + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "o1", + "o3-mini", + "chatgpt-4o-latest", + "dall-e-3", + "text-embedding-3-large", + "text-embedding-3-small", + "omni-moderation-latest", + // --- NEW voice family with no "realtime" substring (the blind spot) --- + "gpt-live-1", + "gpt-live-1-mini", +]; + +describe("ws-realtime known-voice-models canary detection", () => { + it("flags a new voice family whose id lacks the 'realtime' substring (gpt-live-*)", () => { + const { unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS); + + // The whole point: the new gpt-live-* family is surfaced as unknown drift. + expect(unknown).toContain("gpt-live-1"); + expect(unknown).toContain("gpt-live-1-mini"); + }); + + it("does not flag legitimately-known voice/audio models (no false positives)", () => { + const { unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS); + + // Every known-voice model in the payload stays green. + for (const known of [ + "gpt-realtime", + "gpt-realtime-2.1", + "gpt-realtime-mini", + "gpt-audio-mini", + "gpt-4o-transcribe", + "whisper-1", + "gpt-4o-realtime-preview", + "tts-1", + ]) { + expect(knownVoiceModelFamilies.has(normalizeVoiceModelFamily(known))).toBe(true); + expect(unknown).not.toContain(known); + } + }); + + it("does not flag dated-snapshot / build-tag variants of known families", () => { + // Regression guard for live Drift Tests run 28968203340: these 7 real ids + // were flagged as critical drift by the pre-normalization id-set matcher. + // Family normalization collapses each onto an already-known family, so none + // may appear in `unknown`. + const { unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS); + for (const snapshot of KNOWN_FAMILY_SNAPSHOTS) { + expect(unknown).not.toContain(snapshot); + } + }); + + it("normalizes dated snapshots and build tags onto their family", () => { + // Dated snapshot `-YYYY-MM-DD` and build tag `-NNN`/`-NNNN` are stripped. + expect(normalizeVoiceModelFamily("tts-1-1106")).toBe("tts-1"); + expect(normalizeVoiceModelFamily("tts-1-hd-1106")).toBe("tts-1-hd"); + expect(normalizeVoiceModelFamily("gpt-audio-2025-08-28")).toBe("gpt-audio"); + expect(normalizeVoiceModelFamily("gpt-4o-mini-transcribe-2025-12-15")).toBe( + "gpt-4o-mini-transcribe", + ); + expect(normalizeVoiceModelFamily("gpt-4o-mini-tts-2025-03-20")).toBe("gpt-4o-mini-tts"); + // A single-digit trailing tag is NOT a build tag: gpt-live-1 stays a new + // family and must remain flaggable. + expect(normalizeVoiceModelFamily("gpt-live-1")).toBe("gpt-live-1"); + expect(normalizeVoiceModelFamily("gpt-live-1-mini")).toBe("gpt-live-1-mini"); + expect(knownVoiceModelFamilies.has("gpt-live-1")).toBe(false); + }); + + it("is byte-identical to the shared normalizeModelFamily(id, 'openai') primitive", () => { + // Characterization guard for the A3.2 equivalence refactor: normalizeVoiceModelFamily + // is now a thin wrapper over normalizeModelFamily(id, "openai"), and the migrated + // wrapper must yield the SAME normalization for every representative id AND an + // identical knownVoiceModelFamilies set as re-seeding through the shared primitive. + for (const id of [...REPRESENTATIVE_MODELS, ...knownVoiceModelFamilies]) { + expect(normalizeVoiceModelFamily(id)).toBe(normalizeModelFamily(id, "openai")); + } + const reseeded = new Set( + [...knownVoiceModelFamilies].map((f) => normalizeModelFamily(f, "openai")), + ); + expect([...knownVoiceModelFamilies].sort()).toEqual([...reseeded].sort()); + }); + + it("does not flag non-voice chat/image/embedding models as voice drift", () => { + const { candidateModels, unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS); + + for (const nonVoice of [ + "gpt-4o", + "gpt-4o-mini", + "gpt-5", + "gpt-5-mini", + "o1", + "o3-mini", + "chatgpt-4o-latest", + "dall-e-3", + "text-embedding-3-large", + "text-embedding-3-small", + "omni-moderation-latest", + ]) { + expect(isVoiceModelId(nonVoice)).toBe(false); + expect(candidateModels).not.toContain(nonVoice); + expect(unknown).not.toContain(nonVoice); + } + }); + + it("reports GA realtime presence when a GA model exists", () => { + const { hasGA } = detectVoiceModelDrift(REPRESENTATIVE_MODELS); + expect(hasGA).toBe(true); + }); + + it("gpt-live-* is the ONLY unknown in the representative payload", () => { + // Confirms the matcher is neither too narrow (misses gpt-live) nor too broad + // (drags in non-voice models). The unknown set is exactly the new family. + const { unknown } = detectVoiceModelDrift(REPRESENTATIVE_MODELS); + expect([...unknown].sort()).toEqual(["gpt-live-1", "gpt-live-1-mini"]); + }); +}); diff --git a/src/agui-recorder.ts b/src/agui-recorder.ts index 96f72d8c..bb0720e1 100644 --- a/src/agui-recorder.ts +++ b/src/agui-recorder.ts @@ -20,6 +20,55 @@ import type { Logger } from "./logger.js"; */ export const NO_USER_MESSAGE_SENTINEL = "__NO_USER_MESSAGE__"; +/** + * Default ceiling (bytes) for the in-memory AG-UI record buffer. The recorder + * tees every upstream SSE chunk straight to the client AND buffers a copy so it + * can `Buffer.concat(chunks).toString()` + parse the SSE events into a fixture + * once the stream ends. With no cap, a large upstream response (amplified since + * the real-key fixture-miss passthrough landed) builds a string past V8's + * ~512 MiB max string length and throws `RangeError: Invalid string length`. + * 64 MiB mirrors the generic proxy path's `DEFAULT_MAX_PROXY_BUFFER_BYTES` and + * is generous for any real agent turn. Overridable via + * `AGUIRecordConfig.maxRecordBufferBytes`. + */ +export const DEFAULT_AGUI_RECORD_BUFFER_BYTES = 64 * 1024 * 1024; // 64 MiB + +/** + * Absolute hard ceiling (bytes) for the AG-UI record buffer, independent of the + * configurable `maxRecordBufferBytes`, mirroring the generic proxy path's + * `PROXY_BUFFER_HARD_CEILING`. 256 MiB of bytes can never decode to more than + * 256 Mi UTF-16 code units, comfortably below V8's 2^29 - 1 string-length limit, + * so the eventual `Buffer.concat(chunks).toString()` can never throw. + */ +export const AGUI_RECORD_BUFFER_HARD_CEILING = 256 * 1024 * 1024; // 256 MiB + +/** + * Test-only override of the effective AG-UI record-buffer ceiling. Lets the cap + * suite exercise the over-cap truncation path with a small body instead of + * streaming hundreds of MB. `undefined` (the default) uses the configured / + * default cap. NEVER set from production code. + */ +let aguiRecordBufferCeilingOverride: number | undefined; + +/** @internal test-only — see {@link aguiRecordBufferCeilingOverride}. */ +export function setAGUIRecordBufferCeilingForTests(value: number | undefined): void { + aguiRecordBufferCeilingOverride = value; +} + +/** + * Resolve the effective AG-UI record-buffer byte cap: honor a test-only + * override, else the configured `maxRecordBufferBytes` (clamped to the hard + * ceiling), else the default. Non-finite / non-positive configured values fall + * back to the default. + */ +function resolveAGUIRecordBufferCap(configured: number | undefined): number { + if (aguiRecordBufferCeilingOverride !== undefined) return aguiRecordBufferCeilingOverride; + if (configured == null || !Number.isFinite(configured) || configured <= 0) { + return DEFAULT_AGUI_RECORD_BUFFER_BYTES; + } + return Math.min(configured, AGUI_RECORD_BUFFER_HARD_CEILING); +} + /** * Proxy an unmatched AG-UI request to a real upstream agent, record the * SSE event stream as a fixture on disk and in memory, and relay the @@ -152,6 +201,15 @@ function teeUpstreamStream( const chunks: Buffer[] = []; let clientWriteFailed = false; + // Bound the in-memory record buffer so a single huge upstream response + // cannot build a string past V8's ~512 MiB limit + // (RangeError: Invalid string length) when we `Buffer.concat`+stringify + // it below. The client relay above is INDEPENDENT of this buffer, so + // capping it never truncates what the client receives — the cap means + // "don't journal", not "don't answer" (mirrors the generic proxy path). + const recordBufferCap = resolveAGUIRecordBufferCap(config.maxRecordBufferBytes); + let bufferedBytes = 0; + let recordTruncated = false; upstreamRes.on("data", (chunk: Buffer) => { // Relay to client in real time @@ -166,8 +224,21 @@ function teeUpstreamStream( ); } } - // Buffer for fixture construction + // Buffer for fixture construction, bounded by the record-buffer cap. + // Once the cap is crossed we stop accumulating and free the buffer so + // heap growth is bounded and the end-handler skips concat/stringify. + if (recordTruncated) return; + if (bufferedBytes + chunk.length > recordBufferCap) { + recordTruncated = true; + chunks.length = 0; + bufferedBytes = 0; + logger?.warn( + `AG-UI upstream response exceeded the ${recordBufferCap}-byte record buffer cap — relaying full body to client, but skipping fixture recording to bound memory`, + ); + return; + } chunks.push(chunk); + bufferedBytes += chunk.length; }); let settled = false; @@ -217,6 +288,20 @@ function teeUpstreamStream( ); } + // Record buffer cap tripped: the buffered copy is partial (and was + // freed), so we cannot faithfully build a fixture. The client already + // received every byte via the live tee above, so just skip recording + // — never stringify the (now-empty) buffer, never persist a truncated + // fixture. Mirrors the generic proxy path's over-cap "skip recording, + // still answer the client" behavior. + if (recordTruncated) { + logger.warn( + "AG-UI record buffer cap exceeded — response relayed to client, recording skipped", + ); + resolve(clientStatus); + return; + } + // Parse buffered SSE events const buffered = Buffer.concat(chunks).toString(); const events = parseSSEEvents(buffered, logger); diff --git a/src/agui-types.ts b/src/agui-types.ts index 4fb2253b..03fbf8e8 100644 --- a/src/agui-types.ts +++ b/src/agui-types.ts @@ -411,4 +411,17 @@ export interface AGUIRecordConfig { upstream: string; fixturePath?: string; proxyOnly?: boolean; + /** + * Maximum number of bytes the AG-UI recorder will accumulate in memory from a + * single proxied upstream SSE stream in order to parse + journal it. The full + * stream is still relayed to the client byte-for-byte in real time; this cap + * only bounds the in-memory buffer that is later `Buffer.concat`-ed and + * stringified for fixture construction. Once exceeded the recorder stops + * appending to the buffer, marks the recording truncated, and skips fixture + * construction — preventing both unbounded heap growth and the + * `RangeError: Invalid string length` a >512MB `Buffer.concat(chunks).toString()` + * would otherwise throw. Clamped to `AGUI_RECORD_BUFFER_HARD_CEILING`. + * Default: 64 MiB. + */ + maxRecordBufferBytes?: number; } diff --git a/src/cli.ts b/src/cli.ts index 4e366e49..7056472d 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -8,6 +8,7 @@ import { Logger, type LogLevel } from "./logger.js"; import { watchFixtures } from "./watcher.js"; import { AGUIMock } from "./agui-mock.js"; import { resolveFixturesValue } from "./fixtures-remote.js"; +import { readProviderKeysFromEnv } from "./provider-auth.js"; import type { Fixture, ChaosConfig, RecordConfig } from "./types.js"; const HELP = ` @@ -276,6 +277,10 @@ if (values.record || values["proxy-only"]) { } record = { providers, + // aimock's own upstream keys, sourced from AIMOCK_PROVIDER_*_KEY env vars + // (not CLI flags — secrets must not appear in `ps`). Injected on a + // fixture-miss passthrough when the caller sent no/dummy credential. + providerKeys: readProviderKeysFromEnv(), // In proxy-only mode with only URL sources, fixturePath is never consumed // (recorder.ts skips disk writes when proxyOnly is set). Leave it undefined // rather than resolving a URL string as a filesystem path. diff --git a/src/config-loader.ts b/src/config-loader.ts index a02e4500..871904ca 100644 --- a/src/config-loader.ts +++ b/src/config-loader.ts @@ -90,6 +90,10 @@ export interface VectorConfig { export interface AimockConfig { llm?: { fixtures?: string; + latency?: number; + chunkSize?: number; + replaySpeed?: number; + logLevel?: "silent" | "warn" | "info" | "debug"; chaos?: ChaosConfig; record?: RecordConfig; }; @@ -115,10 +119,22 @@ export async function startFromConfig( ): Promise<{ llmock: LLMock; url: string }> { const logger = new Logger("info"); + // A non-positive replaySpeed fails calculateDelay's `speed > 0` check and applies the + // full delay rather than none. Mirrors the fixture-level guard in fixture-loader.ts. + let replaySpeed = config.llm?.replaySpeed; + if (replaySpeed != null && (!Number.isFinite(replaySpeed) || replaySpeed <= 0)) { + logger.warn(`llm.replaySpeed must be a positive number, got ${replaySpeed}. Ignoring.`); + replaySpeed = undefined; + } + // Load fixtures if specified const llmock = new LLMock({ port: overrides?.port ?? config.port ?? 0, host: overrides?.host ?? config.host ?? "127.0.0.1", + latency: config.llm?.latency, + chunkSize: config.llm?.chunkSize, + replaySpeed, + logLevel: config.llm?.logLevel, chaos: config.llm?.chaos, record: config.llm?.record, metrics: config.metrics, diff --git a/src/fal-audio.ts b/src/fal-audio.ts index 62aa6a12..f79a8a64 100644 --- a/src/fal-audio.ts +++ b/src/fal-audio.ts @@ -577,6 +577,9 @@ async function tryRecordAudioQueueWalk(args: { submitPath: pathname, body, headers: buildForwardHeaders(req), + // aimock's built-in fal key (Authorization: Key), injected by the walk on + // a no/dummy caller credential; a real caller key overrides. + builtinKey: record.providerKeys?.fal, pollIntervalMs: record.fal?.pollIntervalMs, timeoutMs: record.fal?.timeoutMs, upstreamTimeoutMs: record.upstreamTimeoutMs, diff --git a/src/fal.ts b/src/fal.ts index 9fb16ba9..268aace6 100644 --- a/src/fal.ts +++ b/src/fal.ts @@ -36,6 +36,7 @@ import { sanitizeHeaderValue, } from "./recorder.js"; import { resolveUpstreamUrl } from "./url.js"; +import { applyProviderAuth } from "./provider-auth.js"; import type { Journal } from "./journal.js"; import { audioToFalFile } from "./fal-audio.js"; @@ -885,6 +886,14 @@ export async function walkFalQueue(args: { fallbackResultPath: (requestId: string) => string; /** Warn sink for the same-origin envelope-URL gate (omitting it only mutes the warns). */ logger?: Logger; + /** + * aimock's built-in fal key, when configured. Injected as `Authorization: Key + * ` on every walk fetch (submit/status/result) if the caller sent no + * credential or a dummy placeholder; a real caller credential overrides. This + * is the single chokepoint for all fal queue walks — the queue path doesn't + * route through `proxyAndRecord`, so own-key injection lives here. + */ + builtinKey?: string; }): Promise { const { upstreamBase, @@ -897,8 +906,14 @@ export async function walkFalQueue(args: { fallbackStatusPath, fallbackResultPath, logger, + builtinKey, } = args; + // Inject aimock's own fal credential onto the walk headers up front so every + // fetch below (submit + polls) carries it. fal's scheme ignores the target + // URL, so any resolvable URL suffices. + applyProviderAuth(headers, resolveUpstreamUrl(upstreamBase, submitPath), "fal", builtinKey); + const deadline = Date.now() + timeoutMs; const perFetchTimeoutMs = clampTimeout(upstreamTimeoutMs, DEFAULT_FAL_FETCH_TIMEOUT_MS); // Bound every upstream fetch: the per-fetch timeout, additionally clamped @@ -1065,6 +1080,9 @@ async function proxyAndRecordFalQueueSubmit(args: { submitPath: strippedPath, body, headers: buildForwardHeaders(req), + // aimock's built-in fal key (Authorization: Key), injected by the walk on + // a no/dummy caller credential; a real caller key overrides. + builtinKey: record.providerKeys?.fal, pollIntervalMs: record.fal?.pollIntervalMs, timeoutMs: record.fal?.timeoutMs, upstreamTimeoutMs: record.upstreamTimeoutMs, diff --git a/src/fixture-loader.ts b/src/fixture-loader.ts index 9d911fcb..4fe26366 100644 Binary files a/src/fixture-loader.ts and b/src/fixture-loader.ts differ diff --git a/src/index.ts b/src/index.ts index b93c3030..8abe128d 100644 --- a/src/index.ts +++ b/src/index.ts @@ -177,6 +177,8 @@ export { collapseCohereSSE, collapseBedrockEventStream, collapseStreamingResponse, + MAX_COLLAPSE_STRING_LENGTH, + setCollapseStringLimitForTests, } from "./stream-collapse.js"; export type { CollapseResult } from "./stream-collapse.js"; @@ -224,7 +226,12 @@ export type { // AG-UI export { AGUIMock } from "./agui-mock.js"; -export { proxyAndRecordAGUI } from "./agui-recorder.js"; +export { + proxyAndRecordAGUI, + DEFAULT_AGUI_RECORD_BUFFER_BYTES, + AGUI_RECORD_BUFFER_HARD_CEILING, + setAGUIRecordBufferCeilingForTests, +} from "./agui-recorder.js"; export type { AGUIMockOptions, AGUIRunAgentInput, diff --git a/src/journal.ts b/src/journal.ts index 93f3c4dc..59dc2578 100644 --- a/src/journal.ts +++ b/src/journal.ts @@ -1,8 +1,46 @@ import { generateId } from "./helpers.js"; -import type { Fixture, FixtureMatch, JournalEntry } from "./types.js"; +import type { ChatCompletionRequest, Fixture, FixtureMatch, JournalEntry } from "./types.js"; import { DEFAULT_TEST_ID } from "./constants.js"; export { DEFAULT_TEST_ID } from "./constants.js"; +/** + * Maximum UTF-8 byte length of a serialized request body retained in a + * journal entry. Bodies whose JSON serialization exceeds this byte size are + * replaced with a truncation marker. + * + * Only the request `body` field is capped here. `headers` and `path` are + * retained uncapped, but they are naturally bounded by Node's default ~16 KB + * maximum HTTP header size, so the per-entry overhead beyond the body cap + * remains small. Combined with the journal's maxEntries limit (default 1000), + * the total `JSON.stringify(journal.getAll())` output stays well under V8's + * ~512 MB string limit even at maximum capacity. + */ +const JOURNAL_BODY_CAP_BYTES = 64 * 1024; // 64 KB + +/** + * If `body` serializes to more than JOURNAL_BODY_CAP_BYTES (in UTF-8 bytes), + * replace it with a truncation marker. Returns the original body when within + * the cap, or null when `body` is null. + * + * The gate uses Buffer.byteLength (UTF-8 bytes) rather than .length (UTF-16 + * code units) to match the constant name and to correctly cap multibyte + * content (e.g. CJK/emoji) whose code-unit count falls under the threshold + * but whose byte size does not. + */ +function capBody(body: ChatCompletionRequest | null): ChatCompletionRequest | null { + if (body === null) return null; + const serialized = JSON.stringify(body); + if (Buffer.byteLength(serialized, "utf8") <= JOURNAL_BODY_CAP_BYTES) return body; + // Cast: the marker is not a real ChatCompletionRequest, but the field type + // is already nullable-union and downstream consumers (e.g. GET /journal) + // treat the body as opaque JSON — the truncation marker is safe to store. + return { + __aimock_truncated: true, + originalByteSize: Buffer.byteLength(serialized, "utf8"), + note: "body truncated by aimock journal cap (64 KB limit)", + } as unknown as ChatCompletionRequest; +} + /** * Compare two field values, handling RegExp by source+flags rather than reference. */ @@ -101,6 +139,7 @@ export class Journal { id: generateId("req"), timestamp: Date.now(), ...entry, + body: capBody(entry.body), }; this.entries.push(full); // FIFO eviction when over capacity. Array.prototype.shift() is O(n) diff --git a/src/provider-auth.ts b/src/provider-auth.ts new file mode 100644 index 00000000..b00f436e --- /dev/null +++ b/src/provider-auth.ts @@ -0,0 +1,238 @@ +import type { RecordProviderKey } from "./types.js"; + +/** + * Default marker prefix identifying a "dummy" credential that a caller sends + * only to satisfy an SDK's non-empty API-key requirement (e.g. `sk-aimock-...`). + * A caller credential that is absent OR begins with this prefix is treated as a + * placeholder that aimock is free to override with its own built-in upstream key. + * A caller credential that does NOT begin with this prefix is treated as a real + * key and forwarded verbatim (the caller overrides aimock). + * + * Overridable via the `AIMOCK_DUMMY_KEY_MARKER` env var for setups that mint + * placeholder keys under a different prefix. + */ +export const DEFAULT_DUMMY_KEY_MARKER = "sk-aimock-"; + +/** Resolve the active dummy-key marker, honoring the env override. */ +export function getDummyKeyMarker(): string { + const override = process.env.AIMOCK_DUMMY_KEY_MARKER; + return override && override.length > 0 ? override : DEFAULT_DUMMY_KEY_MARKER; +} + +/** + * How a given provider carries its API credential on the wire. aimock injects + * its built-in key using the provider's native scheme so the upstream accepts + * it. Providers whose auth is a signed/exchanged credential (Bedrock SigV4, + * Vertex AI OAuth) are deliberately absent — those are NOT simple static-key + * schemes, so aimock never rewrites their auth and always forwards the caller's + * credential unchanged. + * + * Scheme kinds (all static long-lived keys): + * - bearer `Authorization: Bearer ` (OpenAI, OpenRouter, Cohere, Grok/xAI, Ollama) + * - fal-key `Authorization: Key ` (fal.ai — note the `Key ` prefix, NOT `Bearer`) + * - x-api-key `x-api-key: ` (Anthropic) + * - x-goog-api-key `x-goog-api-key: ` (Gemini, Gemini Interactions, Veo) + * - api-key `api-key: ` (Azure OpenAI static-key auth) + * - xi-api-key `xi-api-key: ` (ElevenLabs) + * + * Note on Azure: Azure OpenAI also supports Microsoft Entra ID `Authorization: + * Bearer ` OAuth. aimock only ever injects the static `api-key` header, + * and a real Entra bearer token from the caller is never dummy-prefixed, so it + * is always forwarded verbatim (the caller overrides aimock). + */ +type AuthScheme = + | { kind: "bearer" } // Authorization: Bearer + | { kind: "fal-key" } // Authorization: Key + | { kind: "x-api-key" } // x-api-key: + | { kind: "x-goog-api-key" } // x-goog-api-key: + | { kind: "api-key" } // api-key: + | { kind: "xi-api-key" }; // xi-api-key: + +const PROVIDER_AUTH_SCHEMES: Partial> = { + // Bearer-token providers. + openai: { kind: "bearer" }, + openrouter: { kind: "bearer" }, + cohere: { kind: "bearer" }, + grok: { kind: "bearer" }, // xAI + ollama: { kind: "bearer" }, // Ollama Cloud / bearer-gated Ollama servers + // Anthropic. + anthropic: { kind: "x-api-key" }, + // Google (Gemini + Veo) use the x-goog-api-key header scheme. + gemini: { kind: "x-goog-api-key" }, + "gemini-interactions": { kind: "x-goog-api-key" }, + veo: { kind: "x-goog-api-key" }, + // Azure OpenAI static-key auth. + azure: { kind: "api-key" }, + // ElevenLabs. + elevenlabs: { kind: "xi-api-key" }, + // fal.ai — uses the `Key ` prefix on Authorization, NOT `Bearer `. + fal: { kind: "fal-key" }, +}; + +/** + * Case-insensitively delete a header from a forward-header map, returning the + * value that was present (if any). Header maps preserve the original casing of + * incoming headers, so we can't assume a canonical key. + */ +function takeHeader(headers: Record, name: string): string | undefined { + const lowerName = name.toLowerCase(); + let found: string | undefined; + for (const key of Object.keys(headers)) { + if (key.toLowerCase() === lowerName) { + found = headers[key]; + delete headers[key]; + } + } + return found; +} + +/** + * Extract the caller's credential value for a given auth scheme from the + * forwarded headers, if present. For bearer, strips the `Bearer ` prefix. + */ +function readCallerCredential( + headers: Record, + scheme: AuthScheme, +): string | undefined { + if (scheme.kind === "bearer" || scheme.kind === "fal-key") { + const raw = scanHeader(headers, "authorization"); + if (raw === undefined) return undefined; + // Strip the scheme prefix (`Bearer ` for bearer, `Key ` for fal) if present, + // otherwise treat the whole value as the credential. + const prefix = scheme.kind === "fal-key" ? "Key" : "Bearer"; + const match = new RegExp(`^\\s*${prefix}\\s+(.+)$`, "i").exec(raw); + return match ? match[1].trim() : raw.trim(); + } + return scanHeader(headers, authHeaderName(scheme)); +} + +/** + * The wire header name a non-Authorization scheme carries its credential in. + * (bearer/fal-key both use `Authorization` and are handled separately.) + */ +function authHeaderName(scheme: AuthScheme): string { + switch (scheme.kind) { + case "x-api-key": + return "x-api-key"; + case "x-goog-api-key": + return "x-goog-api-key"; + case "api-key": + return "api-key"; + case "xi-api-key": + return "xi-api-key"; + case "bearer": + case "fal-key": + return "authorization"; + } +} + +/** Case-insensitive header lookup that does not mutate the map. */ +function scanHeader(headers: Record, name: string): string | undefined { + const lowerName = name.toLowerCase(); + for (const key of Object.keys(headers)) { + if (key.toLowerCase() === lowerName) return headers[key]; + } + return undefined; +} + +/** + * Decide whether aimock should override the caller's credential with its own + * built-in key. Override when the caller sent no credential OR sent a dummy + * placeholder (prefixed with the active marker). A real caller key is left + * untouched so the caller can always override aimock. + */ +function shouldOverride(callerCredential: string | undefined, marker: string): boolean { + if (callerCredential === undefined || callerCredential.length === 0) return true; + return callerCredential.startsWith(marker); +} + +/** + * Inject aimock's built-in upstream credential into the forwarded headers when + * appropriate (opt-in, backward-compatible). Mutates `forwardHeaders` in place + * and may mutate `target` (query-param schemes; none currently). + * + * Precedence: + * - No built-in key configured for this provider → no-op (forward verbatim). + * - Provider not a static-key scheme → no-op (Bedrock SigV4 / Vertex AI OAuth). + * - Caller credential absent or dummy-prefixed → inject built-in key. + * - Caller credential is a real key → forward it unchanged. + * + * @param forwardHeaders headers already built by buildForwardHeaders(req) + * @param target upstream URL (reserved for query-param auth schemes) + * @param providerKey which provider this request is routed to + * @param builtinKey aimock's configured key for this provider (if any) + */ +export function applyProviderAuth( + forwardHeaders: Record, + target: URL, + providerKey: RecordProviderKey, + builtinKey: string | undefined, +): void { + void target; // reserved for future query-param schemes (e.g. Gemini ?key=) + if (!builtinKey) return; // feature inert for this provider + + const scheme = PROVIDER_AUTH_SCHEMES[providerKey]; + if (!scheme) return; // signed/OAuth provider — never rewrite auth + + const marker = getDummyKeyMarker(); + const callerCredential = readCallerCredential(forwardHeaders, scheme); + if (!shouldOverride(callerCredential, marker)) return; // caller sent a real key + + // Strip any existing auth headers for this scheme (across casings) before + // setting aimock's key, so a dummy caller key can't linger alongside ours. + switch (scheme.kind) { + case "bearer": + takeHeader(forwardHeaders, "authorization"); + forwardHeaders["Authorization"] = `Bearer ${builtinKey}`; + break; + case "fal-key": + takeHeader(forwardHeaders, "authorization"); + forwardHeaders["Authorization"] = `Key ${builtinKey}`; + break; + case "x-api-key": + takeHeader(forwardHeaders, "x-api-key"); + forwardHeaders["x-api-key"] = builtinKey; + break; + case "x-goog-api-key": + takeHeader(forwardHeaders, "x-goog-api-key"); + forwardHeaders["x-goog-api-key"] = builtinKey; + break; + case "api-key": + takeHeader(forwardHeaders, "api-key"); + forwardHeaders["api-key"] = builtinKey; + break; + case "xi-api-key": + takeHeader(forwardHeaders, "xi-api-key"); + forwardHeaders["xi-api-key"] = builtinKey; + break; + } +} + +/** + * Read per-provider built-in keys from the environment. Returns undefined when + * no provider key is configured so the feature stays fully inert by default. + * + * Each wired static-key provider reads `AIMOCK_PROVIDER__KEY`. An + * empty-string value is treated as unset (existing truthiness pattern), keeping + * the feature inert. `gemini-interactions` is not read here: it reuses the + * Gemini key via the lookup-key remap in `proxyAndRecord`. Signed/OAuth + * providers (`bedrock`, `vertexai`) have no env var — aimock never rewrites + * their auth. + */ +export function readProviderKeysFromEnv( + env: NodeJS.ProcessEnv = process.env, +): Partial> | undefined { + const keys: Partial> = {}; + if (env.AIMOCK_PROVIDER_OPENAI_KEY) keys.openai = env.AIMOCK_PROVIDER_OPENAI_KEY; + if (env.AIMOCK_PROVIDER_ANTHROPIC_KEY) keys.anthropic = env.AIMOCK_PROVIDER_ANTHROPIC_KEY; + if (env.AIMOCK_PROVIDER_GEMINI_KEY) keys.gemini = env.AIMOCK_PROVIDER_GEMINI_KEY; + if (env.AIMOCK_PROVIDER_OPENROUTER_KEY) keys.openrouter = env.AIMOCK_PROVIDER_OPENROUTER_KEY; + if (env.AIMOCK_PROVIDER_COHERE_KEY) keys.cohere = env.AIMOCK_PROVIDER_COHERE_KEY; + if (env.AIMOCK_PROVIDER_GROK_KEY) keys.grok = env.AIMOCK_PROVIDER_GROK_KEY; + if (env.AIMOCK_PROVIDER_OLLAMA_KEY) keys.ollama = env.AIMOCK_PROVIDER_OLLAMA_KEY; + if (env.AIMOCK_PROVIDER_VEO_KEY) keys.veo = env.AIMOCK_PROVIDER_VEO_KEY; + if (env.AIMOCK_PROVIDER_AZURE_KEY) keys.azure = env.AIMOCK_PROVIDER_AZURE_KEY; + if (env.AIMOCK_PROVIDER_ELEVENLABS_KEY) keys.elevenlabs = env.AIMOCK_PROVIDER_ELEVENLABS_KEY; + if (env.AIMOCK_PROVIDER_FAL_KEY) keys.fal = env.AIMOCK_PROVIDER_FAL_KEY; + return Object.keys(keys).length > 0 ? keys : undefined; +} diff --git a/src/recorder.ts b/src/recorder.ts index eb299688..a4faaa1c 100644 --- a/src/recorder.ts +++ b/src/recorder.ts @@ -20,16 +20,25 @@ import type { Logger } from "./logger.js"; import { collapseStreamingResponse, capturedRedactedData } from "./stream-collapse.js"; import { writeErrorResponse } from "./sse-writer.js"; import { resolveUpstreamUrl } from "./url.js"; +import { applyProviderAuth } from "./provider-auth.js"; import { getTestId, slugifyTestId, slugifyContext } from "./helpers.js"; import { DEFAULT_TEST_ID } from "./constants.js"; /** Headers to strip when proxying — hop-by-hop (RFC 2616 §13.5.1) + client-set. */ /** * Default ceiling (bytes) for the in-memory proxy-path buffer. Chosen well - * under V8's ~512 MiB max string length so `rawBuffer.toString()` / - * stream-collapse never throws `RangeError: Invalid string length`, and so a - * single huge proxied response cannot spike the heap unbounded. Overridable - * via `RecordConfig.maxProxyBufferBytes` / `--max-proxy-buffer-bytes`. + * under V8's ~512 MiB max string length so `rawBuffer.toString()` cannot throw + * `RangeError: Invalid string length`, and so a single huge proxied response + * cannot spike the heap unbounded. Overridable via + * `RecordConfig.maxProxyBufferBytes` / `--max-proxy-buffer-bytes`. + * + * NOTE: this byte cap bounds the RAW BUFFER only. `stream-collapse` accumulates + * its own per-channel strings (`content`/`reasoning`/tool `arguments`/etc.) from + * that buffer and is ALSO reachable directly (exported from index.ts), so it + * carries its OWN independent guard (`MAX_COLLAPSE_STRING_LENGTH`) rather than + * relying on this cap — an earlier version of this comment wrongly asserted + * stream-collapse "never throws Invalid string length", which was false for the + * unbounded `content += delta` accumulators (the ~1/sec prod RangeError). */ export const DEFAULT_MAX_PROXY_BUFFER_BYTES = 64 * 1024 * 1024; // 64 MiB @@ -431,6 +440,12 @@ export async function proxyAndRecord( // Forward all request headers except hop-by-hop and client-set ones. const forwardHeaders = buildForwardHeaders(req); + // If aimock owns a built-in upstream key for this provider, inject it now + // (opt-in, backward-compatible). A real caller credential overrides it; an + // absent or dummy-prefixed caller credential is replaced. gemini-interactions + // reuses the Gemini key, mirroring the upstream-URL remap above. + applyProviderAuth(forwardHeaders, target, providerKey, record.providerKeys?.[lookupKey]); + const requestBody = rawBody ?? JSON.stringify(request); // Make upstream request diff --git a/src/router.ts b/src/router.ts index fe6ec21b..66b71a07 100644 --- a/src/router.ts +++ b/src/router.ts @@ -70,6 +70,47 @@ export function getTextContent(content: string | ContentPart[] | null): string | return null; } +/** + * Text of the last user message, for `userMessage` fixture matching. Normally + * this is the text of the final `user` message. But some SDKs serialise a + * single multimodal user turn (prompt text + attachment) into TWO consecutive + * user messages — a text-only one FOLLOWED by an attachment-only one, e.g. + * `[{role:"user", content:"describe this"}, {role:"user", content:[{type:"image_url",...}]}]` + * (observed with Microsoft Agent Framework's `agent_framework_openai` image + * path). The trailing attachment-only message has NO extractable text, so the + * naive "last user message" lookup returns `null` and — because no fixture can + * key on empty text — every such multimodal fixture misses. We therefore skip + * trailing user messages that carry no text and use the nearest preceding user + * message that does. This is deliberately narrow: it only skips text-LESS + * (`null`) user messages — a message whose text extracts to an explicit empty + * string `""` is a present-but-empty body and IS returned (so a fixture can key + * on it), while a genuine multi-message user turn (each message HAS text) is + * unaffected and still matched on its final message. Returns the nearest + * preceding user message whose text is non-null (including `""`); returns `null` + * only when no user message carries any text part at all. + */ +export function getLastUserText(messages: ChatMessage[]): string | null { + for (let i = messages.length - 1; i >= 0; i--) { + if (messages[i].role !== "user") continue; + const text = getTextContent(messages[i].content); + if (text !== null) return text; + } + return null; +} + +/** + * Test a matcher RegExp against a string WITHOUT mutating the caller-supplied + * regex. `RegExp.prototype.test` advances `lastIndex` as a side effect on + * `/g` and `/y` regexes; fixtures hold caller-owned RegExp objects, so testing + * them in place would clobber the caller's positional state (and a `/g` regex + * reused across match calls would match intermittently). We test a fresh clone + * (same source + flags) so the caller's object is never touched and every test + * starts from index 0. + */ +function regexTest(re: RegExp, text: string): boolean { + return new RegExp(re.source, re.flags).test(text); +} + /** * Result of {@link matchFixtureDiagnostic}: the matched fixture (or `null`) plus * the number of fixtures that matched the request SHAPE (every predicate above @@ -276,9 +317,15 @@ export function matchFixtureDiagnostic( // matchesPattern() in helpers.ts, which is used for search/rerank/moderation // where exact casing rarely matters. if (match.userMessage !== undefined) { - const msg = getLastMessageByRole(effective.messages, "user"); - const text = msg ? getTextContent(msg.content) : null; - if (!text) continue; + // Use the last user message that actually carries text — see + // getLastUserText for why a trailing attachment-only user message + // (multimodal serialisation split) must not shadow the real prompt. + const text = getLastUserText(effective.messages); + // `text === null` means no user message carried any text (e.g. a pure + // attachment turn) — skip. An explicit empty-string body (`""`) is a + // present-but-empty user message and must be allowed through so a fixture + // keyed on empty text can match it (see getLastUserText). + if (text === null) continue; if (typeof match.userMessage === "string") { if (useExactMatch) { if (text !== match.userMessage) continue; @@ -286,8 +333,7 @@ export function matchFixtureDiagnostic( if (!text.includes(match.userMessage)) continue; } } else { - match.userMessage.lastIndex = 0; - if (!match.userMessage.test(text)) continue; + if (!regexTest(match.userMessage, text)) continue; } } @@ -315,6 +361,15 @@ export function matchFixtureDiagnostic( // no constraint — fall through to the next predicate } else { const text = getSystemText(effective.messages); + // Deliberately `!text` (not `text === null`): unlike userMessage/inputText, + // getSystemText returns `""` for BOTH "a present but empty system message" + // AND "no system message at all", so it exposes no absent-vs-empty + // distinction at the request level. Allowing `""` through would make a + // `systemMessage: ""` / `/^$/` fixture a catch-all firing on every + // no-system-message request. There is no shipped-fixture demand for + // matching an empty system prompt, so we keep the falsy guard here. If a + // real need arises, add a getSystemText sibling that returns `null` when + // absent and `""` when present-empty, then switch to `text === null`. if (!text) continue; if (Array.isArray(sm)) { let allPresent = true; @@ -332,8 +387,7 @@ export function matchFixtureDiagnostic( if (!text.includes(sm)) continue; } } else { - sm.lastIndex = 0; - if (!sm.test(text)) continue; + if (!regexTest(sm, text)) continue; } } } @@ -347,6 +401,19 @@ export function matchFixtureDiagnostic( if (!last || last.role !== "tool" || last.tool_call_id !== match.toolCallId) continue; } + // toolResultContains — substring gate on the LAST message's text content + // when that message is a tool result. Same last-message rule as toolCallId + // (a tool-result fixture answers the model's next call after a tool round), + // but discriminates on the result PAYLOAD instead of the call id — needed + // when approve/cancel resumes share the same toolCallId and differ only + // inside the tool-result JSON. + if (match.toolResultContains !== undefined) { + const last = effective.messages[effective.messages.length - 1]; + if (!last || last.role !== "tool") continue; + const text = getTextContent(last.content); + if (text === null || !text.includes(match.toolResultContains)) continue; + } + // toolName — match against any tool definition by function.name if (match.toolName !== undefined) { const tools = effective.tools ?? []; @@ -358,7 +425,10 @@ export function matchFixtureDiagnostic( // Same rationale as userMessage above: fixture authors specify exact strings. if (match.inputText !== undefined) { const embeddingInput = effective.embeddingInput; - if (!embeddingInput) continue; + // `undefined` means the request carried no embedding input (a non-embedding + // request) — skip. An explicit empty string (`""`) is a genuinely-empty + // embedding input and must be allowed through so `inputText: ""` can match. + if (embeddingInput === undefined) continue; if (typeof match.inputText === "string") { if (useExactMatch) { if (embeddingInput !== match.inputText) continue; @@ -366,8 +436,7 @@ export function matchFixtureDiagnostic( if (!embeddingInput.includes(match.inputText)) continue; } } else { - match.inputText.lastIndex = 0; - if (!match.inputText.test(embeddingInput)) continue; + if (!regexTest(match.inputText, embeddingInput)) continue; } } @@ -388,8 +457,7 @@ export function matchFixtureDiagnostic( if (!/^-\d/.test(rest)) continue; } } else { - match.model.lastIndex = 0; - if (!match.model.test(effective.model ?? "")) continue; + if (!regexTest(match.model, effective.model ?? "")) continue; } } diff --git a/src/server.ts b/src/server.ts index 2e9bb4b8..fea22132 100644 --- a/src/server.ts +++ b/src/server.ts @@ -1227,7 +1227,10 @@ export async function createServer( // Delegate to async handler — catch unhandled rejections to prevent Node.js crashes handleHttpRequest(req, res).catch((err: unknown) => { const msg = err instanceof Error ? err.message : "Internal error"; - defaults.logger.warn(`Unhandled request error: ${msg}`); + const stack = err instanceof Error ? (err.stack ?? msg) : msg; + const method = req.method ?? "?"; + const url = req.url ?? "?"; + defaults.logger.warn(`Unhandled request error on ${method} ${url}: ${msg}\n${stack}`); if (!res.headersSent) { res.writeHead(500, { "Content-Type": "application/json" }); res.end(JSON.stringify({ error: { message: msg, type: "server_error" } })); diff --git a/src/stream-collapse.ts b/src/stream-collapse.ts index b9a4d292..5061450f 100644 --- a/src/stream-collapse.ts +++ b/src/stream-collapse.ts @@ -12,6 +12,85 @@ import type { FixtureBlock, RecordProviderKey, ToolCall } from "./types.js"; import type { Logger } from "./logger.js"; import { isHarmonyContent, parseHarmonyContent } from "./harmony.js"; +// --------------------------------------------------------------------------- +// Accumulated-string cap (RangeError: Invalid string length guard) +// --------------------------------------------------------------------------- + +/** + * Hard ceiling (UTF-16 code units) for any SINGLE string a collapser + * accumulates across deltas (`content`, `reasoning`, a tool call's + * `arguments`, `audioB64`, `argsStr`, and the coalesced text of `orderAtoms`). + * + * V8's maximum string length on 64-bit is 2^29 - 1 (~536.8M UTF-16 code + * units); appending past it throws `RangeError: Invalid string length`. The + * proxy path bounds the raw BYTE buffer under `PROXY_BUFFER_HARD_CEILING` + * (256 MiB) before collapse, but the collapser is ALSO reachable directly + * (exported from index.ts) and — even on the proxy path — a single accumulator + * can approach the byte budget in code units, so it needs its own guard rather + * than relying on the byte cap alone. 256Mi code units is comfortably below + * V8's limit while leaving huge headroom for any real completion (whose + * accumulated content can never exceed the total streamed body). + * + * When an append would cross the ceiling the accumulator keeps the ceiling's + * worth of characters, drops the overflow, and marks itself truncated — never + * throwing — mirroring the proxy buffer's "bound then mark truncated, never + * fail the client" discipline. + */ +export const MAX_COLLAPSE_STRING_LENGTH = 256 * 1024 * 1024; // 256 Mi UTF-16 code units + +/** + * Test-only override of {@link MAX_COLLAPSE_STRING_LENGTH}. Lets the cap suite + * exercise the truncation path with a tiny budget instead of building a + * multi-hundred-MB string. `undefined` (the default) uses the real ceiling. + * NEVER set from production code. + */ +let collapseStringLimitOverride: number | undefined; + +/** @internal test-only — see {@link collapseStringLimitOverride}. */ +export function setCollapseStringLimitForTests(value: number | undefined): void { + collapseStringLimitOverride = value; +} + +/** Effective accumulated-string ceiling, honoring any active test-only override. */ +function effectiveCollapseStringLimit(): number { + return collapseStringLimitOverride ?? MAX_COLLAPSE_STRING_LENGTH; +} + +/** + * Guard a raw collapse-input body against V8's max string length BEFORE any + * per-delta accumulation begins. Every collapser accumulates `content` / + * `reasoning` / a tool call's `arguments` / `audioB64` / `argsStr` (and the + * coalesced text of `orderAtoms`) from fragments of `body`; the sum of any one + * of those accumulators can never exceed the total input length, so bounding + * the INPUT under the ceiling bounds EVERY accumulator at once — no per-`+=` + * site can then build a string past the limit and throw + * `RangeError: Invalid string length` (the ~1/sec prod crash). + * + * Returns the body UNCHANGED when it is within the ceiling. When it exceeds the + * ceiling this returns a hard-truncated prefix (never throwing) — the collapse + * result the caller builds from it is marked incomplete via `truncated: true` + * (see {@link isCollapseInputTruncated}). This mirrors the proxy buffer's + * "bound then mark truncated, never fail the client" discipline: on the proxy + * path a body this large has already tripped the byte cap and skipped recording, + * so this is a defense-in-depth backstop that also covers the direct (exported) + * collapse callers where no byte cap ran. + */ +function guardCollapseBody(body: string): string { + const limit = effectiveCollapseStringLimit(); + if (body.length <= limit) return body; + return body.slice(0, limit); +} + +/** + * True when a body exceeded the accumulated-string ceiling and had to be + * truncated by {@link guardCollapseBody}. Collapsers stamp `truncated: true` on + * their result in this case so the recorder skips journaling a partial fixture, + * exactly like a transport-truncated stream. + */ +function isCollapseInputTruncated(body: string): boolean { + return body.length > effectiveCollapseStringLimit(); +} + // --------------------------------------------------------------------------- // Result type shared by all collapse functions // --------------------------------------------------------------------------- @@ -277,7 +356,9 @@ function extractSSEData(lines: string[]): string | undefined { * data: {"id":"chatcmpl-123","choices":[{"delta":{"content":"Hello"}}]}\n\n * data: [DONE]\n\n */ -export function collapseOpenAISSE(body: string): CollapseResult { +export function collapseOpenAISSE(rawBody: string): CollapseResult { + const inputTruncated = isCollapseInputTruncated(rawBody); + const body = guardCollapseBody(rawBody); const lines = splitSSEEvents(body); let content = ""; let reasoning = ""; @@ -487,6 +568,7 @@ export function collapseOpenAISSE(body: string): CollapseResult { ...(firstDroppedSample ? { firstDroppedSample } : {}), ...(harmonyUnparsed ? { harmonyUnparsed: true } : {}), ...(harmonyNote ? { harmonyNote } : {}), + ...(inputTruncated ? { truncated: true } : {}), }; } @@ -498,6 +580,7 @@ export function collapseOpenAISSE(body: string): CollapseResult { ...(firstDroppedSample ? { firstDroppedSample } : {}), ...(harmonyUnparsed ? { harmonyUnparsed: true } : {}), ...(harmonyNote ? { harmonyNote } : {}), + ...(inputTruncated ? { truncated: true } : {}), }; } @@ -512,7 +595,9 @@ export function collapseOpenAISSE(body: string): CollapseResult { * event: message_start\ndata: {...}\n\n * event: content_block_delta\ndata: {"delta":{"type":"text_delta","text":"Hello"}}\n\n */ -export function collapseAnthropicSSE(body: string): CollapseResult { +export function collapseAnthropicSSE(rawBody: string): CollapseResult { + const inputTruncated = isCollapseInputTruncated(rawBody); + const body = guardCollapseBody(rawBody); const blocks = splitSSEEvents(body); let content = ""; let reasoning = ""; @@ -679,6 +764,7 @@ export function collapseAnthropicSSE(body: string): CollapseResult { ...(redactedThinking.length > 0 ? { redactedThinking } : {}), ...(droppedChunks > 0 ? { droppedChunks } : {}), ...(firstDroppedSample ? { firstDroppedSample } : {}), + ...(inputTruncated ? { truncated: true } : {}), }; } @@ -689,6 +775,7 @@ export function collapseAnthropicSSE(body: string): CollapseResult { ...(redactedThinking.length > 0 ? { redactedThinking } : {}), ...(droppedChunks > 0 ? { droppedChunks } : {}), ...(firstDroppedSample ? { firstDroppedSample } : {}), + ...(inputTruncated ? { truncated: true } : {}), }; } @@ -702,7 +789,9 @@ export function collapseAnthropicSSE(body: string): CollapseResult { * Format (data-only, no event prefix, no [DONE]): * data: {"candidates":[{"content":{"parts":[{"text":"Hello"}]}}]}\n\n */ -export function collapseGeminiSSE(body: string): CollapseResult { +export function collapseGeminiSSE(rawBody: string): CollapseResult { + const inputTruncated = isCollapseInputTruncated(rawBody); + const body = guardCollapseBody(rawBody); const lines = splitSSEEvents(body); let content = ""; let reasoning = ""; @@ -809,6 +898,7 @@ export function collapseGeminiSSE(body: string): CollapseResult { ...(normalizedToolCalls.length > 0 ? { toolCalls: normalizedToolCalls } : {}), ...(droppedChunks > 0 ? { droppedChunks } : {}), ...(firstDroppedSample ? { firstDroppedSample } : {}), + ...(inputTruncated ? { truncated: true } : {}), }; } @@ -821,6 +911,7 @@ export function collapseGeminiSSE(body: string): CollapseResult { ...(reasoning ? { reasoning } : {}), ...(droppedChunks > 0 ? { droppedChunks } : {}), ...(firstDroppedSample ? { firstDroppedSample } : {}), + ...(inputTruncated ? { truncated: true } : {}), }; } @@ -829,6 +920,7 @@ export function collapseGeminiSSE(body: string): CollapseResult { ...(reasoning ? { reasoning } : {}), ...(droppedChunks > 0 ? { droppedChunks } : {}), ...(firstDroppedSample ? { firstDroppedSample } : {}), + ...(inputTruncated ? { truncated: true } : {}), }; } @@ -850,7 +942,9 @@ export function collapseGeminiSSE(body: string): CollapseResult { * content is run through the same fail-safe {@link parseHarmonyContent} gate to * capture structured tool calls / reasoning instead of leaking raw tokens. */ -export function collapseOllamaNDJSON(body: string): CollapseResult { +export function collapseOllamaNDJSON(rawBody: string): CollapseResult { + const inputTruncated = isCollapseInputTruncated(rawBody); + const body = guardCollapseBody(rawBody); const lines = body.split("\n").filter((l) => l.trim().length > 0); let content = ""; let reasoning = ""; @@ -954,6 +1048,7 @@ export function collapseOllamaNDJSON(body: string): CollapseResult { ...(firstDroppedSample ? { firstDroppedSample } : {}), ...(harmonyUnparsed ? { harmonyUnparsed: true } : {}), ...(harmonyNote ? { harmonyNote } : {}), + ...(inputTruncated ? { truncated: true } : {}), }; } @@ -964,6 +1059,7 @@ export function collapseOllamaNDJSON(body: string): CollapseResult { ...(firstDroppedSample ? { firstDroppedSample } : {}), ...(harmonyUnparsed ? { harmonyUnparsed: true } : {}), ...(harmonyNote ? { harmonyNote } : {}), + ...(inputTruncated ? { truncated: true } : {}), }; } @@ -977,7 +1073,9 @@ export function collapseOllamaNDJSON(body: string): CollapseResult { * Format: * event: content-delta\ndata: {"type":"content-delta","delta":{"message":{"content":{"text":"Hello"}}}}\n\n */ -export function collapseCohereSSE(body: string): CollapseResult { +export function collapseCohereSSE(rawBody: string): CollapseResult { + const inputTruncated = isCollapseInputTruncated(rawBody); + const body = guardCollapseBody(rawBody); const blocks = splitSSEEvents(body); let content = ""; // Reasoning text assembled from `thinking` content-delta blocks. Cohere's @@ -1129,6 +1227,7 @@ export function collapseCohereSSE(body: string): CollapseResult { ...(reasoning ? { reasoning } : {}), ...(droppedChunks > 0 ? { droppedChunks } : {}), ...(firstDroppedSample ? { firstDroppedSample } : {}), + ...(inputTruncated ? { truncated: true } : {}), }; } @@ -1137,6 +1236,7 @@ export function collapseCohereSSE(body: string): CollapseResult { ...(reasoning ? { reasoning } : {}), ...(droppedChunks > 0 ? { droppedChunks } : {}), ...(firstDroppedSample ? { firstDroppedSample } : {}), + ...(inputTruncated ? { truncated: true } : {}), }; } @@ -1254,8 +1354,18 @@ function decodeEventStreamFrames(buf: Buffer): { * Each frame contains a JSON payload with event types like: * contentBlockDelta, contentBlockStart, etc. */ -export function collapseBedrockEventStream(body: Buffer): CollapseResult { - const { frames, truncated } = decodeEventStreamFrames(body); +export function collapseBedrockEventStream(rawBody: Buffer): CollapseResult { + // Bound the input under V8's max string length before decoding: `content` / + // `reasoning` / a tool call's `arguments` accumulate across frames and can + // never exceed the total decoded bytes, so capping the input buffer bounds + // every accumulator and prevents `RangeError: Invalid string length`. An + // over-ceiling buffer is truncated (a partial trailing frame simply fails CRC + // and stops parsing) and the result is marked `truncated`. + const limit = effectiveCollapseStringLimit(); + const inputTruncated = rawBody.byteLength > limit; + const body = inputTruncated ? rawBody.subarray(0, limit) : rawBody; + const { frames, truncated: frameTruncated } = decodeEventStreamFrames(body); + const truncated = frameTruncated || inputTruncated; let content = ""; // Reasoning text assembled from Converse `reasoningContent.text` deltas. The // Bedrock Converse stream interleaves a `delta.reasoningContent` block carrying @@ -1498,7 +1608,9 @@ export function collapseBedrockEventStream(body: Buffer): CollapseResult { * delta) are still accepted for backward compatibility with previously * recorded fixtures. */ -export function collapseGeminiInteractionsSSE(body: string): CollapseResult { +export function collapseGeminiInteractionsSSE(rawBody: string): CollapseResult { + const inputTruncated = isCollapseInputTruncated(rawBody); + const body = guardCollapseBody(rawBody); const lines = splitSSEEvents(body); let content = ""; let reasoning = ""; @@ -1658,6 +1770,7 @@ export function collapseGeminiInteractionsSSE(body: string): CollapseResult { ...(reasoning ? { reasoning } : {}), ...(droppedChunks > 0 ? { droppedChunks } : {}), ...(firstDroppedSample ? { firstDroppedSample } : {}), + ...(inputTruncated ? { truncated: true } : {}), }; } @@ -1666,6 +1779,7 @@ export function collapseGeminiInteractionsSSE(body: string): CollapseResult { ...(reasoning ? { reasoning } : {}), ...(droppedChunks > 0 ? { droppedChunks } : {}), ...(firstDroppedSample ? { firstDroppedSample } : {}), + ...(inputTruncated ? { truncated: true } : {}), }; } diff --git a/src/types.ts b/src/types.ts index ab8f88ca..51301447 100644 --- a/src/types.ts +++ b/src/types.ts @@ -92,6 +92,14 @@ export interface FixtureMatch { systemMessage?: string | string[] | RegExp; inputText?: string | RegExp; toolCallId?: string; + /** + * Substring matched against the text content of the LAST message when that + * message is a `tool` result (same last-message rule as `toolCallId`). + * Discriminates fixtures whose requests differ ONLY inside the tool-result + * payload — e.g. a human-in-the-loop tool where approve and cancel resume + * with the same toolCallId but different result JSON. + */ + toolResultContains?: string; toolName?: string; model?: string | RegExp; responseFormat?: string; @@ -541,6 +549,11 @@ export interface FixtureFileEntry { systemMessage?: string | string[]; inputText?: string; toolCallId?: string; + /** + * Substring matched against the last tool-result message's text content. + * Mirrors the runtime FixtureMatch.toolResultContains. + */ + toolResultContains?: string; toolName?: string; model?: string; responseFormat?: string; @@ -694,6 +707,19 @@ export type RecordProviderKey = export interface RecordConfig { providers: Partial>; + /** + * aimock's own built-in upstream API keys, keyed by provider. Opt-in and + * backward-compatible: when set for a provider, aimock injects the key on a + * fixture-miss passthrough IF the caller sent no credential or a dummy + * placeholder (see `applyProviderAuth`); a real caller key always overrides. + * Only static-key providers are eligible: OpenAI/OpenRouter/Cohere/Grok/Ollama + * (bearer), Anthropic (x-api-key), Gemini/Gemini-Interactions/Veo + * (x-goog-api-key), Azure OpenAI (api-key), ElevenLabs (xi-api-key), and fal + * (Authorization: Key). Signed/OAuth providers (Bedrock SigV4, Vertex AI + * OAuth) are never rewritten. Sourced from AIMOCK_PROVIDER_*_KEY env vars by + * the CLI (secrets stay out of `ps`). + */ + providerKeys?: Partial>; fixturePath?: string; /** Proxy unmatched requests without saving fixtures or caching in memory. */ proxyOnly?: boolean;