fix(showcase): add Railway image-ref drift assertion to catch atest corruption (CopilotKit#4118)

## Incident

On 2026-04-21, 18 production Railway showcase services were found with
malformed image refs of the form
`ghcr.io/copilotkit/showcase-<slug>atest` — the `:` before `latest` was
dropped, so Docker treated `...atest` as the tag. Railway dutifully
tried to pull an image that doesn't exist on GHCR, so every affected
service began failing to redeploy.

The underlying data has been cleaned up, but the corruption did NOT come
from committed code — it was introduced by an out-of-band MCP/manual API
mutation. Nothing in the repo would have caught this, and nothing
prevents it from happening again.

## Current state

- 41 showcase services now have correct image refs (verified live
against Railway in this PR's local run).
- No source-controlled guardrail exists today to catch future drift.

## New assertion

`showcase/scripts/verify-railway-image-refs.ts` — queries Railway
GraphQL for every service in the CopilotKit Showcase project
(`6f8c6bff-...`) and asserts each service's image source matches:

```
ghcr.io/copilotkit/<service-name>:latest
```

The regex `^ghcr\.io\/copilotkit\/showcase-[a-z0-9-]+:latest$` covers
every service class — `showcase-<slug>`, `showcase-starter-<slug>`,
`showcase-aimock`, `showcase-pocketbase`, `showcase-ops`. The script
additionally requires the image's trailing path segment to match the
Railway service name exactly (identity invariant: service name ↔ image
name).

On violation the script prints the service name, current image, expected
shape, and reason — then exits 1. On success: `✓ N services verified`,
exit 0.

## Wire-up

Added a `verify-image-refs` job to
`.github/workflows/showcase_deploy.yml` that runs in parallel with
`check-lockfile` and before the deploy matrix. If drift is detected, the
workflow aborts before any build starts. The `notify` job classifies a
verify failure distinctly so Slack alerts say "Railway image-ref drift
detected" rather than a generic pre-build failure.

Uses the existing `RAILWAY_TOKEN` secret already plumbed into the deploy
step — no new secrets needed.

## Local verification

Normal run (all 41 services correctly configured):
```
✓ 41 services verified
```

Simulated-corruption cases (unit-tested the exported `validateImage`):
- `showcase-mastra` → `ghcr.io/copilotkit/showcase-mastraatest` →
**rejected** (the exact incident shape)
- `showcase-mastra` → `ghcr.io/copilotkit/showcase-crewai-crews:latest`
→ rejected (service↔image mismatch)
- `showcase-mastra` → `ghcr.io/copilotkit/showcase-mastra` → rejected
(no tag)
- `showcase-mastra` → `docker.io/copilotkit/showcase-mastra:latest` →
rejected (wrong registry)
- `showcase-mastra` → `ghcr.io/copilotkit/showcase-mastra:v1` → rejected
(wrong tag)
- `showcase-mastra` → `null` → rejected (no source)
- Plus 3 valid cases (`showcase-mastra`, `showcase-starter-agno`,
`showcase-aimock` all at `:latest`) → accepted.

All 9/9 behave as expected.

## Test plan

- [x] `npx tsx showcase/scripts/verify-railway-image-refs.ts` against
current Railway state → `✓ 41 services verified`
- [x] Simulated-corruption validation → `9/9` rejection/acceptance
behaviour correct
- [x] `tsc --noEmit` clean on `showcase/scripts`
- [x] `prettier --check` passes on both modified files
- [x] `pnpm --filter=@copilotkit/showcase-scripts test` → 21 files /
1079 tests pass (no regressions from pre-existing suite)
- [x] YAML parse of `showcase_deploy.yml` succeeds
- [ ] CI workflow run on this PR includes the new `verify-image-refs`
job and it goes green against current Railway state

Author: jpr5

.github/workflows/showcase_deploy.yml

@@ -218,8 +218,30 @@ jobs:
           node-version: 22.x
       - run: pnpm install --frozen-lockfile --ignore-scripts
 
+  # Drift assertion — fails the workflow BEFORE the build matrix fans out
+  # if any Railway showcase service has a malformed image ref. On 2026-04-21
+  # 18 services were found with `ghcr.io/copilotkit/showcase-<slug>atest`
+  # (missing `:` before `latest`) after an out-of-band API mutation; since
+  # no committed code touched those refs, only a live-Railway assertion can
+  # catch that class of corruption. Runs unconditionally on every deploy
+  # run so a drifted env can never silently ship.
+  verify-image-refs:
+    needs: [detect-changes]
+    if: needs.detect-changes.outputs.has_changes == 'true'
+    runs-on: ubuntu-latest
+    timeout-minutes: 3
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: 22.x
+      - name: Verify Railway image refs
+        env:
+          RAILWAY_TOKEN: ${{ secrets.RAILWAY_TOKEN }}
+        run: npx tsx showcase/scripts/verify-railway-image-refs.ts
+
   build:
-    needs: [detect-changes, check-lockfile]
+    needs: [detect-changes, check-lockfile, verify-image-refs]
     if: needs.detect-changes.outputs.has_changes == 'true'
     runs-on: depot-ubuntu-24.04-4
     timeout-minutes: ${{ matrix.service.timeout }}
@@ -465,7 +487,7 @@ jobs:
           exit 1
 
   notify:
-    needs: [detect-changes, check-lockfile, build]
+    needs: [detect-changes, check-lockfile, verify-image-refs, build]
     if: always()
     permissions:
       contents: read
@@ -637,6 +659,7 @@ jobs:
           URL="${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
           DETECT="${{ needs.detect-changes.result }}"
           LOCKFILE="${{ needs.check-lockfile.result }}"
+          VERIFY="${{ needs.verify-image-refs.result }}"
           BUILD="${{ needs.build.result }}"
           COUNT="${{ steps.summary.outputs.count }}"
           NOW="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
@@ -704,7 +727,7 @@ jobs:
           # has happened yet. Safe to stay silent — any newer run (or the
           # manual re-trigger) will redo all work from scratch. Don't touch
           # state either way — indeterminate outcome.
-          if [ "$DETECT" = "cancelled" ] || [ "$LOCKFILE" = "cancelled" ]; then
+          if [ "$DETECT" = "cancelled" ] || [ "$LOCKFILE" = "cancelled" ] || [ "$VERIFY" = "cancelled" ]; then
             echo "Pre-build stage cancelled — newer run will redo all work, skipping Slack notification"
             echo "should_post=false" >> "$GITHUB_OUTPUT"
             echo "update_state=false" >> "$GITHUB_OUTPUT"
@@ -728,7 +751,14 @@ jobs:
           fi
 
           # Classify the terminal outcome.
-          if [ "$DETECT" = "failure" ] || [ "$LOCKFILE" = "failure" ]; then
+          if [ "$VERIFY" = "failure" ]; then
+            # Drift assertion fired — one or more Railway services have a
+            # malformed image ref. This is actionable: check the run log
+            # for the per-service violation list (service name, current
+            # image, expected shape).
+            OUTCOME="failure"
+            MSG=":x: *Showcase deploy*: FAILED — Railway image-ref drift detected (inspect run log for affected services)"
+          elif [ "$DETECT" = "failure" ] || [ "$LOCKFILE" = "failure" ]; then
             OUTCOME="failure"
             MSG=":x: *Showcase deploy*: FAILED (pre-build check)"
           elif [ "$BUILD" = "success" ]; then

showcase/scripts/verify-railway-image-refs.ts

@@ -0,0 +1,208 @@
+#!/usr/bin/env npx tsx
+/**
+ * verify-railway-image-refs.ts — Drift assertion for Railway showcase image refs.
+ *
+ * Fetches every service in the CopilotKit Showcase project and validates that
+ * the configured Docker image reference matches the canonical GHCR form:
+ *   ghcr.io/copilotkit/<service-name>:latest
+ *
+ * Backstory: on 2026-04-21, 18 production services were found with malformed
+ * image refs of the form `ghcr.io/copilotkit/showcase-<slug>atest` (missing
+ * the `:` before `latest`, so Docker treats `...atest` as the tag). The root
+ * cause was an out-of-band MCP/manual mutation — no committed code touched
+ * these refs. This script exists so any future corruption, regardless of
+ * source, fails loudly and early in CI before a bad deploy goes out.
+ *
+ * Usage:
+ *   npx tsx showcase/scripts/verify-railway-image-refs.ts
+ *
+ * Requires: RAILWAY_TOKEN env var or ~/.railway/config.json
+ * Exit: 0 when every service matches the canonical shape, 1 on any violation.
+ */
+
+import fs from "fs";
+import path from "path";
+import { fileURLToPath } from "url";
+
+const RAILWAY_API = "https://backboard.railway.com/graphql/v2";
+
+const SHOWCASE = {
+  projectId: "6f8c6bff-a80d-4f8f-b78d-50b32bcf4479",
+  environmentId: "b14919f4-6417-429f-848d-c6ae2201e04f",
+};
+
+// Canonical shape: ghcr.io/copilotkit/<name>:latest where <name> is the
+// service name itself. This single pattern covers showcase-<slug>,
+// showcase-starter-<slug>, showcase-aimock, showcase-pocketbase,
+// showcase-ops, and any future showcase-* service. Enforcing identity
+// between Railway service name and image name (modulo the ghcr.io/copilotkit/
+// prefix and :latest tag) is the invariant — if these ever drift apart we
+// want to know.
+const IMAGE_SHAPE = /^ghcr\.io\/copilotkit\/showcase-[a-z0-9-]+:latest$/;
+
+function getToken(): string {
+  if (process.env.RAILWAY_TOKEN) return process.env.RAILWAY_TOKEN;
+  const configPath = path.join(
+    process.env.HOME || "~",
+    ".railway",
+    "config.json",
+  );
+  if (fs.existsSync(configPath)) {
+    const config = JSON.parse(fs.readFileSync(configPath, "utf-8"));
+    if (config?.user?.token) return config.user.token;
+  }
+  console.error(
+    "No Railway token found. Set RAILWAY_TOKEN or run `railway login`.",
+  );
+  process.exit(1);
+}
+
+async function railwayGql<T = unknown>(
+  query: string,
+  variables: Record<string, unknown> = {},
+): Promise<T> {
+  const token = getToken();
+  const res = await fetch(RAILWAY_API, {
+    method: "POST",
+    headers: {
+      Authorization: `Bearer ${token}`,
+      "Content-Type": "application/json",
+    },
+    body: JSON.stringify({ query, variables }),
+  });
+  if (!res.ok) {
+    throw new Error(`Railway API error: ${res.status} ${await res.text()}`);
+  }
+  const json = (await res.json()) as {
+    data?: T;
+    errors?: Array<{ message: string }>;
+  };
+  if (json.errors?.length) {
+    throw new Error(
+      `Railway GraphQL errors:\n${json.errors.map((e) => `  - ${e.message}`).join("\n")}`,
+    );
+  }
+  return json.data as T;
+}
+
+interface ProjectServicesWithInstances {
+  project: {
+    services: {
+      edges: Array<{
+        node: {
+          id: string;
+          name: string;
+          serviceInstances: {
+            edges: Array<{
+              node: {
+                environmentId: string;
+                source: { image: string | null } | null;
+              };
+            }>;
+          };
+        };
+      }>;
+    };
+  };
+}
+
+interface Violation {
+  service: string;
+  image: string | null;
+  reason: string;
+}
+
+export function validateImage(
+  serviceName: string,
+  image: string | null,
+): Violation | null {
+  if (!image) {
+    return {
+      service: serviceName,
+      image,
+      reason:
+        "no image source configured (expected a Docker image, not a repo)",
+    };
+  }
+  if (!IMAGE_SHAPE.test(image)) {
+    return {
+      service: serviceName,
+      image,
+      reason: `does not match canonical shape ^ghcr\\.io/copilotkit/showcase-[a-z0-9-]+:latest$`,
+    };
+  }
+  const expected = `ghcr.io/copilotkit/${serviceName}:latest`;
+  if (image !== expected) {
+    return {
+      service: serviceName,
+      image,
+      reason: `image name mismatches service name (expected exactly ${expected})`,
+    };
+  }
+  return null;
+}
+
+async function main(): Promise<void> {
+  const data = await railwayGql<ProjectServicesWithInstances>(
+    `query project($id: String!) {
+      project(id: $id) {
+        services {
+          edges { node {
+            id
+            name
+            serviceInstances {
+              edges { node { environmentId source { image } } }
+            }
+          } }
+        }
+      }
+    }`,
+    { id: SHOWCASE.projectId },
+  );
+
+  const services = data.project.services.edges
+    .map((e) => e.node)
+    .filter((s) => s.name.startsWith("showcase-"));
+
+  const violations: Violation[] = [];
+  let checked = 0;
+  for (const svc of services) {
+    const instance = svc.serviceInstances.edges.find(
+      (e) => e.node.environmentId === SHOWCASE.environmentId,
+    );
+    const image = instance?.node.source?.image ?? null;
+    checked++;
+    const v = validateImage(svc.name, image);
+    if (v) violations.push(v);
+  }
+
+  if (violations.length > 0) {
+    console.error(
+      `\n✗ Railway image-ref drift detected (${violations.length}/${checked} services)\n`,
+    );
+    console.error(
+      `Expected shape: ghcr.io/copilotkit/<service-name>:latest` +
+        ` (note the ':' before 'latest')\n`,
+    );
+    for (const v of violations) {
+      console.error(`  ${v.service}`);
+      console.error(`    current:  ${v.image ?? "<unset>"}`);
+      console.error(`    expected: ghcr.io/copilotkit/${v.service}:latest`);
+      console.error(`    reason:   ${v.reason}`);
+    }
+    console.error(
+      `\nFix via Railway dashboard or the showcase deploy-to-railway script.` +
+        ` A common past cause was ':' dropped from ':latest' by an out-of-band API mutation.\n`,
+    );
+    process.exit(1);
+  }
+
+  console.log(`✓ ${checked} services verified`);
+}
+
+if (process.argv[1] === fileURLToPath(import.meta.url)) {
+  main().catch((e) => {
+    console.error(e);
+    process.exit(1);
+  });
+}