|
| 1 | +name: "Showcase: Drift Detection" |
| 2 | + |
| 3 | +on: |
| 4 | + schedule: |
| 5 | + - cron: "0 3 * * *" # Nightly E2E |
| 6 | + - cron: "0 9 * * 1" # Weekly version drift (Monday 9am UTC) |
| 7 | + workflow_dispatch: |
| 8 | + inputs: |
| 9 | + run_version_check: |
| 10 | + description: "Run version drift check" |
| 11 | + type: boolean |
| 12 | + default: false |
| 13 | + |
| 14 | +jobs: |
| 15 | + drift-detection: |
| 16 | + name: E2E Drift Detection |
| 17 | + runs-on: ubuntu-latest |
| 18 | + timeout-minutes: 30 |
| 19 | + strategy: |
| 20 | + fail-fast: false |
| 21 | + matrix: |
| 22 | + include: |
| 23 | + - slug: langgraph-python |
| 24 | + name: "LangGraph (Python)" |
| 25 | + url: https://showcase-langgraph-python-production.up.railway.app |
| 26 | + - slug: mastra |
| 27 | + name: "Mastra" |
| 28 | + url: https://showcase-mastra-production.up.railway.app |
| 29 | + - slug: test-integration-tmp |
| 30 | + name: "Test Integration" |
| 31 | + url: https://showcase-test-integration-tmp-production.up.railway.app |
| 32 | + - slug: crewai-crews |
| 33 | + name: "CrewAI (Crews)" |
| 34 | + url: https://showcase-crewai-crews-production.up.railway.app |
| 35 | + - slug: pydantic-ai |
| 36 | + name: "PydanticAI" |
| 37 | + url: https://showcase-pydantic-ai-production.up.railway.app |
| 38 | + - slug: google-adk |
| 39 | + name: "Google ADK" |
| 40 | + url: https://showcase-google-adk-production.up.railway.app |
| 41 | + - slug: ag2 |
| 42 | + name: "AG2" |
| 43 | + url: https://showcase-ag2-production.up.railway.app |
| 44 | + - slug: agno |
| 45 | + name: "Agno" |
| 46 | + url: https://showcase-agno-production.up.railway.app |
| 47 | + - slug: llamaindex |
| 48 | + name: "LlamaIndex" |
| 49 | + url: https://showcase-llamaindex-production.up.railway.app |
| 50 | + - slug: langgraph-fastapi |
| 51 | + name: "LangGraph (FastAPI)" |
| 52 | + url: https://showcase-langgraph-fastapi-production.up.railway.app |
| 53 | + - slug: langgraph-typescript |
| 54 | + name: "LangGraph (TypeScript)" |
| 55 | + url: https://showcase-langgraph-typescript-production.up.railway.app |
| 56 | + - slug: langroid |
| 57 | + name: "Langroid" |
| 58 | + url: https://showcase-langroid-production.up.railway.app |
| 59 | + - slug: spring-ai |
| 60 | + name: "Spring AI" |
| 61 | + url: https://showcase-spring-ai-production.up.railway.app |
| 62 | + - slug: strands |
| 63 | + name: "AWS Strands" |
| 64 | + url: https://showcase-strands-production.up.railway.app |
| 65 | + - slug: ms-agent-python |
| 66 | + name: "MS Agent Framework (Python)" |
| 67 | + url: https://showcase-ms-agent-python-production.up.railway.app |
| 68 | + - slug: claude-sdk-typescript |
| 69 | + name: "Claude Agent SDK (TypeScript)" |
| 70 | + url: https://showcase-claude-sdk-typescript-production.up.railway.app |
| 71 | + - slug: ms-agent-dotnet |
| 72 | + name: "MS Agent Framework (.NET)" |
| 73 | + url: https://showcase-ms-agent-dotnet-production.up.railway.app |
| 74 | + - slug: claude-sdk-python |
| 75 | + name: "Claude Agent SDK (Python)" |
| 76 | + url: https://showcase-claude-sdk-python-production.up.railway.app |
| 77 | + |
| 78 | + steps: |
| 79 | + - name: Checkout |
| 80 | + uses: actions/checkout@v4 |
| 81 | + |
| 82 | + - name: Setup Node.js |
| 83 | + uses: actions/setup-node@v4 |
| 84 | + with: |
| 85 | + node-version: 20 |
| 86 | + |
| 87 | + - name: Setup pnpm |
| 88 | + uses: pnpm/action-setup@v4 |
| 89 | + |
| 90 | + - name: Install dependencies |
| 91 | + working-directory: showcase/packages/${{ matrix.slug }} |
| 92 | + run: pnpm install --ignore-scripts || true |
| 93 | + |
| 94 | + - name: Install Playwright |
| 95 | + working-directory: showcase/packages/${{ matrix.slug }} |
| 96 | + run: npx playwright install chromium |
| 97 | + |
| 98 | + - name: Check health endpoint |
| 99 | + run: | |
| 100 | + status=$(curl -sf "${{ matrix.url }}/api/health" | jq -r '.status' 2>/dev/null || echo "down") |
| 101 | + echo "Health status: $status" |
| 102 | + if [ "$status" != "ok" ]; then |
| 103 | + echo "::warning::${{ matrix.name }} health check failed (status: $status)" |
| 104 | + fi |
| 105 | +
|
| 106 | + - name: Run E2E tests |
| 107 | + working-directory: showcase/packages/${{ matrix.slug }} |
| 108 | + env: |
| 109 | + BASE_URL: ${{ matrix.url }} |
| 110 | + CI: true |
| 111 | + run: npx playwright test --reporter=github |
| 112 | + |
| 113 | + - name: Create issue on failure |
| 114 | + if: failure() |
| 115 | + uses: actions/github-script@v7 |
| 116 | + with: |
| 117 | + script: | |
| 118 | + const title = `[Drift] ${{ matrix.name }} E2E tests failing`; |
| 119 | + const body = `## Drift Detection Alert |
| 120 | +
|
| 121 | + **Integration**: ${{ matrix.name }} (\`${{ matrix.slug }}\`) |
| 122 | + **URL**: ${{ matrix.url }} |
| 123 | + **Run**: ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId} |
| 124 | +
|
| 125 | + E2E tests are failing against the deployed integration. This may indicate: |
| 126 | + - A CopilotKit SDK update broke compatibility |
| 127 | + - The agent backend is down or misconfigured |
| 128 | + - A dependency update caused a regression |
| 129 | +
|
| 130 | + Please investigate and fix. |
| 131 | + `; |
| 132 | +
|
| 133 | + // Check if an issue already exists |
| 134 | + const { data: issues } = await github.rest.issues.listForRepo({ |
| 135 | + owner: context.repo.owner, |
| 136 | + repo: context.repo.repo, |
| 137 | + labels: 'showcase-drift', |
| 138 | + state: 'open', |
| 139 | + }); |
| 140 | +
|
| 141 | + const existing = issues.find(i => i.title === title); |
| 142 | + if (!existing) { |
| 143 | + await github.rest.issues.create({ |
| 144 | + owner: context.repo.owner, |
| 145 | + repo: context.repo.repo, |
| 146 | + title, |
| 147 | + body, |
| 148 | + labels: ['showcase-drift', '${{ matrix.slug }}'], |
| 149 | + }); |
| 150 | + } |
| 151 | +
|
| 152 | + version-drift: |
| 153 | + name: Version Drift Report |
| 154 | + runs-on: ubuntu-latest |
| 155 | + # Only on weekly Monday schedule or manual trigger |
| 156 | + if: github.event.schedule == '0 9 * * 1' || github.event.inputs.run_version_check == 'true' |
| 157 | + steps: |
| 158 | + - uses: actions/checkout@v4 |
| 159 | + |
| 160 | + - uses: actions/setup-node@v4 |
| 161 | + with: |
| 162 | + node-version: 20 |
| 163 | + |
| 164 | + - uses: actions/setup-python@v5 |
| 165 | + with: |
| 166 | + python-version: "3.12" |
| 167 | + |
| 168 | + - name: Check Python package drift |
| 169 | + id: python_drift |
| 170 | + run: | |
| 171 | + report="" |
| 172 | + for pkg_dir in showcase/packages/*/; do |
| 173 | + req_file="$pkg_dir/requirements.txt" |
| 174 | + [ -f "$req_file" ] || continue |
| 175 | + slug=$(basename "$pkg_dir") |
| 176 | + echo "=== $slug ===" |
| 177 | + while IFS= read -r line; do |
| 178 | + # Skip empty lines and comments |
| 179 | + [[ -z "$line" || "$line" =~ ^# ]] && continue |
| 180 | + # Extract package name and pinned version |
| 181 | + if [[ "$line" =~ ^([a-zA-Z0-9_-]+)\[?[a-z]*\]?==([0-9.]+) ]]; then |
| 182 | + pkg="${BASH_REMATCH[1]}" |
| 183 | + pinned="${BASH_REMATCH[2]}" |
| 184 | + latest=$(pip index versions "$pkg" 2>/dev/null | head -1 | grep -oP '\([\d.]+\)' | tr -d '()' || echo "unknown") |
| 185 | + if [ "$latest" != "unknown" ] && [ "$latest" != "$pinned" ]; then |
| 186 | + echo " $pkg: pinned=$pinned latest=$latest" |
| 187 | + report+="| $slug | $pkg | $pinned | $latest |\n" |
| 188 | + fi |
| 189 | + fi |
| 190 | + done < "$req_file" |
| 191 | + done |
| 192 | + echo "report<<EOF" >> $GITHUB_OUTPUT |
| 193 | + echo -e "$report" >> $GITHUB_OUTPUT |
| 194 | + echo "EOF" >> $GITHUB_OUTPUT |
| 195 | +
|
| 196 | + - name: Check npm package drift |
| 197 | + id: npm_drift |
| 198 | + run: | |
| 199 | + report="" |
| 200 | + for pkg_dir in showcase/packages/*/; do |
| 201 | + pkg_json="$pkg_dir/package.json" |
| 202 | + [ -f "$pkg_json" ] || continue |
| 203 | + slug=$(basename "$pkg_dir") |
| 204 | + echo "=== $slug ===" |
| 205 | + # Check key deps: @copilotkit/*, @mastra/*, @ag-ui/* |
| 206 | + node -e " |
| 207 | + const pkg = require('./$pkg_json'); |
| 208 | + const deps = {...(pkg.dependencies || {})}; |
| 209 | + const interesting = Object.entries(deps).filter(([k]) => |
| 210 | + k.startsWith('@copilotkit/') || k.startsWith('@copilotkitnext/') || |
| 211 | + k.startsWith('@mastra/') || k.startsWith('@ag-ui/') || |
| 212 | + k === 'langchain' || k === 'langgraph' |
| 213 | + ); |
| 214 | + interesting.forEach(([name, version]) => { |
| 215 | + console.log(name + '=' + version); |
| 216 | + }); |
| 217 | + " 2>/dev/null | while IFS='=' read -r name version; do |
| 218 | + if [[ "$version" =~ ^[0-9] ]]; then |
| 219 | + latest=$(npm view "$name" version 2>/dev/null || echo "unknown") |
| 220 | + if [ "$latest" != "unknown" ] && [ "$latest" != "$version" ]; then |
| 221 | + echo " $name: pinned=$version latest=$latest" |
| 222 | + report+="| $slug | $name | $version | $latest |\n" |
| 223 | + fi |
| 224 | + fi |
| 225 | + done |
| 226 | + done |
| 227 | + echo "report<<EOF" >> $GITHUB_OUTPUT |
| 228 | + echo -e "$report" >> $GITHUB_OUTPUT |
| 229 | + echo "EOF" >> $GITHUB_OUTPUT |
| 230 | +
|
| 231 | + - name: Create drift report issue |
| 232 | + if: steps.python_drift.outputs.report != '' || steps.npm_drift.outputs.report != '' |
| 233 | + uses: actions/github-script@v7 |
| 234 | + with: |
| 235 | + script: | |
| 236 | + const pythonDrift = `${{ steps.python_drift.outputs.report }}`.trim(); |
| 237 | + const npmDrift = `${{ steps.npm_drift.outputs.report }}`.trim(); |
| 238 | +
|
| 239 | + if (!pythonDrift && !npmDrift) return; |
| 240 | +
|
| 241 | + const title = `[Drift] Showcase packages have outdated pinned versions`; |
| 242 | + const body = `## Weekly Version Drift Report |
| 243 | +
|
| 244 | + Showcase packages have pinned versions that differ from the latest releases. |
| 245 | + Review each and update if the new version is compatible. |
| 246 | +
|
| 247 | + ${pythonDrift ? `### Python Packages\n| Package | Dep | Pinned | Latest |\n|---------|-----|--------|--------|\n${pythonDrift}` : ''} |
| 248 | +
|
| 249 | + ${npmDrift ? `### npm Packages\n| Package | Dep | Pinned | Latest |\n|---------|-----|--------|--------|\n${npmDrift}` : ''} |
| 250 | +
|
| 251 | + **Action**: For each outdated dep, check the Dojo example for the correct version. |
| 252 | + Update \`requirements.txt\` / \`package.json\`, rebuild, and verify demos still work. |
| 253 | + `; |
| 254 | +
|
| 255 | + const { data: issues } = await github.rest.issues.listForRepo({ |
| 256 | + owner: context.repo.owner, |
| 257 | + repo: context.repo.repo, |
| 258 | + labels: 'showcase-drift,version-drift', |
| 259 | + state: 'open', |
| 260 | + }); |
| 261 | +
|
| 262 | + if (issues.length === 0) { |
| 263 | + await github.rest.issues.create({ |
| 264 | + owner: context.repo.owner, |
| 265 | + repo: context.repo.repo, |
| 266 | + title, |
| 267 | + body, |
| 268 | + labels: ['showcase-drift', 'version-drift'], |
| 269 | + }); |
| 270 | + } else { |
| 271 | + await github.rest.issues.update({ |
| 272 | + owner: context.repo.owner, |
| 273 | + repo: context.repo.repo, |
| 274 | + issue_number: issues[0].number, |
| 275 | + body, |
| 276 | + }); |
| 277 | + } |
0 commit comments