Rebuild weekly search report on the durable analytics API (fail-loud)

.github/workflows/static-quality.yml

@@ -22,7 +22,7 @@ jobs:
       # Cover the gap-analysis script too: it ships from scripts/ and was
       # previously neither format-checked nor type-checked in CI (the other
       # scripts/ files predate this gate and are out of scope here).
-      - run: npx prettier --check "src/**/*.ts" "scripts/gap-analysis/**/*.ts"
+      - run: npx prettier --check "src/**/*.ts" "scripts/gap-analysis/**/*.ts" "scripts/weekly-search-report/**/*.ts"
 
   typecheck-scripts:
     runs-on: ubuntu-latest

.github/workflows/weekly-search-report.yml

@@ -0,0 +1,82 @@
+name: Weekly Search Report
+# Runs the Pathfinder weekly search-query report on a 7-day lookback and
+# publishes a deterministic (no-LLM) report page to Notion. Replaces the old
+# claude.ai routine that scraped ephemeral Railway stdout logs and quietly
+# posted an error page when its token was missing.
+#
+# IMPORTANT: this job works only from the durable query_log-backed analytics
+# JSON API. It does NOT read the indexed repos and does NOT reproduce queries
+# against the live MCP — doing so would self-inflate the analytics it measures.
+#
+# Fail-loud: unlike the retired routine, a run that cannot fetch its data (or
+# cannot publish) exits non-zero and posts a Slack alert to #oss-alerts instead
+# of publishing a degraded "could not generate" page. A red run + Slack alert is
+# the intended signal.
+#
+# Required repository secrets:
+#   PATHFINDER_ANALYTICS_TOKEN  Bearer token for GET /api/analytics/* on the
+#                               production MCP (https://mcp.copilotkit.ai). When
+#                               unset, the script FAILS LOUD (non-zero exit +
+#                               Slack alert) — the report cannot run without it.
+#   NOTION_TOKEN                Notion integration token used to publish the
+#                               report page. A publish failure is fail-loud.
+#   SLACK_WEBHOOK_OSS_ALERTS    Incoming-webhook URL (org-level secret shared by
+#                               every workflow). Mapped to the script's
+#                               SLACK_WEBHOOK env. Posted to only on failure.
+#                               When unset, the alert no-ops but the non-zero
+#                               exit still stands.
+on:
+  schedule:
+    # Weekly, Sunday ~09:07 UTC — off the :00 mark to avoid the cron stampede.
+    # A Sunday run reports the prior Mon–Sun 7-day window.
+    - cron: "7 9 * * 0"
+  workflow_dispatch:
+
+# Stateless report — no prior-run artifact to read, so no `actions: read`.
+permissions:
+  contents: read
+
+jobs:
+  weekly-search-report:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
+        with:
+          persist-credentials: false
+
+      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
+        with:
+          node-version: "22"
+
+      - run: npm ci
+
+      - name: Run weekly search report
+        env:
+          # Read-only analytics access. Unset → script FAILS LOUD (no token,
+          # no report). This secret already exists in the repo (shared with
+          # monthly-gap-analysis).
+          PATHFINDER_ANALYTICS_TOKEN: ${{ secrets.PATHFINDER_ANALYTICS_TOKEN }}
+          # Notion publish target. Publish failure is fail-loud.
+          NOTION_TOKEN: ${{ secrets.NOTION_TOKEN }}
+          # Slack alert (failure only). The script reads SLACK_WEBHOOK; the
+          # org-level secret is SLACK_WEBHOOK_OSS_ALERTS. Unset → alert no-ops,
+          # non-zero exit still stands.
+          SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
+          # Pathfinder project page where weekly reports land.
+          NOTION_PARENT_PAGE_ID: "3373aa38-1852-8152-a10b-e4aa1b8a667e"
+          # Production analytics API base.
+          ANALYTICS_BASE_URL: "https://mcp.copilotkit.ai"
+          # 7-day lookback window.
+          REPORT_DAYS: "7"
+        run: npx tsx scripts/weekly-search-report/weekly-search-report.ts --report /tmp/weekly-report.md
+
+      # Keep the rendered report as a build artifact for inspection even when
+      # Notion publishing is not yet configured.
+      - name: Upload weekly report artifact
+        if: always()
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: weekly-search-report
+          path: /tmp/weekly-report.md
+          if-no-files-found: ignore

scripts/weekly-search-report/fixtures.ts

@@ -0,0 +1,169 @@
+/// <reference types="node" />
+/**
+ * Recorded analytics-API JSON fixtures for the weekly-search-report tests.
+ *
+ * These match the API CONTRACT exactly (see weekly-search-report.ts types):
+ *   summary           → AnalyticsSummary
+ *   /queries          → TopQuery[]    (only queries with >=1 result)
+ *   /empty-queries    → EmptyQuery[]
+ *   /tool-breakdown   → ToolBreakdownRow[] (count desc)
+ */
+
+import type {
+  AnalyticsSummary,
+  TopQuery,
+  EmptyQuery,
+  ToolBreakdownRow,
+} from "./weekly-search-report.js";
+
+export const SUMMARY_FIXTURE: AnalyticsSummary = {
+  total_queries_window: 1234,
+  unique_ip_count_window: 87,
+  unique_session_count_window: 142,
+  empty_result_count_window: 96,
+  empty_result_rate_window: 0.0778,
+  low_confidence_count_window: 30,
+  low_confidence_rate_window: 0.0243,
+  avg_latency_ms_window: 412,
+  p95_latency_ms_window: 1180,
+  queries_by_source: [
+    { source_name: "claude-code", count: 800 },
+    { source_name: "cursor", count: 434 },
+  ],
+  queries_per_day_window: [
+    { day: "2026-06-15", count: 210 },
+    { day: "2026-06-16", count: 188 },
+    { day: "2026-06-17", count: 175 },
+    { day: "2026-06-18", count: 160 },
+    { day: "2026-06-19", count: 199 },
+    { day: "2026-06-20", count: 150 },
+    { day: "2026-06-21", count: 152 },
+  ],
+  earliest_query_day: "2026-06-15",
+};
+
+export const QUERIES_FIXTURE: TopQuery[] = [
+  {
+    query_text: "how to use CoAgents with LangGraph",
+    tool_name: "search-docs",
+    count: 50,
+    avg_result_count: 4.2,
+    avg_top_score: 0.81,
+  },
+  {
+    query_text: "useCopilotAction frontend tool example",
+    tool_name: "search-docs",
+    count: 40,
+    avg_result_count: 3.1,
+    avg_top_score: 0.77,
+  },
+  {
+    query_text: "CopilotKit runtime backend setup",
+    tool_name: "search-code",
+    count: 30,
+    avg_result_count: 5.0,
+    avg_top_score: 0.9,
+  },
+  {
+    query_text: "ag-ui protocol event types",
+    tool_name: "search-ag-ui-docs",
+    count: 25,
+    avg_result_count: 2.0,
+    avg_top_score: 0.7,
+  },
+  {
+    query_text: "generative ui rendering custom component",
+    tool_name: "search-docs",
+    count: 20,
+    avg_result_count: 1.5,
+    avg_top_score: 0.66,
+  },
+  {
+    query_text: "v2 migration useCopilotChat hook",
+    tool_name: "search-docs",
+    count: 18,
+    avg_result_count: 2.2,
+    avg_top_score: 0.72,
+  },
+  {
+    query_text: "how to theme the chat ui with css",
+    tool_name: "search-docs",
+    count: 15,
+    avg_result_count: 3.0,
+    avg_top_score: 0.69,
+  },
+  {
+    query_text: "MCP middleware configuration",
+    tool_name: "search-code",
+    count: 12,
+    avg_result_count: 4.0,
+    avg_top_score: 0.83,
+  },
+  {
+    query_text: "human in the loop interrupt approval",
+    tool_name: "search-docs",
+    count: 10,
+    avg_result_count: 1.0,
+    avg_top_score: 0.6,
+  },
+  {
+    query_text: "streaming events tool call output",
+    tool_name: "search-docs",
+    count: 8,
+    avg_result_count: 2.5,
+    avg_top_score: 0.71,
+  },
+  {
+    query_text: "shared state useCoAgent context",
+    tool_name: "search-docs",
+    count: 7,
+    avg_result_count: 2.0,
+    avg_top_score: 0.74,
+  },
+  {
+    query_text: "getting started quickstart install",
+    tool_name: "search-docs",
+    count: 6,
+    avg_result_count: 6.0,
+    avg_top_score: 0.92,
+  },
+  {
+    query_text: "something completely unrelated and weird",
+    tool_name: "search-docs",
+    count: 3,
+    avg_result_count: 1.0,
+    avg_top_score: 0.5,
+  },
+];
+
+export const EMPTY_QUERIES_FIXTURE: EmptyQuery[] = [
+  {
+    query_text: "copilotkit stripe billing integration",
+    tool_name: "search-docs",
+    source_name: "claude-code",
+    count: 14,
+    last_seen: "2026-06-21T10:00:00Z",
+  },
+  {
+    query_text: "deploy copilotkit to cloudflare workers",
+    tool_name: "search-docs",
+    source_name: "cursor",
+    count: 9,
+    last_seen: "2026-06-20T09:00:00Z",
+  },
+  {
+    query_text: "websocket transport custom adapter",
+    tool_name: "search-code",
+    source_name: "claude-code",
+    count: 4,
+    last_seen: "2026-06-19T11:00:00Z",
+  },
+];
+
+export const TOOL_BREAKDOWN_FIXTURE: ToolBreakdownRow[] = [
+  { tool_name: "search-docs", count: 700 },
+  { tool_name: "search-code", count: 300 },
+  { tool_name: "search-ag-ui-docs", count: 120 },
+  { tool_name: "explore-bash", count: 80 },
+  { tool_name: "explore-grep", count: 34 },
+];