From 6667a5598fbff0df1d3b26c79b65ce61e4e0c431 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Mon, 22 Jun 2026 16:04:08 -0700 Subject: [PATCH] Post weekly report success digest to #engr MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit success → #engr (SLACK_WEBHOOK_ENGR) with a digest + [report] link; failures unchanged on #oss-alerts; no-op if the success webhook is unset. --- .github/workflows/weekly-search-report.yml | 25 ++++-- .../weekly-search-report.test.ts | 82 +++++++++++++++++++ .../weekly-search-report.ts | 53 +++++++++++- 3 files changed, 150 insertions(+), 10 deletions(-) diff --git a/.github/workflows/weekly-search-report.yml b/.github/workflows/weekly-search-report.yml index 40f15a9..e59b9b7 100644 --- a/.github/workflows/weekly-search-report.yml +++ b/.github/workflows/weekly-search-report.yml @@ -13,6 +13,11 @@ name: Weekly Search Report # of publishing a degraded "could not generate" page. A red run + Slack alert is # the intended signal. # +# On SUCCESS, the script additionally posts a one-line digest to #engr (a SEPARATE +# webhook from the failure alert) so a healthy weekly report is visible to +# engineering. The success ping is best-effort: an unset success webhook simply +# skips the ping and never affects the exit code or any fail-loud path. +# # Required repository secrets: # PATHFINDER_ANALYTICS_TOKEN Bearer token for GET /api/analytics/* on the # production MCP (https://mcp.copilotkit.ai). When @@ -22,9 +27,13 @@ name: Weekly Search Report # report page. A publish failure is fail-loud. # SLACK_WEBHOOK_OSS_ALERTS Incoming-webhook URL (org-level secret shared by # every workflow). Mapped to the script's -# SLACK_WEBHOOK env. Posted to only on failure. -# When unset, the alert no-ops but the non-zero -# exit still stands. +# SLACK_WEBHOOK env. Posted to only on FAILURE +# (#oss-alerts). When unset, the alert no-ops but +# the non-zero exit still stands. +# SLACK_WEBHOOK_ENGR Incoming-webhook URL for #engr (B0T app). Mapped +# to the script's SLACK_ENGR_WEBHOOK env. Posted to +# only on SUCCESS (the weekly digest). When unset, +# the success ping is simply skipped. on: schedule: # Weekly, Sunday ~09:07 UTC — off the :00 mark to avoid the cron stampede. @@ -59,10 +68,14 @@ jobs: PATHFINDER_ANALYTICS_TOKEN: ${{ secrets.PATHFINDER_ANALYTICS_TOKEN }} # Notion publish target. Publish failure is fail-loud. NOTION_TOKEN: ${{ secrets.NOTION_TOKEN }} - # Slack alert (failure only). The script reads SLACK_WEBHOOK; the - # org-level secret is SLACK_WEBHOOK_OSS_ALERTS. Unset → alert no-ops, - # non-zero exit still stands. + # Slack alert (failure only → #oss-alerts). The script reads + # SLACK_WEBHOOK; the org-level secret is SLACK_WEBHOOK_OSS_ALERTS. + # Unset → alert no-ops, non-zero exit still stands. SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }} + # Slack success digest (success only → #engr). The script reads + # SLACK_ENGR_WEBHOOK; the org-level secret is SLACK_WEBHOOK_ENGR. + # Unset → the success ping is simply skipped. + SLACK_ENGR_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_ENGR }} # Pathfinder project page where weekly reports land. NOTION_PARENT_PAGE_ID: "3373aa38-1852-8152-a10b-e4aa1b8a667e" # Production analytics API base. diff --git a/scripts/weekly-search-report/weekly-search-report.test.ts b/scripts/weekly-search-report/weekly-search-report.test.ts index 4db12ed..411a34b 100644 --- a/scripts/weekly-search-report/weekly-search-report.test.ts +++ b/scripts/weekly-search-report/weekly-search-report.test.ts @@ -23,6 +23,7 @@ import { assertValidSummary, buildObservations, sanitizeCell, + makePostSlack, type NotionClientLike, type RunDeps, type EmptyQuery, @@ -45,6 +46,7 @@ interface Recorder { deps: RunDeps; notionCalls: Array<{ title: string; markdown: string }>; slackCalls: string[]; + successCalls: string[]; exitCodes: number[]; fetchedPaths: string[]; writtenReports: Array<{ path: string; markdown: string }>; @@ -60,6 +62,7 @@ function makeRecorder( ): Recorder { const notionCalls: Array<{ title: string; markdown: string }> = []; const slackCalls: string[] = []; + const successCalls: string[] = []; const exitCodes: number[] = []; const fetchedPaths: string[] = []; const writtenReports: Array<{ path: string; markdown: string }> = []; @@ -95,6 +98,9 @@ function makeRecorder( postSlack: async (text: string) => { slackCalls.push(text); }, + postSuccess: async (text: string) => { + successCalls.push(text); + }, writeReport: overrides.writeReport ?? ((path: string, markdown: string) => { @@ -118,6 +124,7 @@ function makeRecorder( deps, notionCalls, slackCalls, + successCalls, exitCodes, fetchedPaths, writtenReports, @@ -325,6 +332,81 @@ describe("happy path", () => { }); }); +// ── success ping → #engr (digest with a [report] mrkdwn link) ────────────────-- +// +// On a fully successful run, AFTER publishNotion returns the page url, the +// script posts ONE success digest to the SEPARATE #engr webhook (postSuccess), +// while the FAILURE poster (postSlack → #oss-alerts) is never touched. The +// digest carries the headline numbers and renders the Notion link as an +// `` mrkdwn hyperlink (a "[report]" link, NOT a bare url). + +describe("success ping → #engr", () => { + it("posts exactly one success digest with the headline numbers and a mrkdwn link; never touches the failure poster", async () => { + const rec = makeRecorder(); + await runCatchingExit(rec.deps); + + // Exactly one success ping; the failure poster is untouched. + expect(rec.successCalls).toHaveLength(1); + expect(rec.slackCalls).toHaveLength(0); + expect(rec.exitCodes).not.toContain(1); + + const msg = rec.successCalls[0]; + // Headline numbers from SUMMARY_FIXTURE (1234 tool calls, 87 IPs, 7.8% empty). + expect(msg).toContain("1234"); + expect(msg).toContain("87"); + expect(msg).toContain("7.8%"); + // Top category for the fixture is Agents/CoAgents/AG-UI (CoAgents 50 + ag-ui 25). + expect(msg).toContain("Agents/CoAgents/AG-UI"); + // The link is rendered as an mrkdwn hyperlink, NOT a bare url. + expect(msg).toContain("https://notion.example/page"); + expect(msg).toContain("|report>"); + expect(msg).toContain(""); + expect(msg).not.toContain(" https://notion.example/page "); + }); + + it("does NOT post a success ping and DOES post the failure alert on a fetch failure (fail-loud unchanged)", async () => { + const rec = makeRecorder({ + fetchJson: async (path: string): Promise => { + if (path.includes("/tool-breakdown")) { + throw new Error( + "Analytics fetch failed: 500 Internal Server Error for /api/analytics/tool-breakdown", + ); + } + if (path.includes("/summary")) return SUMMARY_FIXTURE as unknown as T; + if (path.includes("/empty-queries")) + return EMPTY_QUERIES_FIXTURE as unknown as T; + if (path.includes("/queries")) return QUERIES_FIXTURE as unknown as T; + throw new Error(`unexpected path ${path}`); + }, + }); + await runCatchingExit(rec.deps); + + // Fail-loud: failure poster fires, success poster does not. + expect(rec.successCalls).toHaveLength(0); + expect(rec.slackCalls.length).toBeGreaterThan(0); + expect(rec.slackCalls[0]).toMatch(/FAILED/i); + expect(rec.exitCodes).toContain(1); + }); + + it("posts a no-link '(report published)' digest when publishNotion returns null", async () => { + const rec = makeRecorder({ publishNotion: async () => null }); + await runCatchingExit(rec.deps); + + expect(rec.successCalls).toHaveLength(1); + expect(rec.slackCalls).toHaveLength(0); + const msg = rec.successCalls[0]; + expect(msg).toContain("(report published)"); + expect(msg).not.toContain("|report>"); + }); + + it("the success poster no-ops (never throws) when its webhook env is unset", async () => { + // The success poster mirrors makePostSlack: an unset webhook is a logged + // no-op, never a throw. Posting must resolve without error. + const post = makePostSlack(""); + await expect(post("anything")).resolves.toBeUndefined(); + }); +}); + // ── parseReportDays ─────────────────────────────────────────────────────────── describe("parseReportDays", () => { diff --git a/scripts/weekly-search-report/weekly-search-report.ts b/scripts/weekly-search-report/weekly-search-report.ts index ef8ab1a..e76f4c3 100644 --- a/scripts/weekly-search-report/weekly-search-report.ts +++ b/scripts/weekly-search-report/weekly-search-report.ts @@ -30,10 +30,17 @@ * NOTION_PARENT_PAGE_ID Parent page id (default the Pathfinder project * page). * SLACK_WEBHOOK Incoming-webhook URL the script posts FAILURE - * alerts to. The WORKFLOW maps the org secret - * (SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}). + * alerts to (#oss-alerts). The WORKFLOW maps the + * org secret (SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}). * If unset, the alert no-ops but the non-zero exit * still stands. + * SLACK_ENGR_WEBHOOK Incoming-webhook URL the script posts a SUCCESS + * digest to (#engr), so a healthy weekly report is + * visible to engineering. The WORKFLOW maps the org + * secret (SLACK_ENGR_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_ENGR }}). + * Distinct from the failure SLACK_WEBHOOK. If unset, + * the success ping is simply skipped (it never + * affects the exit code or any fail-loud path). * ANALYTICS_BASE_URL Override the analytics host (default prod). * REPORT_DAYS Lookback window in days (default 7). * @@ -897,6 +904,7 @@ export interface RunEnv { NOTION_TOKEN?: string; NOTION_PARENT_PAGE_ID?: string; SLACK_WEBHOOK?: string; + SLACK_ENGR_WEBHOOK?: string; ANALYTICS_BASE_URL?: string; REPORT_DAYS?: string; } @@ -908,8 +916,13 @@ export interface RunDeps { fetchJson: (path: string) => Promise; /** Publish the report to Notion. Returns the page URL (or null). Throws on failure. */ publishNotion: (title: string, markdown: string) => Promise; - /** Post a Slack alert (no-op if the webhook is unset). Never throws. */ + /** Post a Slack FAILURE alert to #oss-alerts (no-op if the webhook is unset). Never throws. */ postSlack: (text: string) => Promise; + /** + * Post a Slack SUCCESS digest to #engr (no-op if the webhook is unset). Never + * throws and never affects the exit code — purely best-effort visibility. + */ + postSuccess: (text: string) => Promise; /** * Write the rendered report to a local path (mkdir -p its parent first). * Injected so the --report write is a unit-testable fail-loud surface like @@ -1162,8 +1175,9 @@ export async function run(deps: RunDeps): Promise { } // Notion publish failure after a good fetch is also fail-loud. + let url: string | null; try { - const url = await deps.publishNotion(title, markdown); + url = await deps.publishNotion(title, markdown); deps.log(`[weekly-report] Published to Notion: ${url ?? "(no url)"}`); } catch (err) { const reason = `Notion publish failed: ${String( @@ -1175,9 +1189,37 @@ export async function run(deps: RunDeps): Promise { return; } + // Best-effort SUCCESS visibility to #engr. postSuccess never throws and never + // touches the exit code, so it cannot affect the fail-loud contract above. + await deps.postSuccess(buildSuccessDigest(bundle, url)); + deps.log("[weekly-report] Done."); } +/** + * Build the one-line Slack mrkdwn SUCCESS digest posted to #engr after a healthy + * publish. The Notion link is rendered as an `` hyperlink (a + * `[report]` link, NOT the raw url); when there is no url (publish returned + * null) the trailing segment is the plain text `(report published)` instead. + * The `top:` segment is omitted when there are no categorized queries. + */ +export function buildSuccessDigest( + bundle: AnalyticsBundle, + url: string | null, +): string { + const { summary } = bundle; + const emptyRatePct = (summary.empty_result_rate_window * 100).toFixed(1); + const cats = categorizeQueries(bundle.queries); + const topSegment = cats.length > 0 ? ` · top: ${cats[0].category}` : ""; + const link = url ? `<${url}|report>` : "(report published)"; + return ( + `:bar_chart: Pathfinder weekly search report — ` + + `${summary.total_queries_window} tool calls · ` + + `${summary.unique_ip_count_window} unique IPs${topSegment} · ` + + `${emptyRatePct}% empty · ${link}` + ); +} + // ── Real-environment wiring (only runs when invoked directly) ───────────────── function buildRealDeps(): RunDeps { @@ -1186,6 +1228,7 @@ function buildRealDeps(): RunDeps { NOTION_TOKEN: process.env.NOTION_TOKEN, NOTION_PARENT_PAGE_ID: process.env.NOTION_PARENT_PAGE_ID, SLACK_WEBHOOK: process.env.SLACK_WEBHOOK, + SLACK_ENGR_WEBHOOK: process.env.SLACK_ENGR_WEBHOOK, ANALYTICS_BASE_URL: process.env.ANALYTICS_BASE_URL, REPORT_DAYS: process.env.REPORT_DAYS, }; @@ -1194,6 +1237,7 @@ function buildRealDeps(): RunDeps { const notionToken = env.NOTION_TOKEN ?? ""; const parentPageId = env.NOTION_PARENT_PAGE_ID ?? DEFAULT_PARENT_PAGE_ID; const webhook = env.SLACK_WEBHOOK ?? ""; + const engrWebhook = env.SLACK_ENGR_WEBHOOK ?? ""; return { env, @@ -1201,6 +1245,7 @@ function buildRealDeps(): RunDeps { fetchJson: makeFetchJson(baseUrl, token), publishNotion: makePublishNotion(notionToken, parentPageId), postSlack: makePostSlack(webhook), + postSuccess: makePostSlack(engrWebhook), writeReport: (path: string, markdown: string) => { mkdirSync(dirname(path), { recursive: true }); writeFileSync(path, markdown, "utf-8");