Post weekly search-report success digest to #engr

.github/workflows/weekly-search-report.yml

@@ -13,6 +13,11 @@ name: Weekly Search Report
 # of publishing a degraded "could not generate" page. A red run + Slack alert is
 # the intended signal.
 #
+# On SUCCESS, the script additionally posts a one-line digest to #engr (a SEPARATE
+# webhook from the failure alert) so a healthy weekly report is visible to
+# engineering. The success ping is best-effort: an unset success webhook simply
+# skips the ping and never affects the exit code or any fail-loud path.
+#
 # Required repository secrets:
 #   PATHFINDER_ANALYTICS_TOKEN  Bearer token for GET /api/analytics/* on the
 #                               production MCP (https://mcp.copilotkit.ai). When
@@ -22,9 +27,13 @@ name: Weekly Search Report
 #                               report page. A publish failure is fail-loud.
 #   SLACK_WEBHOOK_OSS_ALERTS    Incoming-webhook URL (org-level secret shared by
 #                               every workflow). Mapped to the script's
-#                               SLACK_WEBHOOK env. Posted to only on failure.
-#                               When unset, the alert no-ops but the non-zero
-#                               exit still stands.
+#                               SLACK_WEBHOOK env. Posted to only on FAILURE
+#                               (#oss-alerts). When unset, the alert no-ops but
+#                               the non-zero exit still stands.
+#   SLACK_WEBHOOK_ENGR          Incoming-webhook URL for #engr (B0T app). Mapped
+#                               to the script's SLACK_ENGR_WEBHOOK env. Posted to
+#                               only on SUCCESS (the weekly digest). When unset,
+#                               the success ping is simply skipped.
 on:
   schedule:
     # Weekly, Sunday ~09:07 UTC — off the :00 mark to avoid the cron stampede.
@@ -59,10 +68,14 @@ jobs:
           PATHFINDER_ANALYTICS_TOKEN: ${{ secrets.PATHFINDER_ANALYTICS_TOKEN }}
           # Notion publish target. Publish failure is fail-loud.
           NOTION_TOKEN: ${{ secrets.NOTION_TOKEN }}
-          # Slack alert (failure only). The script reads SLACK_WEBHOOK; the
-          # org-level secret is SLACK_WEBHOOK_OSS_ALERTS. Unset → alert no-ops,
-          # non-zero exit still stands.
+          # Slack alert (failure only → #oss-alerts). The script reads
+          # SLACK_WEBHOOK; the org-level secret is SLACK_WEBHOOK_OSS_ALERTS.
+          # Unset → alert no-ops, non-zero exit still stands.
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}
+          # Slack success digest (success only → #engr). The script reads
+          # SLACK_ENGR_WEBHOOK; the org-level secret is SLACK_WEBHOOK_ENGR.
+          # Unset → the success ping is simply skipped.
+          SLACK_ENGR_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_ENGR }}
           # Pathfinder project page where weekly reports land.
           NOTION_PARENT_PAGE_ID: "3373aa38-1852-8152-a10b-e4aa1b8a667e"
           # Production analytics API base.

scripts/weekly-search-report/weekly-search-report.test.ts

@@ -23,6 +23,7 @@ import {
   assertValidSummary,
   buildObservations,
   sanitizeCell,
+  makePostSlack,
   type NotionClientLike,
   type RunDeps,
   type EmptyQuery,
@@ -45,6 +46,7 @@ interface Recorder {
   deps: RunDeps;
   notionCalls: Array<{ title: string; markdown: string }>;
   slackCalls: string[];
+  successCalls: string[];
   exitCodes: number[];
   fetchedPaths: string[];
   writtenReports: Array<{ path: string; markdown: string }>;
@@ -60,6 +62,7 @@ function makeRecorder(
 ): Recorder {
   const notionCalls: Array<{ title: string; markdown: string }> = [];
   const slackCalls: string[] = [];
+  const successCalls: string[] = [];
   const exitCodes: number[] = [];
   const fetchedPaths: string[] = [];
   const writtenReports: Array<{ path: string; markdown: string }> = [];
@@ -95,6 +98,9 @@ function makeRecorder(
     postSlack: async (text: string) => {
       slackCalls.push(text);
     },
+    postSuccess: async (text: string) => {
+      successCalls.push(text);
+    },
     writeReport:
       overrides.writeReport ??
       ((path: string, markdown: string) => {
@@ -118,6 +124,7 @@ function makeRecorder(
     deps,
     notionCalls,
     slackCalls,
+    successCalls,
     exitCodes,
     fetchedPaths,
     writtenReports,
@@ -325,6 +332,81 @@ describe("happy path", () => {
   });
 });
 
+// ── success ping → #engr (digest with a [report] mrkdwn link) ────────────────--
+//
+// On a fully successful run, AFTER publishNotion returns the page url, the
+// script posts ONE success digest to the SEPARATE #engr webhook (postSuccess),
+// while the FAILURE poster (postSlack → #oss-alerts) is never touched. The
+// digest carries the headline numbers and renders the Notion link as an
+// `<url|report>` mrkdwn hyperlink (a "[report]" link, NOT a bare url).
+
+describe("success ping → #engr", () => {
+  it("posts exactly one success digest with the headline numbers and a <url|report> mrkdwn link; never touches the failure poster", async () => {
+    const rec = makeRecorder();
+    await runCatchingExit(rec.deps);
+
+    // Exactly one success ping; the failure poster is untouched.
+    expect(rec.successCalls).toHaveLength(1);
+    expect(rec.slackCalls).toHaveLength(0);
+    expect(rec.exitCodes).not.toContain(1);
+
+    const msg = rec.successCalls[0];
+    // Headline numbers from SUMMARY_FIXTURE (1234 tool calls, 87 IPs, 7.8% empty).
+    expect(msg).toContain("1234");
+    expect(msg).toContain("87");
+    expect(msg).toContain("7.8%");
+    // Top category for the fixture is Agents/CoAgents/AG-UI (CoAgents 50 + ag-ui 25).
+    expect(msg).toContain("Agents/CoAgents/AG-UI");
+    // The link is rendered as an <url|report> mrkdwn hyperlink, NOT a bare url.
+    expect(msg).toContain("https://notion.example/page");
+    expect(msg).toContain("|report>");
+    expect(msg).toContain("<https://notion.example/page|report>");
+    expect(msg).not.toContain(" https://notion.example/page ");
+  });
+
+  it("does NOT post a success ping and DOES post the failure alert on a fetch failure (fail-loud unchanged)", async () => {
+    const rec = makeRecorder({
+      fetchJson: async <T>(path: string): Promise<T> => {
+        if (path.includes("/tool-breakdown")) {
+          throw new Error(
+            "Analytics fetch failed: 500 Internal Server Error for /api/analytics/tool-breakdown",
+          );
+        }
+        if (path.includes("/summary")) return SUMMARY_FIXTURE as unknown as T;
+        if (path.includes("/empty-queries"))
+          return EMPTY_QUERIES_FIXTURE as unknown as T;
+        if (path.includes("/queries")) return QUERIES_FIXTURE as unknown as T;
+        throw new Error(`unexpected path ${path}`);
+      },
+    });
+    await runCatchingExit(rec.deps);
+
+    // Fail-loud: failure poster fires, success poster does not.
+    expect(rec.successCalls).toHaveLength(0);
+    expect(rec.slackCalls.length).toBeGreaterThan(0);
+    expect(rec.slackCalls[0]).toMatch(/FAILED/i);
+    expect(rec.exitCodes).toContain(1);
+  });
+
+  it("posts a no-link '(report published)' digest when publishNotion returns null", async () => {
+    const rec = makeRecorder({ publishNotion: async () => null });
+    await runCatchingExit(rec.deps);
+
+    expect(rec.successCalls).toHaveLength(1);
+    expect(rec.slackCalls).toHaveLength(0);
+    const msg = rec.successCalls[0];
+    expect(msg).toContain("(report published)");
+    expect(msg).not.toContain("|report>");
+  });
+
+  it("the success poster no-ops (never throws) when its webhook env is unset", async () => {
+    // The success poster mirrors makePostSlack: an unset webhook is a logged
+    // no-op, never a throw. Posting must resolve without error.
+    const post = makePostSlack("");
+    await expect(post("anything")).resolves.toBeUndefined();
+  });
+});
+
 // ── parseReportDays ───────────────────────────────────────────────────────────
 
 describe("parseReportDays", () => {

scripts/weekly-search-report/weekly-search-report.ts

@@ -30,10 +30,17 @@
  *   NOTION_PARENT_PAGE_ID       Parent page id (default the Pathfinder project
  *                               page).
  *   SLACK_WEBHOOK               Incoming-webhook URL the script posts FAILURE
- *                               alerts to. The WORKFLOW maps the org secret
- *                               (SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}).
+ *                               alerts to (#oss-alerts). The WORKFLOW maps the
+ *                               org secret (SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}).
  *                               If unset, the alert no-ops but the non-zero exit
  *                               still stands.
+ *   SLACK_ENGR_WEBHOOK          Incoming-webhook URL the script posts a SUCCESS
+ *                               digest to (#engr), so a healthy weekly report is
+ *                               visible to engineering. The WORKFLOW maps the org
+ *                               secret (SLACK_ENGR_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_ENGR }}).
+ *                               Distinct from the failure SLACK_WEBHOOK. If unset,
+ *                               the success ping is simply skipped (it never
+ *                               affects the exit code or any fail-loud path).
  *   ANALYTICS_BASE_URL          Override the analytics host (default prod).
  *   REPORT_DAYS                 Lookback window in days (default 7).
  *
@@ -897,6 +904,7 @@ export interface RunEnv {
   NOTION_TOKEN?: string;
   NOTION_PARENT_PAGE_ID?: string;
   SLACK_WEBHOOK?: string;
+  SLACK_ENGR_WEBHOOK?: string;
   ANALYTICS_BASE_URL?: string;
   REPORT_DAYS?: string;
 }
@@ -908,8 +916,13 @@ export interface RunDeps {
   fetchJson: <T>(path: string) => Promise<T>;
   /** Publish the report to Notion. Returns the page URL (or null). Throws on failure. */
   publishNotion: (title: string, markdown: string) => Promise<string | null>;
-  /** Post a Slack alert (no-op if the webhook is unset). Never throws. */
+  /** Post a Slack FAILURE alert to #oss-alerts (no-op if the webhook is unset). Never throws. */
   postSlack: (text: string) => Promise<void>;
+  /**
+   * Post a Slack SUCCESS digest to #engr (no-op if the webhook is unset). Never
+   * throws and never affects the exit code — purely best-effort visibility.
+   */
+  postSuccess: (text: string) => Promise<void>;
   /**
    * Write the rendered report to a local path (mkdir -p its parent first).
    * Injected so the --report write is a unit-testable fail-loud surface like
@@ -1162,8 +1175,9 @@ export async function run(deps: RunDeps): Promise<void> {
   }
 
   // Notion publish failure after a good fetch is also fail-loud.
+  let url: string | null;
   try {
-    const url = await deps.publishNotion(title, markdown);
+    url = await deps.publishNotion(title, markdown);
     deps.log(`[weekly-report] Published to Notion: ${url ?? "(no url)"}`);
   } catch (err) {
     const reason = `Notion publish failed: ${String(
@@ -1175,9 +1189,37 @@ export async function run(deps: RunDeps): Promise<void> {
     return;
   }
 
+  // Best-effort SUCCESS visibility to #engr. postSuccess never throws and never
+  // touches the exit code, so it cannot affect the fail-loud contract above.
+  await deps.postSuccess(buildSuccessDigest(bundle, url));
+
   deps.log("[weekly-report] Done.");
 }
 
+/**
+ * Build the one-line Slack mrkdwn SUCCESS digest posted to #engr after a healthy
+ * publish. The Notion link is rendered as an `<url|report>` hyperlink (a
+ * `[report]` link, NOT the raw url); when there is no url (publish returned
+ * null) the trailing segment is the plain text `(report published)` instead.
+ * The `top:` segment is omitted when there are no categorized queries.
+ */
+export function buildSuccessDigest(
+  bundle: AnalyticsBundle,
+  url: string | null,
+): string {
+  const { summary } = bundle;
+  const emptyRatePct = (summary.empty_result_rate_window * 100).toFixed(1);
+  const cats = categorizeQueries(bundle.queries);
+  const topSegment = cats.length > 0 ? ` · top: ${cats[0].category}` : "";
+  const link = url ? `<${url}|report>` : "(report published)";
+  return (
+    `:bar_chart: Pathfinder weekly search report — ` +
+    `${summary.total_queries_window} tool calls · ` +
+    `${summary.unique_ip_count_window} unique IPs${topSegment} · ` +
+    `${emptyRatePct}% empty · ${link}`
+  );
+}
+
 // ── Real-environment wiring (only runs when invoked directly) ─────────────────
 
 function buildRealDeps(): RunDeps {
@@ -1186,6 +1228,7 @@ function buildRealDeps(): RunDeps {
     NOTION_TOKEN: process.env.NOTION_TOKEN,
     NOTION_PARENT_PAGE_ID: process.env.NOTION_PARENT_PAGE_ID,
     SLACK_WEBHOOK: process.env.SLACK_WEBHOOK,
+    SLACK_ENGR_WEBHOOK: process.env.SLACK_ENGR_WEBHOOK,
     ANALYTICS_BASE_URL: process.env.ANALYTICS_BASE_URL,
     REPORT_DAYS: process.env.REPORT_DAYS,
   };
@@ -1194,13 +1237,15 @@ function buildRealDeps(): RunDeps {
   const notionToken = env.NOTION_TOKEN ?? "";
   const parentPageId = env.NOTION_PARENT_PAGE_ID ?? DEFAULT_PARENT_PAGE_ID;
   const webhook = env.SLACK_WEBHOOK ?? "";
+  const engrWebhook = env.SLACK_ENGR_WEBHOOK ?? "";
 
   return {
     env,
     argv: process.argv,
     fetchJson: makeFetchJson(baseUrl, token),
     publishNotion: makePublishNotion(notionToken, parentPageId),
     postSlack: makePostSlack(webhook),
+    postSuccess: makePostSlack(engrWebhook),
     writeReport: (path: string, markdown: string) => {
       mkdirSync(dirname(path), { recursive: true });
       writeFileSync(path, markdown, "utf-8");