From c9ededab466802d85224ebb510be6fc7a22ba7f2 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Mon, 22 Jun 2026 14:28:44 -0700 Subject: [PATCH 1/3] Add tool-breakdown endpoint and unique IP/session analytics counts Add getToolBreakdown reader (full tool_name, mirrors getToolCounts) + GET /api/analytics/tool-breakdown route, and unique_ip_count_window / unique_session_count_window to the analytics summary. Query_log-backed, additive. Includes reader + HTTP route tests. --- .../analytics-auth-length-check.test.ts | 1 + src/__tests__/analytics-error-log-ip.test.ts | 1 + .../analytics-sendfile-correlation.test.ts | 1 + src/__tests__/analytics-server.test.ts | 129 +++++++++++++ src/__tests__/analytics.test.ts | 174 +++++++++++++++++- src/db/analytics.ts | 111 ++++++++++- src/server.ts | 37 ++++ 7 files changed, 452 insertions(+), 2 deletions(-) diff --git a/src/__tests__/analytics-auth-length-check.test.ts b/src/__tests__/analytics-auth-length-check.test.ts index b7d7824..f737491 100644 --- a/src/__tests__/analytics-auth-length-check.test.ts +++ b/src/__tests__/analytics-auth-length-check.test.ts @@ -21,6 +21,7 @@ vi.mock("../db/analytics.js", () => ({ getEmptyQueries: vi.fn(), getBlockedQueries: vi.fn(), getToolCounts: vi.fn(), + getToolBreakdown: vi.fn(), })); vi.mock("../config.js", () => ({ diff --git a/src/__tests__/analytics-error-log-ip.test.ts b/src/__tests__/analytics-error-log-ip.test.ts index be1bb88..1804bdc 100644 --- a/src/__tests__/analytics-error-log-ip.test.ts +++ b/src/__tests__/analytics-error-log-ip.test.ts @@ -15,6 +15,7 @@ vi.mock("../db/analytics.js", () => ({ getEmptyQueries: vi.fn(), getBlockedQueries: vi.fn(), getToolCounts: vi.fn(), + getToolBreakdown: vi.fn(), })); vi.mock("../config.js", () => ({ diff --git a/src/__tests__/analytics-sendfile-correlation.test.ts b/src/__tests__/analytics-sendfile-correlation.test.ts index 6ddb59e..2a0705d 100644 --- a/src/__tests__/analytics-sendfile-correlation.test.ts +++ b/src/__tests__/analytics-sendfile-correlation.test.ts @@ -19,6 +19,7 @@ vi.mock("../db/analytics.js", () => ({ getEmptyQueries: vi.fn(), getBlockedQueries: vi.fn(), getToolCounts: vi.fn(), + getToolBreakdown: vi.fn(), })); vi.mock("../config.js", () => ({ diff --git a/src/__tests__/analytics-server.test.ts b/src/__tests__/analytics-server.test.ts index af9cd82..c278470 100644 --- a/src/__tests__/analytics-server.test.ts +++ b/src/__tests__/analytics-server.test.ts @@ -8,6 +8,7 @@ const mockGetTopQueries = vi.fn(); const mockGetEmptyQueries = vi.fn(); const mockGetBlockedQueries = vi.fn(); const mockGetToolCounts = vi.fn(); +const mockGetToolBreakdown = vi.fn(); vi.mock("../db/analytics.js", () => ({ getAnalyticsSummary: (...args: unknown[]) => mockGetAnalyticsSummary(...args), @@ -15,6 +16,7 @@ vi.mock("../db/analytics.js", () => ({ getEmptyQueries: (...args: unknown[]) => mockGetEmptyQueries(...args), getBlockedQueries: (...args: unknown[]) => mockGetBlockedQueries(...args), getToolCounts: (...args: unknown[]) => mockGetToolCounts(...args), + getToolBreakdown: (...args: unknown[]) => mockGetToolBreakdown(...args), })); vi.mock("../config.js", () => ({ @@ -88,6 +90,8 @@ function buildTestApp() { mockGetBlockedQueries(...args), getToolCounts: (...args: Parameters) => mockGetToolCounts(...args), + getToolBreakdown: (...args: Parameters) => + mockGetToolBreakdown(...args), analyticsHtmlPath, }); @@ -450,6 +454,131 @@ describe("Analytics server routes (HTTP-level)", () => { }); }); + // ---- /api/analytics/tool-breakdown --------------------------------------- + // + // HTTP-level coverage for the un-collapsed tool_name breakdown route, the + // sibling of /api/analytics/tool-counts. Mirrors the tool-counts wiring: + // the route forwards the parsed `days` window (first arg) and filter + // (second arg) to getToolBreakdown and returns its rows as JSON. + // --------------------------------------------------------------------------- + + describe("GET /api/analytics/tool-breakdown", () => { + it("returns the breakdown rows with a valid token and forwards days", async () => { + mockGetAnalyticsConfigFn.mockReturnValue({ + enabled: true, + log_queries: true, + retention_days: 90, + token: "secret", + }); + const rows = [ + { tool_name: "search-pathfinder", count: 128 }, + { tool_name: "explore-docs", count: 42 }, + ]; + mockGetToolBreakdown.mockResolvedValue(rows); + + await startApp(); + const res = await request( + server, + "GET", + "/api/analytics/tool-breakdown?days=14", + { Authorization: "Bearer secret" }, + ); + + expect(res.status).toBe(200); + const body = JSON.parse(res.body); + expect(body).toEqual(rows); + // days=14 (non-default) is forwarded as the first arg so a regression + // that drops the param and reapplies the default of 7 is caught. + expect(mockGetToolBreakdown).toHaveBeenCalledWith(14, {}); + }); + + it("defaults days to 7 when not provided", async () => { + mockGetAnalyticsConfigFn.mockReturnValue({ + enabled: true, + log_queries: true, + retention_days: 90, + token: "secret", + }); + mockGetToolBreakdown.mockResolvedValue([]); + + await startApp(); + const res = await request( + server, + "GET", + "/api/analytics/tool-breakdown", + { + Authorization: "Bearer secret", + }, + ); + + expect(res.status).toBe(200); + expect(mockGetToolBreakdown).toHaveBeenCalledWith(7, {}); + }); + + it("returns 401 without a token", async () => { + mockGetAnalyticsConfigFn.mockReturnValue({ + enabled: true, + log_queries: true, + retention_days: 90, + token: "secret", + }); + + await startApp(); + const res = await request(server, "GET", "/api/analytics/tool-breakdown"); + + expect(res.status).toBe(401); + expect(mockGetToolBreakdown).not.toHaveBeenCalled(); + }); + + it("rejects days=abc with 400", async () => { + mockGetAnalyticsConfigFn.mockReturnValue({ + enabled: true, + log_queries: true, + retention_days: 90, + token: "secret", + }); + mockGetToolBreakdown.mockResolvedValue([]); + + await startApp(); + const res = await request( + server, + "GET", + "/api/analytics/tool-breakdown?days=abc", + { Authorization: "Bearer secret" }, + ); + + expect(res.status).toBe(400); + expect(mockGetToolBreakdown).not.toHaveBeenCalled(); + }); + + it("returns 500 with a clean error envelope when the DB call rejects", async () => { + mockGetAnalyticsConfigFn.mockReturnValue({ + enabled: true, + log_queries: true, + retention_days: 90, + token: "secret", + }); + mockGetToolBreakdown.mockRejectedValue(new Error("boom")); + const consoleErrSpy = vi + .spyOn(console, "error") + .mockImplementation(() => {}); + + await startApp(); + const res = await request( + server, + "GET", + "/api/analytics/tool-breakdown", + { Authorization: "Bearer secret" }, + ); + + expect(res.status).toBe(500); + const body = JSON.parse(res.body); + expect(body.error).toBe("Failed to fetch tool breakdown"); + expect(res.body).not.toContain("boom"); + consoleErrSpy.mockRestore(); + }); + }); + // ---- Auto-generated token via HTTP --------------------------------------- describe("auto-generated token (HTTP-level)", () => { diff --git a/src/__tests__/analytics.test.ts b/src/__tests__/analytics.test.ts index 01fa4c3..167f8c9 100644 --- a/src/__tests__/analytics.test.ts +++ b/src/__tests__/analytics.test.ts @@ -13,6 +13,7 @@ import { getEmptyQueries, getBlockedQueries, getToolCounts, + getToolBreakdown, cleanupOldQueryLogs, REDACTED_QUERY_TEXT, P95_LATENCY_ROW_CAP, @@ -22,7 +23,6 @@ import { REQUEST_SOURCE_VALUES, ALL_TIME_DAYS, ROLLING_WINDOW_CAP_DAYS, - BROWSE_QUERY_TEXT, } from "../db/analytics.js"; import type { QueryLogEntry } from "../db/analytics.js"; @@ -1945,3 +1945,175 @@ describe("request-source clause on top/empty/tool-count readers", () => { expect(sql).not.toContain("request_source"); }); }); + +// --------------------------------------------------------------------------- +// getAnalyticsSummary unique IP / session counts (weekly-report fields) +// +// The windowed summary subquery gains two COUNT(DISTINCT ...) aggregates so +// the weekly search report can headline "M unique IPs across S sessions". +// They live INSIDE the existing windowed summaryRes SELECT so they inherit +// its exact predicates (date window, latency_ms >= 0, redacted exclusion, +// request_source, buildFilterClauses) and reconcile with total_queries_window. +// NULL client_ip / session_id are ignored via FILTER. +// --------------------------------------------------------------------------- + +describe("getAnalyticsSummary unique IP/session counts", () => { + // Mock order: total, windowed summary, latency rows, by-source, per-day, + // earliest-day. The unique counts are columns on the SAME windowed summary + // row (call index 1), so they're returned alongside total/empty/avg_latency. + function mockSummary(summaryRow: Record) { + mockQuery + .mockResolvedValueOnce({ rows: [{ count: 1000 }] }) // total + .mockResolvedValueOnce({ rows: [summaryRow] }) // windowed summary + .mockResolvedValueOnce({ rows: [] }) // latency + .mockResolvedValueOnce({ rows: [] }) // by source + .mockResolvedValueOnce({ rows: [] }) // per day + .mockResolvedValueOnce({ rows: [{ earliest_day: null }] }); // earliest day + } + + it("returns unique_ip_count_window and unique_session_count_window from the windowed summary row", async () => { + mockSummary({ + total: 200, + empty: 10, + avg_latency: 45, + unique_ip_count_window: 37, + unique_session_count_window: 52, + }); + + const result = await getAnalyticsSummary(); + + expect(result.unique_ip_count_window).toBe(37); + expect(result.unique_session_count_window).toBe(52); + }); + + it("computes both counts in the SAME windowed subquery as total_queries_window (one summary call, COUNT DISTINCT with NULL-ignoring FILTER)", async () => { + mockSummary({ + total: 200, + empty: 10, + avg_latency: 45, + unique_ip_count_window: 37, + unique_session_count_window: 52, + }); + + await getAnalyticsSummary({}, 7); + + // The windowed summary subquery is the one that selects `AS total`. + const summaryCall = mockQuery.mock.calls.find( + (call) => + typeof call[0] === "string" && + /AS total\b/.test(call[0]) && + /latency_ms >= 0/.test(call[0]), + ); + expect(summaryCall, "windowed summary query not found").toBeDefined(); + const sql = summaryCall![0] as string; + expect(sql).toMatch( + /COUNT\(DISTINCT client_ip\)\s+FILTER\s*\(WHERE client_ip\s+IS NOT NULL\)::int\s+AS unique_ip_count_window/i, + ); + expect(sql).toMatch( + /COUNT\(DISTINCT session_id\)\s+FILTER\s*\(WHERE session_id\s+IS NOT NULL\)::int\s+AS unique_session_count_window/i, + ); + }); + + it("coerces the unique counts defensively (string-typed driver values become numbers; missing → 0)", async () => { + // node-postgres deserializes some integer/numeric columns as strings; + // the summary numerics are coerced via toFiniteNumber. A missing column + // must default to 0, not NaN/undefined. + mockSummary({ + total: 200, + empty: 10, + avg_latency: 45, + unique_ip_count_window: "37", + // unique_session_count_window intentionally absent + }); + + const result = await getAnalyticsSummary(); + + expect(result.unique_ip_count_window).toBe(37); + expect(result.unique_session_count_window).toBe(0); + }); +}); + +// --------------------------------------------------------------------------- +// getToolBreakdown — full tool_name breakdown (NOT prefix-collapsed) +// +// Exact getToolCounts body with split_part(tool_name, '-', 1) replaced by the +// full tool_name. Same predicate set as getToolCounts: buildDateWindow + +// buildRequestSourceClause + latency_ms >= 0 + sourceOnlyFilter (tool_type +// dropped). NO blocked filter; NO redacted exclusion (preserves the +// getToolCounts divergence). Returns [{ tool_name, count }] ordered count desc. +// Drives the weekly report's per-search-tool + explore-command breakdowns. +// --------------------------------------------------------------------------- + +describe("getToolBreakdown", () => { + it("returns FULL tool names as distinct rows (not collapsed to the hyphen prefix)", async () => { + mockQuery.mockResolvedValueOnce({ + rows: [ + { tool_name: "search-docs", count: 120 }, + { tool_name: "search-code", count: 80 }, + { tool_name: "explore-file", count: 15 }, + ], + }); + + const result = await getToolBreakdown(7); + + // All three full names appear separately — NOT collapsed to search/explore. + expect(result).toHaveLength(3); + expect(result.map((r) => r.tool_name)).toEqual([ + "search-docs", + "search-code", + "explore-file", + ]); + expect(result[0]).toEqual({ tool_name: "search-docs", count: 120 }); + + const [sql, params] = mockQuery.mock.calls[0]; + // Uses the FULL tool_name, never the prefix-collapsing split_part. + expect(sql).not.toContain("split_part"); + expect(sql).toMatch(/GROUP BY tool_name/); + expect(sql).toMatch(/ORDER BY count DESC/); + // Same predicate set as getToolCounts. + expect(sql).toContain("latency_ms >= 0"); + // No blocked clause, no redacted exclusion (matches getToolCounts). + expect(sql).not.toContain("blocked"); + expect(sql).not.toContain("REDACTED"); + // Default request-source filter binds "user" (real-users-by-default). + expect(params).toEqual([7, "user"]); + }); + + it("returns empty array when no queries exist", async () => { + mockQuery.mockResolvedValueOnce({ rows: [] }); + const result = await getToolBreakdown(7); + expect(result).toEqual([]); + }); + + it("drops the tool_type filter (like getToolCounts) so the breakdown is never circular", async () => { + mockQuery.mockResolvedValueOnce({ rows: [] }); + await getToolBreakdown(7, { tool_type: "search" }); + + const [sql] = mockQuery.mock.calls[0]; + // tool_type would defeat the breakdown — its clause must be absent. + // (`LIKE` still appears via the default service-traffic exclusion on + // session_id, so we assert specifically against the tool_type predicate + // shape — `tool_name = $N OR tool_name LIKE $N+1 || '-%'`.) + expect(sql).not.toContain("tool_name = $"); + expect(sql).not.toContain("tool_name LIKE"); + }); + + it("honors the source filter", async () => { + mockQuery.mockResolvedValueOnce({ rows: [] }); + await getToolBreakdown(7, { source: "docs" }); + + const [sql, params] = mockQuery.mock.calls[0]; + expect(sql).toContain("source_name = $"); + expect(params).toContain("docs"); + }); + + it("honors request_source: 'analysis' as exact match", async () => { + mockQuery.mockResolvedValueOnce({ rows: [] }); + await getToolBreakdown(7, { request_source: "analysis" }); + + const [sql, params] = mockQuery.mock.calls[0]; + expect(sql).toContain("request_source = $"); + expect(sql).not.toContain("request_source IS NULL"); + expect(params).toContain("analysis"); + }); +}); diff --git a/src/db/analytics.ts b/src/db/analytics.ts index 2933040..726f47e 100644 --- a/src/db/analytics.ts +++ b/src/db/analytics.ts @@ -191,6 +191,21 @@ export interface AnalyticsSummary { */ low_confidence_rate_window: number; avg_latency_ms_window: number; + /** + * Distinct non-NULL `client_ip` values in the window. Computed inside the + * windowed summary subquery (via `COUNT(DISTINCT ...) FILTER (WHERE ... IS + * NOT NULL)`) so it shares total_queries_window's exact predicates (date + * window, latency_ms >= 0, redacted exclusion, request-source, + * buildFilterClauses) and reconciles with it. NULL IPs are ignored. Headline + * input for the weekly search report ("M unique IPs across S sessions"). + */ + unique_ip_count_window: number; + /** + * Distinct non-NULL `session_id` values in the window. Same windowed + * population and predicates as unique_ip_count_window; NULL sessions are + * ignored. Headline input for the weekly search report. + */ + unique_session_count_window: number; p95_latency_ms_window: number; /** * True when the p95 latency was computed over a random sample capped at @@ -264,6 +279,18 @@ export interface ToolCount { count: number; } +/** + * One row per FULL `tool_name` (e.g. `search-docs`, `search-code`, + * `explore-file`) — the un-collapsed counterpart to {@link ToolCount}, whose + * `tool_type` is only the first-hyphen prefix. Produced by + * {@link getToolBreakdown} for the weekly search report's per-tool and + * explore-command breakdowns. + */ +export interface ToolBreakdown { + tool_name: string; + count: number; +} + export interface AnalyticsFilter { tool_type?: string; source?: string; @@ -767,7 +794,9 @@ export async function getAnalyticsSummary( AND top_score IS NOT NULL AND top_score < $${lowConfIdx2} )::int AS low_confidence, - COALESCE(avg(latency_ms)::int, 0) AS avg_latency + COALESCE(avg(latency_ms)::int, 0) AS avg_latency, + COUNT(DISTINCT client_ip) FILTER (WHERE client_ip IS NOT NULL)::int AS unique_ip_count_window, + COUNT(DISTINCT session_id) FILTER (WHERE session_id IS NOT NULL)::int AS unique_session_count_window FROM query_log ${summaryWhere}`, [ @@ -933,6 +962,14 @@ export async function getAnalyticsSummary( const emptyWindow = toFiniteNumber(s.empty); const lowConfidenceWindow = toFiniteNumber(s.low_confidence); const avgLatencyWindow = toFiniteNumber(s.avg_latency); + // Unique IP / session counts over the SAME windowed population as + // total_queries_window (they're columns on the windowed summary subquery). + // Coerced defensively because node-postgres can deserialize the ::int + // COUNT(DISTINCT ...) as a string, and a missing column must default to 0. + const uniqueIpCountWindow = toFiniteNumber(s.unique_ip_count_window); + const uniqueSessionCountWindow = toFiniteNumber( + s.unique_session_count_window, + ); // Normalize undefined (truly missing) to null so consumers get a // consistent shape regardless of whether the DB returned an empty // row, a row with NULL, or no row at all. @@ -959,6 +996,8 @@ export async function getAnalyticsSummary( low_confidence_rate_window: totalWindow > 0 ? lowConfidenceWindow / totalWindow : 0, avg_latency_ms_window: avgLatencyWindow, + unique_ip_count_window: uniqueIpCountWindow, + unique_session_count_window: uniqueSessionCountWindow, p95_latency_ms_window: p95Latency, // Only set when the cap was actually hit so existing consumers (tests, // older UI builds) can treat the absence of the flag as "exact". @@ -1263,6 +1302,76 @@ export async function getToolCounts( })); } +/** + * Get query counts grouped by the FULL `tool_name` (e.g. `search-docs`, + * `search-code`, `explore-file`) rather than the first-hyphen prefix that + * {@link getToolCounts} collapses to. This is the EXACT getToolCounts body + * with `split_part(tool_name, '-', 1)` replaced by the full `tool_name`, so + * its counts reconcile with the existing tool-counts donut and the summary + * totals. The weekly search report uses it for the per-search-tool breakdown + * (search-docs vs search-code vs search-ag-ui-*) and the explore/bash command + * breakdown (explore-* tool names). + * + * Predicate set is identical to getToolCounts: the date window + * ({@link buildDateWindow}), the request-source clause + * ({@link buildRequestSourceClause}, real-users-by-default), `latency_ms >= 0` + * (exclude backfilled rows), and the `source`-only filter (the `tool_type` + * filter is dropped — filtering the breakdown by tool type would be circular). + * + * Intentional divergences carried over from getToolCounts (do NOT "fix for + * consistency"): + * - Does NOT exclude redacted rows (`query_text = REDACTED_QUERY_TEXT`): + * `tool_name` survives redaction, so they contribute valid tool-usage + * signal. + * - Does NOT add a `blocked = FALSE` clause: it mirrors getToolCounts + * exactly, which also omits the `blocked` filter, so the two reconcile + * against the same tool-counts donut. (Some other windowed readers, e.g. + * getEmptyQueries, do filter `blocked = FALSE`; the breakdown deliberately + * does not, to stay 1:1 with getToolCounts.) + */ +export async function getToolBreakdown( + days: number = 7, + filter: AnalyticsFilter = {}, +): Promise { + const pool = getPool(); + + // tool_type intentionally ignored (matches getToolCounts): filtering the + // breakdown by tool type would be circular. Only `source` is honored. + const { tool_type: _ignoredToolType, ...rest } = filter; + void _ignoredToolType; + const sourceOnlyFilter: AnalyticsFilter = rest; + + const { + clauses: fc, + params: fp, + nextIdx, + } = buildFilterClauses(sourceOnlyFilter); + const dw = buildDateWindow(sourceOnlyFilter, days, nextIdx); + // request_source survives in `rest` (only tool_type is stripped above), so + // the breakdown honors the same real-users-by-default behavior as every + // other windowed aggregate. + const rs = buildRequestSourceClause(sourceOnlyFilter, dw.nextIdx); + // Exclude backfilled rows (latency_ms < 0) so the breakdown counts match the + // windowed aggregates used elsewhere (summary, latency, per-day, tool-counts). + const baseClauses = [...dw.clauses, ...rs.clauses, "latency_ms >= 0"]; + const where = whereAnd(baseClauses, fc); + const { rows } = await pool.query( + `SELECT + tool_name, + count(*)::int AS count + FROM query_log + ${where} + GROUP BY tool_name + ORDER BY count DESC`, + [...fp, ...dw.params, ...rs.params], + ); + + return rows.map((r: Record) => ({ + tool_name: r.tool_name as string, + count: r.count as number, + })); +} + /** * Get Atlas-specific retrieval health for the current window. * diff --git a/src/server.ts b/src/server.ts index 78f5e8e..28df625 100644 --- a/src/server.ts +++ b/src/server.ts @@ -86,6 +86,7 @@ import { getEmptyQueries, getBlockedQueries, getToolCounts, + getToolBreakdown, normalizeRequestSource, REQUEST_SOURCE_HEADER, REQUEST_SOURCE_VALUES, @@ -3041,6 +3042,7 @@ export interface AnalyticsRouteDeps { getEmptyQueries?: typeof getEmptyQueries; getBlockedQueries?: typeof getBlockedQueries; getToolCounts?: typeof getToolCounts; + getToolBreakdown?: typeof getToolBreakdown; analyticsHtmlPath?: string; } @@ -3058,6 +3060,7 @@ export function registerAnalyticsRoutes( const _getEmptyQueries = deps.getEmptyQueries ?? getEmptyQueries; const _getBlockedQueries = deps.getBlockedQueries ?? getBlockedQueries; const _getToolCounts = deps.getToolCounts ?? getToolCounts; + const _getToolBreakdown = deps.getToolBreakdown ?? getToolBreakdown; const _analyticsHtmlPath = deps.analyticsHtmlPath ?? path.resolve(__dirname, "../docs/analytics.html"); @@ -3222,6 +3225,40 @@ export function registerAnalyticsRoutes( }, ); + // Full tool_name breakdown (un-collapsed counterpart to tool-counts, which + // only exposes the first-hyphen prefix). Powers the weekly search report's + // per-search-tool and explore-command breakdowns. Same filter/window parsing + // as the sibling tool-counts endpoint. + app.get( + "/api/analytics/tool-breakdown", + analyticsAuth, + async (req: Request, res: Response) => { + const parsed = parseAnalyticsFilter(req); + if (!parsed.ok) { + res.status(parsed.status).json(parsed.body); + return; + } + const daysParsed = parseDaysOrError(req); + if (!daysParsed.ok) { + res.status(daysParsed.status).json(daysParsed.body); + return; + } + try { + const breakdown = await _getToolBreakdown( + daysParsed.value, + parsed.filter, + ); + res.json(breakdown); + } catch (err) { + console.error( + `[analytics] Tool breakdown failed (filter=${JSON.stringify(parsed.filter)} days=${daysParsed.value} ip=${clientIp(req, trustProxy)}):`, + err, + ); + res.status(500).json({ error: "Failed to fetch tool breakdown" }); + } + }, + ); + app.get("/api/analytics/auth-mode", (req: Request, res: Response) => { res.json(getAuthMode(req)); }); From 94e1b7d0af23ebaf1e6aef1febffaa9edbfd0b99 Mon Sep 17 00:00:00 2001 From: Jordan Ritter Date: Mon, 22 Jun 2026 14:28:44 -0700 Subject: [PATCH 2/3] Add weekly search-query report generator Deterministic (no-LLM) report that reads the durable analytics API and publishes to Notion. Fails loud (non-zero exit + Slack, never a degraded page) on missing token / fetch failure / malformed payload. Renders native Notion tables, derives window/title labels from the actual queried window, and sanitizes all cell/observation text. --- scripts/weekly-search-report/fixtures.ts | 169 +++ .../weekly-search-report.test.ts | 959 +++++++++++++ .../weekly-search-report.ts | 1228 +++++++++++++++++ 3 files changed, 2356 insertions(+) create mode 100644 scripts/weekly-search-report/fixtures.ts create mode 100644 scripts/weekly-search-report/weekly-search-report.test.ts create mode 100644 scripts/weekly-search-report/weekly-search-report.ts diff --git a/scripts/weekly-search-report/fixtures.ts b/scripts/weekly-search-report/fixtures.ts new file mode 100644 index 0000000..18b4c08 --- /dev/null +++ b/scripts/weekly-search-report/fixtures.ts @@ -0,0 +1,169 @@ +/// +/** + * Recorded analytics-API JSON fixtures for the weekly-search-report tests. + * + * These match the API CONTRACT exactly (see weekly-search-report.ts types): + * summary → AnalyticsSummary + * /queries → TopQuery[] (only queries with >=1 result) + * /empty-queries → EmptyQuery[] + * /tool-breakdown → ToolBreakdownRow[] (count desc) + */ + +import type { + AnalyticsSummary, + TopQuery, + EmptyQuery, + ToolBreakdownRow, +} from "./weekly-search-report.js"; + +export const SUMMARY_FIXTURE: AnalyticsSummary = { + total_queries_window: 1234, + unique_ip_count_window: 87, + unique_session_count_window: 142, + empty_result_count_window: 96, + empty_result_rate_window: 0.0778, + low_confidence_count_window: 30, + low_confidence_rate_window: 0.0243, + avg_latency_ms_window: 412, + p95_latency_ms_window: 1180, + queries_by_source: [ + { source_name: "claude-code", count: 800 }, + { source_name: "cursor", count: 434 }, + ], + queries_per_day_window: [ + { day: "2026-06-15", count: 210 }, + { day: "2026-06-16", count: 188 }, + { day: "2026-06-17", count: 175 }, + { day: "2026-06-18", count: 160 }, + { day: "2026-06-19", count: 199 }, + { day: "2026-06-20", count: 150 }, + { day: "2026-06-21", count: 152 }, + ], + earliest_query_day: "2026-06-15", +}; + +export const QUERIES_FIXTURE: TopQuery[] = [ + { + query_text: "how to use CoAgents with LangGraph", + tool_name: "search-docs", + count: 50, + avg_result_count: 4.2, + avg_top_score: 0.81, + }, + { + query_text: "useCopilotAction frontend tool example", + tool_name: "search-docs", + count: 40, + avg_result_count: 3.1, + avg_top_score: 0.77, + }, + { + query_text: "CopilotKit runtime backend setup", + tool_name: "search-code", + count: 30, + avg_result_count: 5.0, + avg_top_score: 0.9, + }, + { + query_text: "ag-ui protocol event types", + tool_name: "search-ag-ui-docs", + count: 25, + avg_result_count: 2.0, + avg_top_score: 0.7, + }, + { + query_text: "generative ui rendering custom component", + tool_name: "search-docs", + count: 20, + avg_result_count: 1.5, + avg_top_score: 0.66, + }, + { + query_text: "v2 migration useCopilotChat hook", + tool_name: "search-docs", + count: 18, + avg_result_count: 2.2, + avg_top_score: 0.72, + }, + { + query_text: "how to theme the chat ui with css", + tool_name: "search-docs", + count: 15, + avg_result_count: 3.0, + avg_top_score: 0.69, + }, + { + query_text: "MCP middleware configuration", + tool_name: "search-code", + count: 12, + avg_result_count: 4.0, + avg_top_score: 0.83, + }, + { + query_text: "human in the loop interrupt approval", + tool_name: "search-docs", + count: 10, + avg_result_count: 1.0, + avg_top_score: 0.6, + }, + { + query_text: "streaming events tool call output", + tool_name: "search-docs", + count: 8, + avg_result_count: 2.5, + avg_top_score: 0.71, + }, + { + query_text: "shared state useCoAgent context", + tool_name: "search-docs", + count: 7, + avg_result_count: 2.0, + avg_top_score: 0.74, + }, + { + query_text: "getting started quickstart install", + tool_name: "search-docs", + count: 6, + avg_result_count: 6.0, + avg_top_score: 0.92, + }, + { + query_text: "something completely unrelated and weird", + tool_name: "search-docs", + count: 3, + avg_result_count: 1.0, + avg_top_score: 0.5, + }, +]; + +export const EMPTY_QUERIES_FIXTURE: EmptyQuery[] = [ + { + query_text: "copilotkit stripe billing integration", + tool_name: "search-docs", + source_name: "claude-code", + count: 14, + last_seen: "2026-06-21T10:00:00Z", + }, + { + query_text: "deploy copilotkit to cloudflare workers", + tool_name: "search-docs", + source_name: "cursor", + count: 9, + last_seen: "2026-06-20T09:00:00Z", + }, + { + query_text: "websocket transport custom adapter", + tool_name: "search-code", + source_name: "claude-code", + count: 4, + last_seen: "2026-06-19T11:00:00Z", + }, +]; + +export const TOOL_BREAKDOWN_FIXTURE: ToolBreakdownRow[] = [ + { tool_name: "search-docs", count: 700 }, + { tool_name: "search-code", count: 300 }, + { tool_name: "search-ag-ui-docs", count: 120 }, + { tool_name: "explore-bash", count: 80 }, + { tool_name: "explore-grep", count: 34 }, +]; diff --git a/scripts/weekly-search-report/weekly-search-report.test.ts b/scripts/weekly-search-report/weekly-search-report.test.ts new file mode 100644 index 0000000..4db12ed --- /dev/null +++ b/scripts/weekly-search-report/weekly-search-report.test.ts @@ -0,0 +1,959 @@ +import { describe, it, expect, vi } from "vitest"; +import { mkdirSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; +import { + parseReportDays, + reportPathArgFrom, + categorizeQuery, + categorizeQueries, + topNQueries, + reportWindow, + reportTitle, + renderMarkdown, + type AnalyticsBundle, + exploreBreakdown, + searchVsExploreSplit, + markdownToNotionBlocks, + batchBlocks, + NOTION_RICH_TEXT_LIMIT, + NOTION_MAX_BLOCKS_PER_REQUEST, + CATEGORY_TAXONOMY, + publishNotionWithClient, + run, + assertValidSummary, + buildObservations, + sanitizeCell, + type NotionClientLike, + type RunDeps, + type EmptyQuery, +} from "./weekly-search-report.js"; +import { + SUMMARY_FIXTURE, + QUERIES_FIXTURE, + EMPTY_QUERIES_FIXTURE, + TOOL_BREAKDOWN_FIXTURE, +} from "./fixtures.js"; + +// ── Test harness for run() ──────────────────────────────────────────────────── +// +// run() takes injected fetch/notion/slack so the fail-loud paths are exercised +// WITHOUT the network. Each fake records whether/how it was called so we can +// assert the exact fail-loud contract: on any fetch failure, exit is non-zero, +// Slack is attempted, and Notion is NEVER called. + +interface Recorder { + deps: RunDeps; + notionCalls: Array<{ title: string; markdown: string }>; + slackCalls: string[]; + exitCodes: number[]; + fetchedPaths: string[]; + writtenReports: Array<{ path: string; markdown: string }>; +} + +function makeRecorder( + overrides: Partial<{ + token: string; + fetchJson: RunDeps["fetchJson"]; + publishNotion: RunDeps["publishNotion"]; + writeReport: RunDeps["writeReport"]; + }> = {}, +): Recorder { + const notionCalls: Array<{ title: string; markdown: string }> = []; + const slackCalls: string[] = []; + const exitCodes: number[] = []; + const fetchedPaths: string[] = []; + const writtenReports: Array<{ path: string; markdown: string }> = []; + + const deps: RunDeps = { + env: { + PATHFINDER_ANALYTICS_TOKEN: overrides.token ?? "tok-123", + NOTION_TOKEN: "notion-tok", + NOTION_PARENT_PAGE_ID: "parent-123", + SLACK_WEBHOOK: "https://slack.example/webhook", + ANALYTICS_BASE_URL: "https://mcp.example", + REPORT_DAYS: "7", + }, + argv: ["node", "script"], + fetchJson: + overrides.fetchJson ?? + (async (path: string): Promise => { + fetchedPaths.push(path); + if (path.includes("/summary")) return SUMMARY_FIXTURE as unknown as T; + if (path.includes("/tool-breakdown")) + return TOOL_BREAKDOWN_FIXTURE as unknown as T; + if (path.includes("/empty-queries")) + return EMPTY_QUERIES_FIXTURE as unknown as T; + if (path.includes("/queries")) return QUERIES_FIXTURE as unknown as T; + throw new Error(`unexpected path ${path}`); + }), + publishNotion: + overrides.publishNotion ?? + (async (title: string, markdown: string) => { + notionCalls.push({ title, markdown }); + return "https://notion.example/page"; + }), + postSlack: async (text: string) => { + slackCalls.push(text); + }, + writeReport: + overrides.writeReport ?? + ((path: string, markdown: string) => { + writtenReports.push({ path, markdown }); + // Default does the REAL write so the on-disk --report test still + // exercises mkdir + writeFileSync; override to simulate a write failure. + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, markdown, "utf-8"); + }), + exit: (code: number) => { + exitCodes.push(code); + // Throw to abort the run the way process.exit would terminate it, so the + // code after a fail-loud exit() does not keep running in the test. + throw new Error(`__EXIT__${code}`); + }, + log: () => {}, + error: () => {}, + }; + + return { + deps, + notionCalls, + slackCalls, + exitCodes, + fetchedPaths, + writtenReports, + }; +} + +async function runCatchingExit(deps: RunDeps): Promise { + try { + await run(deps); + } catch (err) { + if (!String(err).includes("__EXIT__")) throw err; + } +} + +// ── FAIL-LOUD regression tests (the 2026-06-21 silent-error-page failure) ───── + +describe("fail-loud: missing token", () => { + it("exits non-zero, attempts Slack, and NEVER publishes Notion when the token is missing", async () => { + const rec = makeRecorder({ token: "" }); + await runCatchingExit(rec.deps); + + expect(rec.exitCodes).toContain(1); + // Slack alert attempted with a greppable failure reason. + expect(rec.slackCalls.length).toBeGreaterThan(0); + expect(rec.slackCalls[0]).toMatch(/FAILED/i); + // The anti-pattern being removed: NO degraded/error Notion page. + expect(rec.notionCalls).toHaveLength(0); + }); + + it("does not even attempt a fetch when the token is missing", async () => { + const rec = makeRecorder({ token: "" }); + await runCatchingExit(rec.deps); + expect(rec.fetchedPaths).toHaveLength(0); + }); +}); + +describe("fail-loud: a required endpoint returns non-2xx", () => { + it("exits non-zero, attempts Slack, and NEVER publishes Notion on a 500", async () => { + const rec = makeRecorder({ + fetchJson: async (path: string): Promise => { + if (path.includes("/tool-breakdown")) { + throw new Error( + "Analytics fetch failed: 500 Internal Server Error for /api/analytics/tool-breakdown", + ); + } + if (path.includes("/summary")) return SUMMARY_FIXTURE as unknown as T; + if (path.includes("/empty-queries")) + return EMPTY_QUERIES_FIXTURE as unknown as T; + if (path.includes("/queries")) return QUERIES_FIXTURE as unknown as T; + throw new Error(`unexpected path ${path}`); + }, + }); + await runCatchingExit(rec.deps); + + expect(rec.exitCodes).toContain(1); + expect(rec.slackCalls.length).toBeGreaterThan(0); + expect(rec.slackCalls[0]).toMatch(/FAILED/i); + expect(rec.notionCalls).toHaveLength(0); + }); +}); + +describe("fail-loud: malformed payload", () => { + it("exits non-zero, attempts Slack, and NEVER publishes Notion when summary is malformed", async () => { + const rec = makeRecorder({ + fetchJson: async (path: string): Promise => { + if (path.includes("/summary")) { + // Missing required numeric fields → malformed. + return { not: "a summary" } as unknown as T; + } + if (path.includes("/tool-breakdown")) + return TOOL_BREAKDOWN_FIXTURE as unknown as T; + if (path.includes("/empty-queries")) + return EMPTY_QUERIES_FIXTURE as unknown as T; + if (path.includes("/queries")) return QUERIES_FIXTURE as unknown as T; + throw new Error(`unexpected path ${path}`); + }, + }); + await runCatchingExit(rec.deps); + + expect(rec.exitCodes).toContain(1); + expect(rec.slackCalls.length).toBeGreaterThan(0); + expect(rec.notionCalls).toHaveLength(0); + }); + + it("treats a non-array /queries payload as malformed", async () => { + const rec = makeRecorder({ + fetchJson: async (path: string): Promise => { + if (path.includes("/summary")) return SUMMARY_FIXTURE as unknown as T; + if (path.includes("/tool-breakdown")) + return TOOL_BREAKDOWN_FIXTURE as unknown as T; + if (path.includes("/empty-queries")) + return EMPTY_QUERIES_FIXTURE as unknown as T; + if (path.includes("/queries")) return { oops: true } as unknown as T; // not an array + throw new Error(`unexpected path ${path}`); + }, + }); + await runCatchingExit(rec.deps); + expect(rec.exitCodes).toContain(1); + expect(rec.notionCalls).toHaveLength(0); + }); + + it("throws when a queries_per_day_window row is missing count", () => { + const bad = { + ...SUMMARY_FIXTURE, + queries_per_day_window: [{ day: "2026-06-15" }], // missing count + }; + expect(() => assertValidSummary(bad)).toThrow(/queries_per_day_window/); + }); + + it("exits non-zero and NEVER publishes Notion when a /tool-breakdown row is missing count", async () => { + const rec = makeRecorder({ + fetchJson: async (path: string): Promise => { + if (path.includes("/summary")) return SUMMARY_FIXTURE as unknown as T; + if (path.includes("/tool-breakdown")) + return [{ tool_name: "search-docs" }] as unknown as T; // missing count + if (path.includes("/empty-queries")) + return EMPTY_QUERIES_FIXTURE as unknown as T; + if (path.includes("/queries")) return QUERIES_FIXTURE as unknown as T; + throw new Error(`unexpected path ${path}`); + }, + }); + await runCatchingExit(rec.deps); + expect(rec.exitCodes).toContain(1); + expect(rec.slackCalls.length).toBeGreaterThan(0); + expect(rec.notionCalls).toHaveLength(0); + }); +}); + +describe("fail-loud: Notion publish failure after a successful fetch", () => { + it("exits non-zero and attempts Slack when publishNotion throws", async () => { + const rec = makeRecorder({ + publishNotion: async () => { + throw new Error("Notion 502"); + }, + }); + await runCatchingExit(rec.deps); + expect(rec.exitCodes).toContain(1); + expect(rec.slackCalls.length).toBeGreaterThan(0); + }); +}); + +describe("fail-loud: --report artifact write failure", () => { + it("exits non-zero, attempts Slack, and NEVER publishes Notion when the --report write throws", async () => { + const rec = makeRecorder({ + writeReport: () => { + throw new Error("EACCES: permission denied writing /tmp/x.md"); + }, + }); + rec.deps.argv = ["node", "script", "--report", "/tmp/x.md"]; + await runCatchingExit(rec.deps); + + expect(rec.exitCodes).toContain(1); + // Slack alert attempted with a greppable failure reason. + expect(rec.slackCalls.length).toBeGreaterThan(0); + expect(rec.slackCalls[0]).toMatch(/FAILED/i); + // The whole point of fail-loud symmetry: a write failure must NOT continue + // on to publish a Notion page. + expect(rec.notionCalls).toHaveLength(0); + }); +}); + +// ── Happy path: a full successful run publishes exactly one Notion page ─────── + +describe("happy path", () => { + it("fetches all four endpoints and publishes one Notion page (no Slack, no non-zero exit)", async () => { + const rec = makeRecorder(); + await runCatchingExit(rec.deps); + + expect(rec.fetchedPaths.some((p) => p.includes("/summary"))).toBe(true); + expect(rec.fetchedPaths.some((p) => p.includes("/queries"))).toBe(true); + expect(rec.fetchedPaths.some((p) => p.includes("/empty-queries"))).toBe( + true, + ); + expect(rec.fetchedPaths.some((p) => p.includes("/tool-breakdown"))).toBe( + true, + ); + expect(rec.notionCalls).toHaveLength(1); + expect(rec.slackCalls).toHaveLength(0); + expect(rec.exitCodes).not.toContain(1); + // The title is window-accurate: a true Mon–Sun calendar-week run renders + // "Week of ", otherwise "N days ending ". run() uses the real + // run-instant, so accept either valid form rather than assuming the day. + expect(rec.notionCalls[0].title).toMatch( + /^Pathfinder Search Query Report — (Week of \d{4}-\d{2}-\d{2}|\d+ days ending \d{4}-\d{2}-\d{2})$/, + ); + }); + + it("writes the rendered markdown to disk when --report is supplied", async () => { + const { mkdtempSync, existsSync, readFileSync, rmSync } = + await import("node:fs"); + const { tmpdir } = await import("node:os"); + const { join } = await import("node:path"); + const dir = mkdtempSync(join(tmpdir(), "weekly-report-")); + const reportPath = join(dir, "weekly.md"); + try { + const rec = makeRecorder(); + rec.deps.argv = ["node", "script", "--report", reportPath]; + await runCatchingExit(rec.deps); + expect(existsSync(reportPath)).toBe(true); + const md = readFileSync(reportPath, "utf-8"); + expect(md).toContain("Pathfinder Search Query Report"); + } finally { + rmSync(dir, { recursive: true, force: true }); + } + }); +}); + +// ── parseReportDays ─────────────────────────────────────────────────────────── + +describe("parseReportDays", () => { + it("defaults to 7 when unset or empty", () => { + expect(parseReportDays(undefined)).toBe(7); + expect(parseReportDays("")).toBe(7); + }); + + it("parses a valid positive integer", () => { + expect(parseReportDays("14")).toBe(14); + }); + + it("falls back to 7 on junk / fractional / non-positive", () => { + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + expect(parseReportDays("7.5")).toBe(7); + expect(parseReportDays("-3")).toBe(7); + expect(parseReportDays("0")).toBe(7); + expect(parseReportDays("abc")).toBe(7); + warn.mockRestore(); + }); +}); + +// ── reportPathArgFrom ───────────────────────────────────────────────────────-- + +describe("reportPathArgFrom", () => { + it("returns the resolved path for a normal value", () => { + const r = reportPathArgFrom(["node", "s", "--report", "/tmp/x.md"]); + expect(r).not.toBeNull(); + expect(r!.endsWith("/tmp/x.md")).toBe(true); + }); + + it("returns null when --report is absent or followed by a flag", () => { + expect(reportPathArgFrom(["node", "s"])).toBeNull(); + expect(reportPathArgFrom(["node", "s", "--report"])).toBeNull(); + const warn = vi.spyOn(console, "warn").mockImplementation(() => {}); + expect(reportPathArgFrom(["node", "s", "--report", "--dry"])).toBeNull(); + warn.mockRestore(); + }); +}); + +// ── reportWindow / reportTitle ──────────────────────────────────────────────── +// +// reportWindow is the single source of truth for the data window; it must match +// the server's buildDateWindow (src/db/analytics.ts) for a ?days=N rolling +// request EXACTLY: an inclusive N-UTC-calendar-day window [today-(N-1) .. today]. +// end = UTC date of `now`; start = end - (days - 1). A "calendar week" is only a +// true Mon(start)–Sun(end) span. + +describe("reportWindow", () => { + it("derives an inclusive N-calendar-day window: start = end - (days - 1), not now - days", () => { + // 2026-06-21 is a Sunday. days=7 → start = 06-21 - 6 = Mon 06-15. + const w7 = reportWindow(new Date("2026-06-21T09:07:00Z"), 7); + expect(w7.end).toBe("2026-06-21"); + expect(w7.start).toBe("2026-06-15"); + expect(w7.isCalendarWeek).toBe(true); + + // days=14 → start = 06-21 - 13 = 06-08 (NOT 06-07, the now-days off-by-one). + const w14 = reportWindow(new Date("2026-06-21T09:07:00Z"), 14); + expect(w14.end).toBe("2026-06-21"); + expect(w14.start).toBe("2026-06-08"); + expect(w14.isCalendarWeek).toBe(false); + }); + + it("treats a NON-Sunday 7-day run as NOT a calendar week", () => { + // 2026-06-22 is a Monday → a 7-day window ending here is Tue..Mon, not Mon..Sun. + const w = reportWindow(new Date("2026-06-22T09:07:00Z"), 7); + expect(w.end).toBe("2026-06-22"); + expect(w.start).toBe("2026-06-16"); + expect(w.isCalendarWeek).toBe(false); + }); +}); + +describe("reportTitle", () => { + it("formats the title with the Monday of the window for a true Mon–Sun week", () => { + expect(reportTitle(new Date("2026-06-21T09:07:00Z"), 7)).toBe( + "Pathfinder Search Query Report — Week of 2026-06-15", + ); + }); + + it("uses the explicit N-days-ending framing for a non-7-day window", () => { + const title = reportTitle(new Date("2026-06-21T09:07:00Z"), 14); + expect(title).not.toContain("Week of"); + expect(title).toBe( + "Pathfinder Search Query Report — 14 days ending 2026-06-21", + ); + }); + + it("uses the N-days-ending framing for a 7-day run on a NON-Sunday (not a calendar week)", () => { + // 2026-06-22 is a Monday — a 7-day window here is not Mon–Sun. + const title = reportTitle(new Date("2026-06-22T09:07:00Z"), 7); + expect(title).not.toContain("Week of"); + expect(title).toBe( + "Pathfinder Search Query Report — 7 days ending 2026-06-22", + ); + }); +}); + +describe("renderMarkdown window banner", () => { + const bundle: AnalyticsBundle = { + summary: SUMMARY_FIXTURE, + queries: QUERIES_FIXTURE, + emptyQueries: EMPTY_QUERIES_FIXTURE, + toolBreakdown: TOOL_BREAKDOWN_FIXTURE, + }; + + it("renders a calendar-week banner+title for the default 7-day window on a Sunday", () => { + const md = renderMarkdown(bundle, new Date("2026-06-21T09:07:00Z"), 7); + // Banner shows the true Mon–Sun calendar span and the (week of ) tag. + expect(md).toContain("Window: 7 days (2026-06-15 – 2026-06-21)"); + expect(md).toContain("(week of 2026-06-15)"); + // The H1 title agrees: the canonical "Week of " phrasing. + expect(md).toContain( + "# Pathfinder Search Query Report — Week of 2026-06-15", + ); + }); + + it("renders the corrected (off-by-one-free) span for a 14-day window — start = end - 13", () => { + const md = renderMarkdown(bundle, new Date("2026-06-21T09:07:00Z"), 14); + // start = 06-21 - 13 = 06-08 (NOT 06-07 from the old now-days math). + expect(md).toContain("Window: 14 days (2026-06-08 – 2026-06-21)"); + expect(md).not.toContain("2026-06-07"); + // Neither banner nor title claims a week. + expect(md).not.toContain("week of"); + expect(md).not.toContain("Week of"); + expect(md).toContain( + "# Pathfinder Search Query Report — 14 days ending 2026-06-21", + ); + }); + + it("does NOT print 'Week of' for a 7-day run on a NON-Sunday (not a calendar week)", () => { + // 2026-06-22 is a Monday: the 7-day window ending here is Tue..Mon, not Mon..Sun. + const md = renderMarkdown(bundle, new Date("2026-06-22T09:07:00Z"), 7); + expect(md).toContain("Window: 7 days (2026-06-16 – 2026-06-22)"); + expect(md).not.toContain("week of"); + expect(md).not.toContain("Week of"); + // It renders the N-day-ending form instead. + expect(md).toContain( + "# Pathfinder Search Query Report — 7 days ending 2026-06-22", + ); + }); +}); + +// ── categorization (deterministic taxonomy bucketing) ───────────────────────── + +describe("categorizeQuery", () => { + it("buckets known phrases into their taxonomy category", () => { + expect(categorizeQuery("how to use CoAgents with LangGraph")).toBe( + "Agents/CoAgents/AG-UI", + ); + expect(categorizeQuery("CopilotKit runtime backend setup")).toBe( + "Runtime/Backend", + ); + expect(categorizeQuery("useCopilotAction frontend tool example")).toBe( + "Actions/Frontend tools", + ); + expect(categorizeQuery("how to theme the chat ui with css")).toBe( + "Theming/CSS", + ); + expect(categorizeQuery("v2 migration useCopilotChat hook")).toBe( + "v2 Migration/Hooks", + ); + expect(categorizeQuery("human in the loop interrupt approval")).toBe( + "Human-in-the-loop", + ); + expect(categorizeQuery("MCP middleware configuration")).toBe( + "MCP/Middleware", + ); + expect(categorizeQuery("generative ui rendering custom component")).toBe( + "Generative UI/Rendering", + ); + expect(categorizeQuery("streaming events tool call output")).toBe( + "Streaming/Events", + ); + expect(categorizeQuery("getting started quickstart install")).toBe( + "Getting started/Setup", + ); + }); + + it("falls back to Other for an unmatched query", () => { + expect(categorizeQuery("something completely unrelated and weird")).toBe( + "Other", + ); + }); + + it("is case-insensitive", () => { + expect(categorizeQuery("COAGENTS langgraph")).toBe("Agents/CoAgents/AG-UI"); + }); + + it("every taxonomy bucket name is a valid category (Other always present)", () => { + const names = CATEGORY_TAXONOMY.map((c) => c.category); + expect(names).toContain("Other"); + }); +}); + +describe("categorizeQueries (aggregation by frequency-weighted count)", () => { + it("sums query counts into their categories, sorted by count desc", () => { + const cats = categorizeQueries(QUERIES_FIXTURE); + // Highest single category in the fixture should appear first. + expect(cats.length).toBeGreaterThan(0); + for (let i = 1; i < cats.length; i++) { + expect(cats[i - 1].count).toBeGreaterThanOrEqual(cats[i].count); + } + // The unrelated query (count 3) lands in Other. + const other = cats.find((c) => c.category === "Other"); + expect(other).toBeDefined(); + expect(other!.count).toBe(3); + }); +}); + +// ── topNQueries (top-20 ordering by frequency) ──────────────────────────────── + +describe("topNQueries", () => { + it("returns queries sorted by count desc, capped at N", () => { + const top = topNQueries(QUERIES_FIXTURE, 5); + expect(top).toHaveLength(5); + for (let i = 1; i < top.length; i++) { + expect(top[i - 1].count).toBeGreaterThanOrEqual(top[i].count); + } + expect(top[0].query_text).toBe("how to use CoAgents with LangGraph"); + }); + + it("does not mutate the input array", () => { + const copy = [...QUERIES_FIXTURE]; + topNQueries(QUERIES_FIXTURE, 3); + expect(QUERIES_FIXTURE).toEqual(copy); + }); +}); + +// ── tool breakdown helpers ──────────────────────────────────────────────────── + +describe("exploreBreakdown / searchVsExploreSplit", () => { + it("extracts only explore-* rows for the explore breakdown", () => { + const explore = exploreBreakdown(TOOL_BREAKDOWN_FIXTURE); + expect(explore.map((r) => r.tool_name)).toEqual([ + "explore-bash", + "explore-grep", + ]); + }); + + it("splits total counts into search vs explore by name prefix", () => { + const split = searchVsExploreSplit(TOOL_BREAKDOWN_FIXTURE); + // search-docs 700 + search-code 300 + search-ag-ui-docs 120 = 1120 + expect(split.search).toBe(1120); + // explore-bash 80 + explore-grep 34 = 114 + expect(split.explore).toBe(114); + }); +}); + +// ── cell rendering safety (nullable source_name + sanitizeCell hardening) ───── + +describe("cell rendering safety", () => { + it("renders a null source_name as a safe cell, not the literal string 'null'", () => { + const emptyQueries: EmptyQuery[] = [ + { + query_text: "some query with no source", + tool_name: "search-docs", + source_name: null, + count: 7, + last_seen: "2026-06-21T10:00:00Z", + }, + ]; + const bundle: AnalyticsBundle = { + summary: SUMMARY_FIXTURE, + queries: QUERIES_FIXTURE, + emptyQueries, + toolBreakdown: TOOL_BREAKDOWN_FIXTURE, + }; + const md = renderMarkdown(bundle, new Date("2026-06-21T09:07:00Z"), 7); + // The NULL source must NOT publish the literal text "null" into the cell. + expect(md).not.toContain("| null |"); + // The query row is still rendered (with an empty/(none) source cell). + expect(md).toContain("some query with no source"); + }); + + it("sanitizeCell neutralizes \\r and \\r\\n (no raw carriage returns survive)", () => { + const out = sanitizeCell("a\r\nb\rc"); + expect(out).not.toContain("\r"); + expect(out).not.toContain("\n"); + }); + + it("sanitizeCell escapes pipes in a source-style value", () => { + expect(sanitizeCell("foo|bar")).toBe("foo\\|bar"); + }); + + it("buildObservations does not split an empty-query observation across lines on a newline in query_text", () => { + const emptyQueries: EmptyQuery[] = [ + { + query_text: "line one\nline two", + tool_name: "search-docs", + source_name: "claude-code", + count: 99, + last_seen: "2026-06-21T10:00:00Z", + }, + ]; + const bundle: AnalyticsBundle = { + summary: SUMMARY_FIXTURE, + queries: QUERIES_FIXTURE, + emptyQueries, + toolBreakdown: TOOL_BREAKDOWN_FIXTURE, + }; + const obs = buildObservations(bundle); + const emptyObs = obs.find((o) => + o.includes("Highest-frequency empty query"), + ); + expect(emptyObs).toBeDefined(); + // A newline in query_text must not split the single observation into 2 lines. + expect(emptyObs!).not.toContain("\n"); + }); + + // ── A2: observation bullets must render a literal pipe, never `\|` ────────── + // + // Observations become Notion bulleted_list_item blocks, whose block path does + // NOT unescape `\|` (only the table-cell path does). So a `|` in an + // observation's source text must NOT be markdown-pipe-escaped, or the Notion + // bullet shows the literal backslash. The bullet must carry a real `|`. + it("renders a literal pipe (not \\|) in an observation bullet block when query_text contains a pipe", () => { + const emptyQueries: EmptyQuery[] = [ + { + query_text: "a | b pipe query", + tool_name: "search-docs", + source_name: "claude-code", + count: 99, + last_seen: "2026-06-21T10:00:00Z", + }, + ]; + const bundle: AnalyticsBundle = { + summary: SUMMARY_FIXTURE, + queries: QUERIES_FIXTURE, + emptyQueries, + toolBreakdown: TOOL_BREAKDOWN_FIXTURE, + }; + const md = renderMarkdown(bundle, new Date("2026-06-21T09:07:00Z"), 7); + const blocks = markdownToNotionBlocks(md); + const bullet = blocks.find( + (b) => + b.type === "bulleted_list_item" && + blockText(b).includes("Highest-frequency empty query"), + ); + expect(bullet).toBeDefined(); + // The produced Notion bullet must carry a real pipe, never the escaped form. + expect(blockText(bullet!)).toContain("a | b"); + expect(blockText(bullet!)).not.toContain("\\|"); + }); + + // ── A5: the Top-20 table must escape tool_name like every other text cell ── + it("escapes a pipe in a Top-20 row tool_name so the row cannot be corrupted", () => { + const queries = [ + { + query_text: "frequent query", + tool_name: "search|weird", + count: 9999, + avg_result_count: null, + avg_top_score: null, + }, + ]; + const bundle: AnalyticsBundle = { + summary: SUMMARY_FIXTURE, + queries, + emptyQueries: EMPTY_QUERIES_FIXTURE, + toolBreakdown: TOOL_BREAKDOWN_FIXTURE, + }; + const md = renderMarkdown(bundle, new Date("2026-06-21T09:07:00Z"), 7); + // The raw tool_name pipe must be escaped in the markdown table source so it + // does not split the row into an extra column. + expect(md).toContain("search\\|weird"); + expect(md).not.toContain("| search|weird |"); + // And the rendered Notion table cell unescapes back to a literal pipe. + const table = markdownToNotionBlocks(md).find( + (b) => + b.type === "table" && + ((b as any).table.children[0].table_row.cells[0][0]?.text?.content ?? + "") === "Query", + ) as any; + expect(table).toBeDefined(); + const toolCell = table.table.children[1].table_row.cells[1] + .map((rt: any) => rt.text.content) + .join(""); + expect(toolCell).toBe("search|weird"); + }); +}); + +// ── markdownToNotionBlocks / batchBlocks (reused renderer parity) ────────────── + +function blockText(block: any): string { + const rich = block[block.type]?.rich_text ?? []; + return rich.map((r: any) => r.text.content).join(""); +} + +describe("markdownToNotionBlocks", () => { + it("maps headings and bullets to native block types and drops the leading title H1", () => { + const md = [ + "# Pathfinder Search Query Report — Week of 2026-06-15", + "", + "## Summary", + "- Total tool calls: 5", + "### Sub", + "plain prose", + ].join("\n"); + const blocks = markdownToNotionBlocks(md); + expect(blocks.some((b) => b.type === "heading_1")).toBe(false); + expect(blocks.map((b) => b.type)).toEqual([ + "heading_2", + "bulleted_list_item", + "heading_3", + "paragraph", + ]); + expect(blockText(blocks[0])).toBe("Summary"); + }); + + it("splits a line over the 2000-char cap across rich_text objects", () => { + const longLine = "x".repeat(NOTION_RICH_TEXT_LIMIT * 2 + 5); + const blocks = markdownToNotionBlocks(longLine); + const rich = (blocks[0] as any).paragraph.rich_text; + expect(rich.length).toBeGreaterThan(1); + for (const r of rich) { + expect(r.text.content.length).toBeLessThanOrEqual(NOTION_RICH_TEXT_LIMIT); + } + expect(rich.map((r: any) => r.text.content).join("")).toBe(longLine); + }); + + it("renders a markdown table as a native Notion table block (not pipe-text paragraphs)", () => { + const md = [ + "## Activity by day", + "| Day | Tool calls |", + "| --- | --- |", + "| 2026-06-15 | 12 |", + "| 2026-06-16 | 8 |", + ].join("\n"); + const blocks = markdownToNotionBlocks(md); + + // No paragraph block should carry the raw pipe text. + expect( + blocks.some((b) => b.type === "paragraph" && blockText(b).includes("|")), + ).toBe(false); + + const table = blocks.find((b) => b.type === "table") as any; + expect(table).toBeDefined(); + expect(table.table.table_width).toBe(2); + expect(table.table.has_column_header).toBe(true); + // header + 2 data rows (separator row dropped). + const rows = table.table.children; + expect(rows.length).toBe(3); + for (const r of rows) { + expect(r.type).toBe("table_row"); + expect(r.table_row.cells.length).toBe(2); + } + const cellText = (row: any, col: number): string => + row.table_row.cells[col].map((rt: any) => rt.text.content).join(""); + expect(cellText(rows[0], 0)).toBe("Day"); + expect(cellText(rows[0], 1)).toBe("Tool calls"); + expect(cellText(rows[1], 0)).toBe("2026-06-15"); + expect(cellText(rows[1], 1)).toBe("12"); + expect(cellText(rows[2], 0)).toBe("2026-06-16"); + expect(cellText(rows[2], 1)).toBe("8"); + }); + + it("unescapes markdown-escaped pipes in table cells", () => { + const md = ["| Query | Count |", "| --- | --- |", "| a \\| b | 3 |"].join( + "\n", + ); + const table = markdownToNotionBlocks(md).find( + (b) => b.type === "table", + ) as any; + const content = table.table.children[1].table_row.cells[0][0].text.content; + expect(content).toBe("a | b"); + }); + + it("caps a table at 100 data rows and appends a truncation note", () => { + const lines = ["| Day | Count |", "| --- | --- |"]; + for (let i = 0; i < 150; i++) lines.push(`| d${i} | ${i} |`); + const blocks = markdownToNotionBlocks(lines.join("\n")); + const table = blocks.find((b) => b.type === "table") as any; + // header + 100 data rows + expect(table.table.children.length).toBe(101); + const note = blocks.find( + (b) => b.type === "paragraph" && blockText(b).includes("truncated"), + ); + expect(note).toBeDefined(); + expect(blockText(note!)).toContain("150"); + }); +}); + +describe("batchBlocks", () => { + it("splits >100 blocks into batches of at most 100", () => { + expect(NOTION_MAX_BLOCKS_PER_REQUEST).toBe(100); + const blocks = Array.from({ length: 250 }, (_, i) => ({ id: i })); + const batches = batchBlocks(blocks, NOTION_MAX_BLOCKS_PER_REQUEST); + expect(batches.map((b) => b.length)).toEqual([100, 100, 50]); + expect(batches.flat()).toEqual(blocks); + }); +}); + +// ── Notion publish: partial-page archive on multi-batch append failure ──────── +// +// Notion has no transactional multi-batch create: the page is created with the +// first 100-block batch, then remaining batches are appended. If an append +// throws, the page already exists, so a degraded/orphaned page would be left +// behind while the run reports failure — violating the "never publish a degraded +// page" promise. publishNotionWithClient must best-effort archive that partial +// page, then re-throw the original append error so run() still fails loud. + +describe("publishNotionWithClient: archive partial page on append failure", () => { + // A markdown report large enough to span >100 blocks (forces multi-batch: + // each non-blank line becomes one block). + const multiBatchMarkdown = [ + "# Title (dropped — duplicates page title)", + ...Array.from({ length: 150 }, (_, i) => `- bullet ${i}`), + ].join("\n"); + + function makeFakeClient(): { + client: NotionClientLike; + createCalls: number; + appendCalls: number; + archiveCalls: Array<{ page_id: string; archived: boolean }>; + } { + const archiveCalls: Array<{ page_id: string; archived: boolean }> = []; + let createCalls = 0; + let appendCalls = 0; + const client: NotionClientLike = { + pages: { + create: async () => { + createCalls += 1; + return { id: "page-abc", url: "https://notion.example/page-abc" }; + }, + update: async (args) => { + archiveCalls.push({ + page_id: args.page_id, + archived: args.archived, + }); + return {}; + }, + }, + blocks: { + children: { + append: async () => { + appendCalls += 1; + // First append (the 2nd batch overall) throws. + throw new Error("Notion 502 on append batch"); + }, + }, + }, + }; + return { + client, + get createCalls() { + return createCalls; + }, + get appendCalls() { + return appendCalls; + }, + archiveCalls, + }; + } + + it("re-throws the original append error AND archives the just-created partial page", async () => { + const fake = makeFakeClient(); + await expect( + publishNotionWithClient( + fake.client, + "parent-123", + "My Report Title", + multiBatchMarkdown, + ), + ).rejects.toThrow("Notion 502 on append batch"); + + // The page was created and an append was attempted (multi-batch). + expect(fake.createCalls).toBe(1); + expect(fake.appendCalls).toBe(1); + // The partial page must have been archived best-effort. + expect(fake.archiveCalls).toEqual([ + { page_id: "page-abc", archived: true }, + ]); + }); + + it("returns the page url on the happy path and never archives", async () => { + const archiveCalls: Array<{ page_id: string; archived: boolean }> = []; + const client: NotionClientLike = { + pages: { + create: async () => ({ + id: "page-ok", + url: "https://notion.example/page-ok", + }), + update: async (args) => { + archiveCalls.push({ + page_id: args.page_id, + archived: args.archived, + }); + return {}; + }, + }, + blocks: { + children: { + append: async () => ({}), + }, + }, + }; + const url = await publishNotionWithClient( + client, + "parent-123", + "My Report Title", + multiBatchMarkdown, + ); + expect(url).toBe("https://notion.example/page-ok"); + expect(archiveCalls).toEqual([]); + }); + + it("re-throws the original append error even when the archive attempt itself fails", async () => { + const client: NotionClientLike = { + pages: { + create: async () => ({ id: "page-xyz", url: null }), + update: async () => { + throw new Error("archive also failed"); + }, + }, + blocks: { + children: { + append: async () => { + throw new Error("original append error"); + }, + }, + }, + }; + await expect( + publishNotionWithClient( + client, + "parent-123", + "My Report Title", + multiBatchMarkdown, + ), + ).rejects.toThrow("original append error"); + }); +}); diff --git a/scripts/weekly-search-report/weekly-search-report.ts b/scripts/weekly-search-report/weekly-search-report.ts new file mode 100644 index 0000000..ef8ab1a --- /dev/null +++ b/scripts/weekly-search-report/weekly-search-report.ts @@ -0,0 +1,1228 @@ +#!/usr/bin/env tsx +/// +/** + * weekly-search-report.ts + * + * Weekly (7-day lookback) Pathfinder search-query report. Runs from a scheduled + * GitHub Action and publishes a deterministic (no-LLM) report to Notion. + * + * This REPLACES a brittle claude.ai routine that scraped ephemeral Railway + * stdout logs and, when its token was missing, silently posted a Notion "error + * page" masquerading as a report. The two structural fixes here: + * + * 1. DURABLE SOURCE. It reads the query_log-backed analytics JSON API (the + * same source as monthly-gap-analysis), not Railway stdout. + * 2. FAIL LOUD. A run that cannot fetch its data — missing token, a non-2xx + * from any required endpoint, or a malformed payload — logs a greppable + * "[weekly-report] FAILED: ", POSTs a Slack alert, and exits + * non-zero. It NEVER publishes a degraded/error Notion page. A Notion + * publish failure AFTER a good fetch is also fail-loud. + * + * Unlike gap-analysis, there is NO LLM step: categorization is deterministic + * keyword bucketing against the CATEGORY_TAXONOMY constant below, so the report + * needs no ANTHROPIC_API_KEY. + * + * Secrets / env: + * PATHFINDER_ANALYTICS_TOKEN Bearer token for the analytics API. REQUIRED — + * a missing/empty value is a fail-loud condition + * (this is the exact 2026-06-21 failure mode). + * NOTION_TOKEN Notion integration token (required to publish). + * NOTION_PARENT_PAGE_ID Parent page id (default the Pathfinder project + * page). + * SLACK_WEBHOOK Incoming-webhook URL the script posts FAILURE + * alerts to. The WORKFLOW maps the org secret + * (SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }}). + * If unset, the alert no-ops but the non-zero exit + * still stands. + * ANALYTICS_BASE_URL Override the analytics host (default prod). + * REPORT_DAYS Lookback window in days (default 7). + * + * Usage: + * npx tsx scripts/weekly-search-report/weekly-search-report.ts + * npx tsx scripts/weekly-search-report/weekly-search-report.ts --report /tmp/weekly.md + */ + +import { writeFileSync, mkdirSync } from "node:fs"; +import { resolve, dirname } from "node:path"; +import { fileURLToPath } from "node:url"; + +// ── Types: the analytics API CONTRACT (build fixtures to match) ─────────────── + +export interface QueriesPerDay { + day: string; + count: number; +} + +export interface AnalyticsSummary { + total_queries_window: number; + unique_ip_count_window: number; + unique_session_count_window: number; + empty_result_count_window: number; + empty_result_rate_window: number; + low_confidence_count_window: number; + low_confidence_rate_window: number; + avg_latency_ms_window: number; + p95_latency_ms_window: number; + queries_by_source: Array<{ source_name: string; count: number }>; + queries_per_day_window: QueriesPerDay[]; + earliest_query_day?: string | null; +} + +/** `/queries` row — only queries with >=1 result appear here. */ +export interface TopQuery { + query_text: string; + tool_name: string; + count: number; + // The analytics API returns these averages nullable (e.g. NULL when no + // scored rows). They are not rendered, but the type must match the contract. + avg_result_count: number | null; + avg_top_score: number | null; +} + +/** `/empty-queries` row. */ +export interface EmptyQuery { + query_text: string; + tool_name: string; + // The analytics API returns source_name nullable — a NULL must render as an + // empty cell, NOT the literal text "null". + source_name: string | null; + count: number; + last_seen: string; +} + +/** `/tool-breakdown` row (count desc). */ +export interface ToolBreakdownRow { + tool_name: string; + count: number; +} + +export interface AnalyticsBundle { + summary: AnalyticsSummary; + queries: TopQuery[]; + emptyQueries: EmptyQuery[]; + toolBreakdown: ToolBreakdownRow[]; +} + +// ── Keyword categorization taxonomy (deterministic, NO LLM) ─────────────────── +// +// Ordered, first-match-wins list of (category, keyword[]) rules. The buckets +// mirror the taxonomy the retired routine used. "Other" is the implicit +// fallback when no keyword matches; it is listed last with an empty keyword set +// so the category name is enumerable. + +export interface CategoryRule { + category: string; + keywords: string[]; +} + +export const CATEGORY_TAXONOMY: CategoryRule[] = [ + { + category: "Agents/CoAgents/AG-UI", + keywords: ["coagent", "co-agent", "ag-ui", "agui", "langgraph", "agent"], + }, + { + category: "Runtime/Backend", + keywords: ["runtime", "backend", "server", "endpoint", "api route"], + }, + { + category: "Actions/Frontend tools", + keywords: [ + "usecopilotaction", + "copilotaction", + "frontend tool", + "frontend action", + "action", + ], + }, + { + category: "Theming/CSS", + keywords: ["theme", "theming", "css", "style", "styling"], + }, + { + category: "Generative UI/Rendering", + keywords: ["generative ui", "render", "rendering", "renderandwait"], + }, + { + category: "v2 Migration/Hooks", + keywords: ["v2", "migration", "migrate", "usecopilotchat", "hook"], + }, + { + category: "Chat UI/Components", + keywords: [ + "copilotchat", + "copilotpopup", + "copilotsidebar", + "chat ui", + "component", + ], + }, + { + category: "Getting started/Setup", + keywords: [ + "getting started", + "quickstart", + "quick start", + "install", + "setup", + "tutorial", + ], + }, + { + category: "State/Context", + keywords: ["usecoagent", "shared state", "readable", "state", "context"], + }, + { + category: "Human-in-the-loop", + keywords: [ + "human in the loop", + "human-in-the-loop", + "hitl", + "interrupt", + "approval", + ], + }, + { + category: "MCP/Middleware", + keywords: ["mcp", "middleware"], + }, + { + category: "Streaming/Events", + keywords: ["stream", "streaming", "event", "sse"], + }, + // Implicit fallback — listed so the bucket name is enumerable. No keywords. + { + category: "Other", + keywords: [], + }, +]; + +/** Deterministically categorize one query_text into a taxonomy bucket. */ +export function categorizeQuery(queryText: string): string { + const text = queryText.toLowerCase(); + for (const rule of CATEGORY_TAXONOMY) { + if (rule.keywords.length === 0) continue; // skip the "Other" fallback + if (rule.keywords.some((kw) => text.includes(kw))) { + return rule.category; + } + } + return "Other"; +} + +export interface CategoryCount { + category: string; + count: number; +} + +/** + * Aggregate `/queries` rows into their categories, weighted by each row's + * frequency `count`. Returned sorted by count desc. Categories with zero hits + * are omitted. + */ +export function categorizeQueries(queries: TopQuery[]): CategoryCount[] { + const totals = new Map(); + for (const q of queries) { + const cat = categorizeQuery(q.query_text); + totals.set(cat, (totals.get(cat) ?? 0) + q.count); + } + return [...totals.entries()] + .map(([category, count]) => ({ category, count })) + .sort((a, b) => b.count - a.count); +} + +/** Top-N `/queries` by frequency (count desc). Does not mutate the input. */ +export function topNQueries(queries: TopQuery[], n: number): TopQuery[] { + return [...queries].sort((a, b) => b.count - a.count).slice(0, n); +} + +/** The explore-* rows of a tool breakdown (the bash/explore command breakdown). */ +export function exploreBreakdown(rows: ToolBreakdownRow[]): ToolBreakdownRow[] { + return rows.filter((r) => r.tool_name.startsWith("explore-")); +} + +/** Split total tool-call counts into search-vs-explore by tool-name prefix. */ +export function searchVsExploreSplit(rows: ToolBreakdownRow[]): { + search: number; + explore: number; + other: number; +} { + let search = 0; + let explore = 0; + let other = 0; + for (const r of rows) { + if (r.tool_name.startsWith("search-")) search += r.count; + else if (r.tool_name.startsWith("explore-")) explore += r.count; + else other += r.count; + } + return { search, explore, other }; +} + +// ── Date / window / title ───────────────────────────────────────────────────-- + +/** + * The single source of truth for the report's data window. Computes the window + * EXACTLY as the server's `buildDateWindow` (src/db/analytics.ts) does for a + * `?days=N` rolling request, so every label that names the window — the in-body + * banner AND the page title/H1 — derives from the same dates and can never + * contradict each other or the data that was actually queried. + * + * Server semantics (rolling mode): the read WHERE is + * created_at >= (NOW() AT TIME ZONE 'UTC')::date - (N - 1) + * i.e. an INCLUSIVE N-UTC-calendar-day window `[today-(N-1) .. today]`. So: + * - `end` = the UTC calendar date of `now` (YYYY-MM-DD). + * - `start` = `end - (days - 1)` UTC calendar days (NOT `now - days`, which is + * one day too early and was the source of the off-by-one banner). + * - `isCalendarWeek` = true ONLY when the window is a true Mon–Sun calendar + * week: `days === 7` AND `start` is a Monday AND `end` is a Sunday. + * The production Sunday cron with REPORT_DAYS=7 satisfies this; a + * manual / retried / clock-drifted non-Sunday 7-day run does not, + * so it must NOT be mislabeled "Week of …". + * + * Uses UTC throughout so the GHA-cron date is stable regardless of runner TZ. + */ +export function reportWindow( + now: Date, + days: number, +): { start: string; end: string; isCalendarWeek: boolean } { + // Anchor to the UTC calendar date of `now` (drop the time-of-day). + const endDate = new Date( + Date.UTC(now.getUTCFullYear(), now.getUTCMonth(), now.getUTCDate()), + ); + // Inclusive N-day window ending today → start is N-1 days earlier. + const startDate = new Date(endDate.getTime()); + startDate.setUTCDate(startDate.getUTCDate() - (days - 1)); + + const start = startDate.toISOString().slice(0, 10); + const end = endDate.toISOString().slice(0, 10); + + // getUTCDay(): 0=Sun..6=Sat. A true calendar week is exactly 7 days running + // Monday(start) .. Sunday(end). + const isCalendarWeek = + days === 7 && startDate.getUTCDay() === 1 && endDate.getUTCDay() === 0; + + return { start, end, isCalendarWeek }; +} + +/** + * The report's page title / H1, derived from {@link reportWindow} so it never + * contradicts the in-body banner. "Week of " is used ONLY for a true + * Mon–Sun calendar week (the production default); every other window uses the + * explicit " days ending " framing. + */ +export function reportTitle(now: Date, days: number): string { + const prefix = "Pathfinder Search Query Report —"; + const { start, end, isCalendarWeek } = reportWindow(now, days); + if (isCalendarWeek) { + return `${prefix} Week of ${start}`; + } + return `${prefix} ${days} days ending ${end}`; +} + +// ── days / argv parsing ─────────────────────────────────────────────────────── + +/** + * Parse REPORT_DAYS strictly (positive integer); anything else falls back to the + * 7-day default with a warning rather than passing a bad value to the API. + */ +export function parseReportDays(raw: string | undefined): number { + const DEFAULT_DAYS = 7; + if (raw === undefined || raw.trim() === "") return DEFAULT_DAYS; + const trimmed = raw.trim(); + if (!/^-?\d+$/.test(trimmed)) { + console.warn( + `[weekly-report] REPORT_DAYS="${raw}" is not a valid integer — using default ${DEFAULT_DAYS}.`, + ); + return DEFAULT_DAYS; + } + const days = Number.parseInt(trimmed, 10); + if (!Number.isInteger(days) || days <= 0) { + console.warn( + `[weekly-report] REPORT_DAYS="${raw}" must be a positive integer — using default ${DEFAULT_DAYS}.`, + ); + return DEFAULT_DAYS; + } + return days; +} + +/** + * Resolve `--report ` from an argv array. Returns null when absent, has no + * following token, or the following token is itself a flag (`--report --dry` + * must not write a file literally named "--dry"). + */ +export function reportPathArgFrom(argv: readonly string[]): string | null { + const idx = argv.indexOf("--report"); + if (idx === -1 || idx + 1 >= argv.length) return null; + const next = argv[idx + 1]; + if (next.startsWith("--")) { + console.warn( + `[weekly-report] --report expects a file path but got the flag "${next}" — ignoring --report.`, + ); + return null; + } + return resolve(next); +} + +// ── Payload validation (malformed → fail loud) ──────────────────────────────── + +function isFiniteNumber(v: unknown): v is number { + return typeof v === "number" && Number.isFinite(v); +} + +function isNonEmptyString(v: unknown): v is string { + return typeof v === "string" && v.length > 0; +} + +/** Throw a descriptive Error if the summary payload is missing required fields. */ +export function assertValidSummary(s: unknown): asserts s is AnalyticsSummary { + if (!s || typeof s !== "object") { + throw new Error("summary payload is not an object"); + } + const o = s as Record; + const requiredNums = [ + "total_queries_window", + "unique_ip_count_window", + "unique_session_count_window", + "empty_result_count_window", + "empty_result_rate_window", + "p95_latency_ms_window", + ]; + for (const k of requiredNums) { + if (!isFiniteNumber(o[k])) { + throw new Error(`summary.${k} missing or not a number`); + } + } + if (!Array.isArray(o.queries_per_day_window)) { + throw new Error("summary.queries_per_day_window is not an array"); + } + o.queries_per_day_window.forEach((row, i) => { + if (!row || typeof row !== "object") { + throw new Error(`summary.queries_per_day_window row ${i}: not an object`); + } + const r = row as Record; + if (!isNonEmptyString(r.day)) { + throw new Error( + `summary.queries_per_day_window row ${i}: day missing or not a non-empty string`, + ); + } + if (!isFiniteNumber(r.count)) { + throw new Error( + `summary.queries_per_day_window row ${i}: count missing or not a number`, + ); + } + }); +} + +/** + * Assert `v` is an array AND every row passes `validateRow` (which throws a + * descriptive Error naming the endpoint + bad field). The array-ness check runs + * first so a non-array payload fails loud with a clear message. + */ +function assertArrayOf( + name: string, + v: unknown, + validateRow: (row: Record, i: number) => void, +): asserts v is T[] { + if (!Array.isArray(v)) { + throw new Error(`${name} payload is not an array`); + } + v.forEach((row, i) => { + if (!row || typeof row !== "object") { + throw new Error(`${name} row ${i}: not an object`); + } + validateRow(row as Record, i); + }); +} + +/** Validate a `/queries` (TopQuery) row's required fields. */ +function assertTopQueryRow( + name: string, + r: Record, + i: number, +) { + if (typeof r.query_text !== "string") { + throw new Error(`${name} row ${i}: query_text missing or not a string`); + } + if (typeof r.tool_name !== "string") { + throw new Error(`${name} row ${i}: tool_name missing or not a string`); + } + if (!isFiniteNumber(r.count)) { + throw new Error(`${name} row ${i}: count missing or not a number`); + } +} + +/** Validate an `/empty-queries` (EmptyQuery) row's required fields. */ +function assertEmptyQueryRow( + name: string, + r: Record, + i: number, +) { + if (typeof r.query_text !== "string") { + throw new Error(`${name} row ${i}: query_text missing or not a string`); + } + if (typeof r.tool_name !== "string") { + throw new Error(`${name} row ${i}: tool_name missing or not a string`); + } + // source_name may be string|null (handled in rendering by a separate fix); + // reject only other types. + if (r.source_name !== null && typeof r.source_name !== "string") { + throw new Error(`${name} row ${i}: source_name must be a string or null`); + } + if (!isFiniteNumber(r.count)) { + throw new Error(`${name} row ${i}: count missing or not a number`); + } +} + +/** Validate a `/tool-breakdown` (ToolBreakdownRow) row's required fields. */ +function assertToolBreakdownRow( + name: string, + r: Record, + i: number, +) { + if (typeof r.tool_name !== "string") { + throw new Error(`${name} row ${i}: tool_name missing or not a string`); + } + if (!isFiniteNumber(r.count)) { + throw new Error(`${name} row ${i}: count missing or not a number`); + } +} + +// ── Report rendering ────────────────────────────────────────────────────────── + +const TOP_QUERIES_N = 20; + +export function renderMarkdown( + bundle: AnalyticsBundle, + now: Date, + days: number, +): string { + const { summary, queries, emptyQueries, toolBreakdown } = bundle; + const lines: string[] = []; + + // Banner + title both flow from the SAME reportWindow result, so they can + // never disagree with each other or with the queried ?days=N window. The + // window is calendar-aligned exactly as the server's buildDateWindow: + // start = end - (days - 1) (NOT now - days), end = UTC date of now. + const { start, end, isCalendarWeek } = reportWindow(now, days); + // "(week of )" is appended ONLY for a true Mon–Sun calendar week. + const windowDetail = isCalendarWeek ? ` (week of ${start})` : ""; + + lines.push(`# ${reportTitle(now, days)}`); + lines.push(""); + lines.push( + `Window: ${days} days (${start} – ${end})${windowDetail} · ` + + `Source: analytics API (query_log-backed, read-only)`, + ); + lines.push(""); + + // 1. Header metrics + lines.push("## Header metrics"); + lines.push(""); + lines.push(`- Total tool calls: ${summary.total_queries_window}`); + lines.push(`- Unique IPs: ${summary.unique_ip_count_window}`); + lines.push(`- Unique sessions: ${summary.unique_session_count_window}`); + lines.push( + `- Empty-result count: ${summary.empty_result_count_window} ` + + `(${(summary.empty_result_rate_window * 100).toFixed(1)}%)`, + ); + lines.push(`- p95 latency: ${summary.p95_latency_ms_window} ms`); + lines.push(""); + + // 2. Activity by day + lines.push("## Activity by day"); + lines.push(""); + lines.push("| Day | Tool calls |"); + lines.push("| --- | --- |"); + for (const d of summary.queries_per_day_window) { + lines.push(`| ${sanitizeCell(d.day)} | ${d.count} |`); + } + lines.push(""); + + // 3. Tool breakdown + const split = searchVsExploreSplit(toolBreakdown); + lines.push("## Tool breakdown"); + lines.push(""); + lines.push( + `Search calls: ${split.search} · Explore calls: ${split.explore}` + + (split.other ? ` · Other: ${split.other}` : ""), + ); + lines.push(""); + lines.push("| Tool | Calls |"); + lines.push("| --- | --- |"); + for (const t of toolBreakdown) { + lines.push(`| ${sanitizeCell(t.tool_name)} | ${t.count} |`); + } + lines.push(""); + + // 4. Top query categories + const cats = categorizeQueries(queries); + lines.push("## Top query categories"); + lines.push(""); + lines.push( + "_(categories of queries that returned >=1 result; always-empty traffic is in the Empty-result section)_", + ); + lines.push(""); + lines.push("| Category | Queries |"); + lines.push("| --- | --- |"); + for (const c of cats) { + lines.push(`| ${c.category} | ${c.count} |`); + } + lines.push(""); + + // 5. Top 20 queries + const top = topNQueries(queries, TOP_QUERIES_N); + lines.push(`## Top ${TOP_QUERIES_N} queries by frequency`); + lines.push(""); + lines.push("| Query | Tool | Count |"); + lines.push("| --- | --- | --- |"); + for (const q of top) { + lines.push( + `| ${sanitizeCell(q.query_text)} | ${sanitizeCell(q.tool_name)} | ${q.count} |`, + ); + } + lines.push(""); + + // 6. Empty-result queries + lines.push("## Empty-result queries (what users wanted but didn't find)"); + lines.push(""); + if (emptyQueries.length === 0) { + lines.push("(none)"); + } else { + lines.push("| Query | Tool | Source | Count |"); + lines.push("| --- | --- | --- | --- |"); + for (const e of emptyQueries) { + lines.push( + `| ${sanitizeCell(e.query_text)} | ${sanitizeCell(e.tool_name)} | ${sanitizeCell(e.source_name ?? "(none)")} | ${e.count} |`, + ); + } + } + lines.push(""); + + // 7. Explore / bash command breakdown + const explore = exploreBreakdown(toolBreakdown); + lines.push("## Explore/bash command breakdown"); + lines.push(""); + if (explore.length === 0) { + lines.push("(none)"); + } else { + lines.push("| Command | Calls |"); + lines.push("| --- | --- |"); + for (const e of explore) { + lines.push(`| ${sanitizeCell(e.tool_name)} | ${e.count} |`); + } + } + lines.push(""); + + // 8. Observations (computed ONLY from the current window — stateless) + lines.push("## Observations"); + lines.push(""); + for (const obs of buildObservations(bundle)) { + lines.push(`- ${obs}`); + } + lines.push(""); + + return lines.join("\n"); +} + +/** + * Neutralize a markdown TABLE cell so a pipe in user/analytics text can't break + * the row. Pipes are escaped (`\|`) — the native Notion table path unescapes + * them back via tableCellText — and every flavor of line break (\r\n, \r, \n) + * collapses to a single space so a multi-line value can't split a table row. + * + * Use sanitizeInline (NOT this) for bullet/paragraph text: those block paths do + * NOT unescape `\|`, so escaping a pipe there would publish a literal backslash. + */ +export function sanitizeCell(text: string): string { + return text.replace(/\|/g, "\\|").replace(/\r\n|\r|\n/g, " "); +} + +/** + * Neutralize inline (bullet / paragraph) text. Unlike sanitizeCell this does + * NOT escape `|`: bullets and paragraphs are not markdown tables, and their + * Notion block path does not unescape `\|`, so an escaped pipe would render the + * literal backslash. Newlines still collapse to a space so a single observation + * can't split into extra blocks. + */ +export function sanitizeInline(text: string): string { + return text.replace(/\r\n|\r|\n/g, " "); +} + +/** + * Deterministic observations from the CURRENT 7-day window only. The report is + * stateless (no prior-run history), so observations must NOT reference + * cross-week baselines / medians / trends. + */ +export function buildObservations(bundle: AnalyticsBundle): string[] { + const { summary, queries, emptyQueries } = bundle; + const out: string[] = []; + out.push( + `Empty-result rate is ${(summary.empty_result_rate_window * 100).toFixed(1)}% ` + + `of ${summary.total_queries_window} tool calls in the window.`, + ); + out.push( + `${summary.unique_ip_count_window} unique IPs across ${summary.unique_session_count_window} sessions.`, + ); + const cats = categorizeQueries(queries); + if (cats.length > 0) { + out.push( + `Top category is ${cats[0].category} with ${cats[0].count} queries.`, + ); + } + if (emptyQueries.length > 0) { + const topEmpty = [...emptyQueries].sort((a, b) => b.count - a.count)[0]; + out.push( + `Highest-frequency empty query: "${sanitizeInline(topEmpty.query_text)}" (${topEmpty.count} hits, ${sanitizeInline(topEmpty.tool_name)}).`, + ); + } + return out; +} + +// ── Notion block rendering (mirrors gap-analysis) ───────────────────────────── + +export const NOTION_RICH_TEXT_LIMIT = 2000; +export const NOTION_MAX_BLOCKS_PER_REQUEST = 100; + +interface NotionRichText { + type: "text"; + text: { content: string }; +} + +type NotionBlockType = + | "heading_1" + | "heading_2" + | "heading_3" + | "bulleted_list_item" + | "paragraph" + | "table"; + +interface NotionBlock { + object: "block"; + type: NotionBlockType; + [key: string]: unknown; +} + +interface NotionTableRow { + type: "table_row"; + table_row: { cells: NotionRichText[][] }; +} + +interface NotionTableBlock extends NotionBlock { + type: "table"; + table: { + table_width: number; + has_column_header: boolean; + has_row_header: boolean; + children: NotionTableRow[]; + }; +} + +/** Max DATA rows (excluding header) emitted per Notion table. */ +export const NOTION_MAX_TABLE_ROWS = 100; + +function lineToRichText(line: string): NotionRichText[] { + if (line.length <= NOTION_RICH_TEXT_LIMIT) { + return [{ type: "text", text: { content: line } }]; + } + const spans: NotionRichText[] = []; + for (let i = 0; i < line.length; i += NOTION_RICH_TEXT_LIMIT) { + spans.push({ + type: "text", + text: { content: line.slice(i, i + NOTION_RICH_TEXT_LIMIT) }, + }); + } + return spans; +} + +function makeBlock(type: NotionBlockType, text: string): NotionBlock { + return { + object: "block", + type, + [type]: { rich_text: lineToRichText(text) }, + }; +} + +/** A markdown table row line starts and ends with a pipe (after trimming). */ +function isTableLine(line: string): boolean { + const t = line.trim(); + return t.startsWith("|") && t.endsWith("|") && t.length > 1; +} + +/** A `| --- | :--: |`-style separator row that delimits header from body. */ +function isSeparatorRow(line: string): boolean { + return splitTableRow(line).every((c) => /^:?-{1,}:?$/.test(c.trim())); +} + +/** + * Split one `| a | b |` line into its cell strings (drops the outer pipes). + * Splits on UNESCAPED pipes only, so a `\|` inside a cell (the escaping + * `sanitizeCell` introduced) stays part of that cell. + */ +function splitTableRow(line: string): string[] { + const t = line.trim().replace(/^\|/, "").replace(/\|$/, ""); + // Negative lookbehind: a `|` not preceded by a backslash is a delimiter. + return t.split(/(? lineToRichText(tableCellText(c))) }, + }; +} + +/** + * Build a native Notion `table` block from a contiguous run of `|...|` lines. + * The separator row is dropped; the first remaining row is the header. Tables + * are capped at NOTION_MAX_TABLE_ROWS data rows (a trailing truncation note is + * the caller's responsibility) and every cell respects the rich_text cap. + */ +function tableRunToBlock(run: string[]): { + block: NotionTableBlock; + truncatedFrom: number | null; +} { + const rows = run.filter((l) => !isSeparatorRow(l)).map(splitTableRow); + const width = rows.reduce((max, r) => Math.max(max, r.length), 0); + const header = rows[0] ?? []; + const dataRows = rows.slice(1); + const totalData = dataRows.length; + const keptData = + totalData > NOTION_MAX_TABLE_ROWS + ? dataRows.slice(0, NOTION_MAX_TABLE_ROWS) + : dataRows; + + // Pad ragged rows to a uniform width so Notion accepts the table. + const pad = (cells: string[]): string[] => + cells.length >= width + ? cells + : [...cells, ...Array(width - cells.length).fill("")]; + + const children: NotionTableRow[] = [ + makeTableRow(pad(header)), + ...keptData.map((r) => makeTableRow(pad(r))), + ]; + + return { + block: { + object: "block", + type: "table", + table: { + table_width: width, + has_column_header: true, + has_row_header: false, + children, + }, + }, + truncatedFrom: totalData > NOTION_MAX_TABLE_ROWS ? totalData : null, + }; +} + +/** + * Convert the markdown report into native Notion blocks (headings, bullets, + * paragraphs, and tables). The leading `# ` line is dropped because it + * duplicates the page title (set via properties.title). A contiguous run of + * `|...|` lines becomes one native `table` block (separator row dropped, first + * row as header); tables longer than NOTION_MAX_TABLE_ROWS data rows are capped + * with a trailing truncation note. Every block respects the 2000-char + * rich_text cap. + */ +export function markdownToNotionBlocks(markdown: string): NotionBlock[] { + const blocks: NotionBlock[] = []; + const rawLines = markdown.split("\n"); + + for (let idx = 0; idx < rawLines.length; idx++) { + const line = rawLines[idx]; + if (idx === 0 && line.startsWith("# ")) continue; // drop duplicate-title H1 + + // Gather a contiguous run of table lines and emit one table block. + if (isTableLine(line)) { + const run: string[] = []; + while (idx < rawLines.length && isTableLine(rawLines[idx])) { + run.push(rawLines[idx]); + idx++; + } + idx--; // step back: the for-loop's idx++ re-consumes the non-table line + const { block, truncatedFrom } = tableRunToBlock(run); + blocks.push(block); + if (truncatedFrom !== null) { + blocks.push( + makeBlock( + "paragraph", + `(table truncated to first ${NOTION_MAX_TABLE_ROWS} rows of ${truncatedFrom})`, + ), + ); + } + continue; + } + + if (line.trim() === "") continue; + if (line.startsWith("### ")) { + blocks.push(makeBlock("heading_3", line.slice(4))); + } else if (line.startsWith("## ")) { + blocks.push(makeBlock("heading_2", line.slice(3))); + } else if (line.startsWith("# ")) { + blocks.push(makeBlock("heading_1", line.slice(2))); + } else if (line.startsWith("- ") || line.startsWith("* ")) { + blocks.push(makeBlock("bulleted_list_item", line.slice(2))); + } else { + blocks.push(makeBlock("paragraph", line)); + } + } + return blocks; +} + +export function batchBlocks<T>(blocks: T[], size: number): T[][] { + if (size <= 0) { + throw new Error( + `batchBlocks: size must be a positive integer, got ${size}`, + ); + } + const batches: T[][] = []; + for (let i = 0; i < blocks.length; i += size) { + batches.push(blocks.slice(i, i + size)); + } + return batches; +} + +// ── Injected dependencies (so run() is unit-testable without the network) ───── + +export interface RunEnv { + PATHFINDER_ANALYTICS_TOKEN?: string; + NOTION_TOKEN?: string; + NOTION_PARENT_PAGE_ID?: string; + SLACK_WEBHOOK?: string; + ANALYTICS_BASE_URL?: string; + REPORT_DAYS?: string; +} + +export interface RunDeps { + env: RunEnv; + argv: readonly string[]; + /** Fetch + parse one analytics endpoint. Throws on non-2xx. */ + fetchJson: <T>(path: string) => Promise<T>; + /** Publish the report to Notion. Returns the page URL (or null). Throws on failure. */ + publishNotion: (title: string, markdown: string) => Promise<string | null>; + /** Post a Slack alert (no-op if the webhook is unset). Never throws. */ + postSlack: (text: string) => Promise<void>; + /** + * Write the rendered report to a local path (mkdir -p its parent first). + * Injected so the --report write is a unit-testable fail-loud surface like + * fetch/notion/slack. Throws on any mkdir/write failure. + */ + writeReport: (path: string, markdown: string) => void; + exit: (code: number) => void; + log: (...args: unknown[]) => void; + error: (...args: unknown[]) => void; +} + +const DEFAULT_BASE_URL = "https://mcp.copilotkit.ai"; +const DEFAULT_PARENT_PAGE_ID = "3373aa38-1852-8152-a10b-e4aa1b8a667e"; + +/** + * Build the real (network-backed) fetchJson bound to a base URL + token, modeled + * on monthly-gap-analysis's helper. + */ +export function makeFetchJson( + baseUrl: string, + token: string, +): <T>(path: string) => Promise<T> { + const base = baseUrl.replace(/\/+$/, ""); + return async <T>(path: string): Promise<T> => { + const url = `${base}${path}`; + const res = await fetch(url, { + headers: { + Authorization: `Bearer ${token}`, + Accept: "application/json", + "User-Agent": "pathfinder-weekly-search-report", + }, + }); + if (!res.ok) { + const body = await res.text().catch(() => ""); + throw new Error( + `Analytics fetch failed: ${res.status} ${res.statusText} for ${path}${ + body ? ` — ${body.slice(0, 200)}` : "" + }`, + ); + } + return (await res.json()) as T; + }; +} + +/** + * The minimal slice of the Notion SDK client this module uses. Declared as an + * interface so the publish core (publishNotionWithClient) can be unit-tested + * with an injected fake, without loading the real `@notionhq/client` SDK. + */ +export interface NotionClientLike { + pages: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + create: (args: any) => Promise<{ id: string; url?: string | null }>; + update: (args: { page_id: string; archived: boolean }) => Promise<unknown>; + }; + blocks: { + children: { + // eslint-disable-next-line @typescript-eslint/no-explicit-any + append: (args: any) => Promise<unknown>; + }; + }; +} + +/** + * Publish core: create the page with the first 100-block batch, then append the + * remaining batches. Notion has NO transactional multi-batch create — once + * pages.create succeeds the page exists, so if any later append throws we would + * otherwise be left with a PARTIAL/orphaned page while the run reports failure. + * To honor the "never publish a degraded page" promise, on any append failure we + * make a BEST-EFFORT attempt to archive the just-created page (ignoring errors + * from the archive itself) and then re-throw the ORIGINAL append error so the + * caller still fails loud (Slack + exit 1). + * + * Separated from makePublishNotion (which only wires the real SDK) so it can be + * unit-tested with a fake NotionClientLike. + */ +export async function publishNotionWithClient( + notion: NotionClientLike, + parentPageId: string, + title: string, + markdown: string, +): Promise<string | null> { + const blocks = markdownToNotionBlocks(markdown); + const batches = batchBlocks(blocks, NOTION_MAX_BLOCKS_PER_REQUEST); + const firstBatch = batches[0] ?? []; + const page = await notion.pages.create({ + parent: { page_id: parentPageId }, + properties: { + title: { title: [{ type: "text", text: { content: title } }] }, + }, + children: firstBatch, + }); + try { + for (const batch of batches.slice(1)) { + await notion.blocks.children.append({ + block_id: page.id, + children: batch, + }); + } + } catch (appendErr) { + // The page already exists (Notion can't create multi-batch atomically). + // Best-effort archive the partial page so no degraded page survives; swallow + // any archive error, then re-throw the original append failure. + try { + await notion.pages.update({ page_id: page.id, archived: true }); + } catch { + // Intentionally ignored — archiving is best-effort cleanup. + } + throw appendErr; + } + return page.url ?? null; +} + +/** The real Notion publisher (dynamic import so tests never load the SDK). */ +export function makePublishNotion( + notionToken: string, + parentPageId: string, +): (title: string, markdown: string) => Promise<string | null> { + return async (title: string, markdown: string): Promise<string | null> => { + const { Client } = await import("@notionhq/client"); + const notion = new Client({ auth: notionToken }); + return publishNotionWithClient( + notion as unknown as NotionClientLike, + parentPageId, + title, + markdown, + ); + }; +} + +/** The real Slack poster (no-op when the webhook is unset). Never throws. */ +export function makePostSlack( + webhook: string, +): (text: string) => Promise<void> { + return async (text: string): Promise<void> => { + if (!webhook) { + console.log( + "[weekly-report] SLACK_WEBHOOK unset — skipping Slack alert.", + ); + return; + } + try { + const res = await fetch(webhook, { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ text }), + }); + if (!res.ok) { + console.warn( + `[weekly-report] Slack POST failed: ${res.status} ${res.statusText}`, + ); + } + } catch (err) { + console.warn(`[weekly-report] Slack POST error: ${String(err)}`); + } + }; +} + +/** + * Fetch all four required endpoints and validate their shapes. Throws (the + * fail-loud signal) on any non-2xx (from fetchJson) or malformed payload. + */ +export async function fetchBundle( + deps: Pick<RunDeps, "fetchJson">, + days: number, +): Promise<AnalyticsBundle> { + const summary = await deps.fetchJson<AnalyticsSummary>( + `/api/analytics/summary?days=${days}`, + ); + assertValidSummary(summary); + + const queries = await deps.fetchJson<TopQuery[]>( + `/api/analytics/queries?days=${days}&limit=200`, + ); + assertArrayOf<TopQuery>("/queries", queries, (r, i) => + assertTopQueryRow("/queries", r, i), + ); + + const emptyQueries = await deps.fetchJson<EmptyQuery[]>( + `/api/analytics/empty-queries?days=${days}&limit=200`, + ); + assertArrayOf<EmptyQuery>("/empty-queries", emptyQueries, (r, i) => + assertEmptyQueryRow("/empty-queries", r, i), + ); + + const toolBreakdown = await deps.fetchJson<ToolBreakdownRow[]>( + `/api/analytics/tool-breakdown?days=${days}`, + ); + assertArrayOf<ToolBreakdownRow>("/tool-breakdown", toolBreakdown, (r, i) => + assertToolBreakdownRow("/tool-breakdown", r, i), + ); + + return { summary, queries, emptyQueries, toolBreakdown }; +} + +/** + * The orchestrating run. Dependency-injected so every fail-loud path is + * unit-testable without the network. The contract this enforces is the whole + * point of the rebuild: + * - missing token / any fetch failure / malformed payload → log "FAILED", + * POST a Slack alert, exit(1), and NEVER publish a Notion page. + * - a Notion publish failure AFTER a good fetch → also Slack + exit(1). + */ +export async function run(deps: RunDeps): Promise<void> { + const token = (deps.env.PATHFINDER_ANALYTICS_TOKEN ?? "").trim(); + + // The exact 2026-06-21 failure mode: no token → fail loud, no error page. + if (!token) { + const reason = "PATHFINDER_ANALYTICS_TOKEN is missing or empty"; + deps.error(`[weekly-report] FAILED: ${reason}`); + await deps.postSlack(`Pathfinder weekly search report FAILED: ${reason}`); + deps.exit(1); + return; + } + + const days = parseReportDays(deps.env.REPORT_DAYS); + + let bundle: AnalyticsBundle; + try { + bundle = await fetchBundle(deps, days); + } catch (err) { + const reason = String(err instanceof Error ? err.message : err); + deps.error(`[weekly-report] FAILED: ${reason}`); + await deps.postSlack(`Pathfinder weekly search report FAILED: ${reason}`); + deps.exit(1); + return; + } + + const now = new Date(); + const title = reportTitle(now, days); + const markdown = renderMarkdown(bundle, now, days); + + // A requested --report path is written before publish — it's a local artifact + // the workflow uploads, not an external side effect, and is useful even if the + // publish step later fails. + const reportPath = reportPathArgFrom(deps.argv); + if (reportPath) { + try { + deps.writeReport(reportPath, markdown); + deps.log(`[weekly-report] Report written to ${reportPath}`); + } catch (err) { + const reason = `--report write failed: ${String( + err instanceof Error ? err.message : err, + )}`; + deps.error(`[weekly-report] FAILED: ${reason}`); + await deps.postSlack(`Pathfinder weekly search report FAILED: ${reason}`); + deps.exit(1); + return; + } + } + + // Notion publish failure after a good fetch is also fail-loud. + try { + const url = await deps.publishNotion(title, markdown); + deps.log(`[weekly-report] Published to Notion: ${url ?? "(no url)"}`); + } catch (err) { + const reason = `Notion publish failed: ${String( + err instanceof Error ? err.message : err, + )}`; + deps.error(`[weekly-report] FAILED: ${reason}`); + await deps.postSlack(`Pathfinder weekly search report FAILED: ${reason}`); + deps.exit(1); + return; + } + + deps.log("[weekly-report] Done."); +} + +// ── Real-environment wiring (only runs when invoked directly) ───────────────── + +function buildRealDeps(): RunDeps { + const env: RunEnv = { + PATHFINDER_ANALYTICS_TOKEN: process.env.PATHFINDER_ANALYTICS_TOKEN, + NOTION_TOKEN: process.env.NOTION_TOKEN, + NOTION_PARENT_PAGE_ID: process.env.NOTION_PARENT_PAGE_ID, + SLACK_WEBHOOK: process.env.SLACK_WEBHOOK, + ANALYTICS_BASE_URL: process.env.ANALYTICS_BASE_URL, + REPORT_DAYS: process.env.REPORT_DAYS, + }; + const baseUrl = env.ANALYTICS_BASE_URL ?? DEFAULT_BASE_URL; + const token = (env.PATHFINDER_ANALYTICS_TOKEN ?? "").trim(); + const notionToken = env.NOTION_TOKEN ?? ""; + const parentPageId = env.NOTION_PARENT_PAGE_ID ?? DEFAULT_PARENT_PAGE_ID; + const webhook = env.SLACK_WEBHOOK ?? ""; + + return { + env, + argv: process.argv, + fetchJson: makeFetchJson(baseUrl, token), + publishNotion: makePublishNotion(notionToken, parentPageId), + postSlack: makePostSlack(webhook), + writeReport: (path: string, markdown: string) => { + mkdirSync(dirname(path), { recursive: true }); + writeFileSync(path, markdown, "utf-8"); + }, + exit: (code: number) => process.exit(code), + // eslint-disable-next-line no-console + log: (...args: unknown[]) => console.log(...args), + // eslint-disable-next-line no-console + error: (...args: unknown[]) => console.error(...args), + }; +} + +// Only run when invoked directly (npx tsx … / node …), not when imported by the +// unit tests, which exercise the exported pure helpers + run() with fakes. +const invokedDirectly = + process.argv[1] !== undefined && + resolve(process.argv[1]) === fileURLToPath(import.meta.url); + +if (invokedDirectly) { + console.log("=== Pathfinder Weekly Search Report ==="); + run(buildRealDeps()).catch((err) => { + console.error("[weekly-report] Fatal error:", err); + process.exit(1); + }); +} From b0a4e36efb6270b98d4eb215bc3cef8b8a87da1a Mon Sep 17 00:00:00 2001 From: Jordan Ritter <jpr5@darkridge.com> Date: Mon, 22 Jun 2026 14:28:44 -0700 Subject: [PATCH 3/3] Add weekly search report workflow and gate the new script dir in CI Sunday cron GHA workflow reusing PATHFINDER_ANALYTICS_TOKEN / NOTION_TOKEN / SLACK_WEBHOOK_OSS_ALERTS. Extend tsconfig.scripts.json + static-quality prettier glob to typecheck/format-check scripts/weekly-search-report. --- .github/workflows/static-quality.yml | 2 +- .github/workflows/weekly-search-report.yml | 82 ++++++++++++++++++++++ tsconfig.scripts.json | 2 +- 3 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/weekly-search-report.yml diff --git a/.github/workflows/static-quality.yml b/.github/workflows/static-quality.yml index d3d4db2..f2ba2ca 100644 --- a/.github/workflows/static-quality.yml +++ b/.github/workflows/static-quality.yml @@ -22,7 +22,7 @@ jobs: # Cover the gap-analysis script too: it ships from scripts/ and was # previously neither format-checked nor type-checked in CI (the other # scripts/ files predate this gate and are out of scope here). - - run: npx prettier --check "src/**/*.ts" "scripts/gap-analysis/**/*.ts" + - run: npx prettier --check "src/**/*.ts" "scripts/gap-analysis/**/*.ts" "scripts/weekly-search-report/**/*.ts" typecheck-scripts: runs-on: ubuntu-latest diff --git a/.github/workflows/weekly-search-report.yml b/.github/workflows/weekly-search-report.yml new file mode 100644 index 0000000..40f15a9 --- /dev/null +++ b/.github/workflows/weekly-search-report.yml @@ -0,0 +1,82 @@ +name: Weekly Search Report +# Runs the Pathfinder weekly search-query report on a 7-day lookback and +# publishes a deterministic (no-LLM) report page to Notion. Replaces the old +# claude.ai routine that scraped ephemeral Railway stdout logs and quietly +# posted an error page when its token was missing. +# +# IMPORTANT: this job works only from the durable query_log-backed analytics +# JSON API. It does NOT read the indexed repos and does NOT reproduce queries +# against the live MCP — doing so would self-inflate the analytics it measures. +# +# Fail-loud: unlike the retired routine, a run that cannot fetch its data (or +# cannot publish) exits non-zero and posts a Slack alert to #oss-alerts instead +# of publishing a degraded "could not generate" page. A red run + Slack alert is +# the intended signal. +# +# Required repository secrets: +# PATHFINDER_ANALYTICS_TOKEN Bearer token for GET /api/analytics/* on the +# production MCP (https://mcp.copilotkit.ai). When +# unset, the script FAILS LOUD (non-zero exit + +# Slack alert) — the report cannot run without it. +# NOTION_TOKEN Notion integration token used to publish the +# report page. A publish failure is fail-loud. +# SLACK_WEBHOOK_OSS_ALERTS Incoming-webhook URL (org-level secret shared by +# every workflow). Mapped to the script's +# SLACK_WEBHOOK env. Posted to only on failure. +# When unset, the alert no-ops but the non-zero +# exit still stands. +on: + schedule: + # Weekly, Sunday ~09:07 UTC — off the :00 mark to avoid the cron stampede. + # A Sunday run reports the prior Mon–Sun 7-day window. + - cron: "7 9 * * 0" + workflow_dispatch: + +# Stateless report — no prior-run artifact to read, so no `actions: read`. +permissions: + contents: read + +jobs: + weekly-search-report: + runs-on: ubuntu-latest + timeout-minutes: 15 + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0 + with: + node-version: "22" + + - run: npm ci + + - name: Run weekly search report + env: + # Read-only analytics access. Unset → script FAILS LOUD (no token, + # no report). This secret already exists in the repo (shared with + # monthly-gap-analysis). + PATHFINDER_ANALYTICS_TOKEN: ${{ secrets.PATHFINDER_ANALYTICS_TOKEN }} + # Notion publish target. Publish failure is fail-loud. + NOTION_TOKEN: ${{ secrets.NOTION_TOKEN }} + # Slack alert (failure only). The script reads SLACK_WEBHOOK; the + # org-level secret is SLACK_WEBHOOK_OSS_ALERTS. Unset → alert no-ops, + # non-zero exit still stands. + SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_OSS_ALERTS }} + # Pathfinder project page where weekly reports land. + NOTION_PARENT_PAGE_ID: "3373aa38-1852-8152-a10b-e4aa1b8a667e" + # Production analytics API base. + ANALYTICS_BASE_URL: "https://mcp.copilotkit.ai" + # 7-day lookback window. + REPORT_DAYS: "7" + run: npx tsx scripts/weekly-search-report/weekly-search-report.ts --report /tmp/weekly-report.md + + # Keep the rendered report as a build artifact for inspection even when + # Notion publishing is not yet configured. + - name: Upload weekly report artifact + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: weekly-search-report + path: /tmp/weekly-report.md + if-no-files-found: ignore diff --git a/tsconfig.scripts.json b/tsconfig.scripts.json index 7602826..717bde4 100644 --- a/tsconfig.scripts.json +++ b/tsconfig.scripts.json @@ -4,6 +4,6 @@ "noEmit": true, "rootDir": null }, - "include": ["scripts/gap-analysis/**/*"], + "include": ["scripts/gap-analysis/**/*", "scripts/weekly-search-report/**/*"], "exclude": ["node_modules", "dist"] }