From 70fad71c68a0d80edbeda71392d8be9822dd1490 Mon Sep 17 00:00:00 2001 From: jerelvelarde Date: Tue, 24 Mar 2026 06:02:21 -0700 Subject: [PATCH 1/3] fix: graceful shutdown before V8 OOM crash The runtime process heap grows monotonically until it hits the 256MB V8 limit, causing an abrupt exit-134 crash ~3x every 6 hours. Instead of letting V8 kill the process, detect when heap reaches 235MB and initiate a controlled drain: reject new requests with 503, give in-flight requests 5s to complete, then exit cleanly for Render to restart a fresh instance. Co-Authored-By: Claude Opus 4.6 (1M context) --- apps/runtime/resilience.ts | 50 +++++++++++++++++++++++++++++++++++--- apps/runtime/server.ts | 13 +++++++--- 2 files changed, 57 insertions(+), 6 deletions(-) diff --git a/apps/runtime/resilience.ts b/apps/runtime/resilience.ts index 16dea98..850eb3d 100644 --- a/apps/runtime/resilience.ts +++ b/apps/runtime/resilience.ts @@ -11,8 +11,10 @@ * process stays up for other connections. * - Always exits on uncaughtException (per Node.js guidance) but logs * whether the error was a known network issue or a genuine bug. - * - Periodically logs memory usage and triggers GC when RSS climbs - * toward the container limit (useful with --expose-gc). + * - Periodically logs memory usage, triggers GC when RSS climbs toward + * the container limit, and initiates a **graceful shutdown** when heap + * is dangerously close to the V8 hard limit — preventing the abrupt + * exit-134 OOM crash that kills in-flight requests. */ const IGNORABLE_ERRORS = [ @@ -59,15 +61,57 @@ process.on("uncaughtException", (err: Error) => { process.exit(1); }); -// --- Memory monitoring --- +// --- Memory monitoring with graceful OOM shutdown --- const MEMORY_LOG_INTERVAL_MS = 60_000; const MEMORY_WARN_THRESHOLD_MB = 200; +// Trigger graceful shutdown before V8's hard OOM (--max-old-space-size=256). +// At 235MB heap we're ~20MB from the cliff — enough headroom to drain but +// not so much that we restart unnecessarily. +const GRACEFUL_SHUTDOWN_HEAP_MB = 235; +const GRACEFUL_DRAIN_MS = 5_000; + +let shuttingDown = false; + +/** + * Exported so server.ts can read it in middleware to reject new requests + * while the process is draining before a graceful restart. + */ +export function isShuttingDown(): boolean { + return shuttingDown; +} + +function gracefulShutdown() { + if (shuttingDown) return; + shuttingDown = true; + + const mem = process.memoryUsage(); + const heapMb = Math.round(mem.heapUsed / 1024 / 1024); + console.warn( + `[memory] Heap at ${heapMb}MB — initiating graceful shutdown to avoid OOM crash`, + ); + console.warn( + `[memory] Draining in-flight requests for ${GRACEFUL_DRAIN_MS}ms before exit…`, + ); + + // Give in-flight requests a few seconds to complete, then exit cleanly. + // Render will restart the instance automatically. + setTimeout(() => { + console.warn("[memory] Drain period complete — exiting (code 0)"); + process.exit(0); + }, GRACEFUL_DRAIN_MS).unref(); +} setInterval(() => { const mem = process.memoryUsage(); const rssMb = Math.round(mem.rss / 1024 / 1024); const heapMb = Math.round(mem.heapUsed / 1024 / 1024); + + if (heapMb >= GRACEFUL_SHUTDOWN_HEAP_MB) { + gracefulShutdown(); + return; + } + if (rssMb > MEMORY_WARN_THRESHOLD_MB) { console.warn( `[memory] WARNING RSS=${rssMb}MB heap=${heapMb}MB — approaching limit`, diff --git a/apps/runtime/server.ts b/apps/runtime/server.ts index cefd0ae..438900f 100644 --- a/apps/runtime/server.ts +++ b/apps/runtime/server.ts @@ -4,6 +4,7 @@ import { cors } from "hono/cors"; import { CopilotRuntime, createCopilotEndpointSingleRoute } from "@copilotkit/runtime/v2"; import { LangGraphHttpAgent } from "@copilotkit/runtime/langgraph"; import { registerRealtimeSessionRoute } from "@frenchfryai/runtime"; +import { isShuttingDown } from "./resilience.js"; const agentHost = process.env.LANGGRAPH_DEPLOYMENT_URL || "http://localhost:8123"; const agentUrl = agentHost.startsWith("http") ? agentHost : `http://${agentHost}`; @@ -24,6 +25,15 @@ const copilotApp = createCopilotEndpointSingleRoute({ const app = new Hono(); app.use("/*", cors()); +// Reject new requests during graceful OOM shutdown so in-flight ones can drain +app.use("/*", async (c, next) => { + if (isShuttingDown()) { + c.header("Connection", "close"); + return c.text("Service restarting", 503); + } + return next(); +}); + app.route("/", copilotApp as any); registerRealtimeSessionRoute(app, { @@ -33,9 +43,6 @@ registerRealtimeSessionRoute(app, { }, }); -// Error resilience and memory monitoring — see resilience.ts for details -import "./resilience.js"; - const port = Number(process.env.PORT || 4000); serve({ fetch: app.fetch, port }); console.log(`Runtime server listening at http://localhost:${port}/api/copilotkit`); From 096ab5a9ca33a77b8413b91ae1853b07eeb995a3 Mon Sep 17 00:00:00 2001 From: jerelvelarde Date: Tue, 24 Mar 2026 06:12:24 -0700 Subject: [PATCH 2/3] fix: close HTTP server on graceful shutdown to stop accepting connections The prior graceful shutdown set a flag and returned 503s but never called server.close(), so the Node http.Server kept accepting TCP connections during the drain period. --- apps/runtime/resilience.ts | 9 +++++++++ apps/runtime/server.ts | 5 +++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/apps/runtime/resilience.ts b/apps/runtime/resilience.ts index 850eb3d..7b44815 100644 --- a/apps/runtime/resilience.ts +++ b/apps/runtime/resilience.ts @@ -72,6 +72,7 @@ const GRACEFUL_SHUTDOWN_HEAP_MB = 235; const GRACEFUL_DRAIN_MS = 5_000; let shuttingDown = false; +const shutdownCallbacks: Array<() => void> = []; /** * Exported so server.ts can read it in middleware to reject new requests @@ -81,9 +82,17 @@ export function isShuttingDown(): boolean { return shuttingDown; } +/** + * Register a callback to run when graceful shutdown begins (e.g. server.close()). + */ +export function onShutdown(cb: () => void): void { + shutdownCallbacks.push(cb); +} + function gracefulShutdown() { if (shuttingDown) return; shuttingDown = true; + for (const cb of shutdownCallbacks) cb(); const mem = process.memoryUsage(); const heapMb = Math.round(mem.heapUsed / 1024 / 1024); diff --git a/apps/runtime/server.ts b/apps/runtime/server.ts index 438900f..7a88491 100644 --- a/apps/runtime/server.ts +++ b/apps/runtime/server.ts @@ -4,7 +4,7 @@ import { cors } from "hono/cors"; import { CopilotRuntime, createCopilotEndpointSingleRoute } from "@copilotkit/runtime/v2"; import { LangGraphHttpAgent } from "@copilotkit/runtime/langgraph"; import { registerRealtimeSessionRoute } from "@frenchfryai/runtime"; -import { isShuttingDown } from "./resilience.js"; +import { isShuttingDown, onShutdown } from "./resilience.js"; const agentHost = process.env.LANGGRAPH_DEPLOYMENT_URL || "http://localhost:8123"; const agentUrl = agentHost.startsWith("http") ? agentHost : `http://${agentHost}`; @@ -44,6 +44,7 @@ registerRealtimeSessionRoute(app, { }); const port = Number(process.env.PORT || 4000); -serve({ fetch: app.fetch, port }); +const server = serve({ fetch: app.fetch, port }); +onShutdown(() => server.close()); console.log(`Runtime server listening at http://localhost:${port}/api/copilotkit`); console.log(`Realtime session endpoint at http://localhost:${port}/realtime/session`); From c9e04b229a82e6f9fa74d925601b0179a66fb703 Mon Sep 17 00:00:00 2001 From: jerelvelarde Date: Tue, 24 Mar 2026 06:54:01 -0700 Subject: [PATCH 3/3] fix: address PR review feedback for graceful shutdown - Remove .unref() on drain timer so in-flight requests get the full 5s window - Wrap shutdown callbacks in try/catch to prevent one failure from skipping the rest - Guard server.close() against undefined ref during startup race - Add Retry-After header on 503 to help LBs/clients back off --- apps/runtime/resilience.ts | 6 ++++-- apps/runtime/server.ts | 6 ++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/apps/runtime/resilience.ts b/apps/runtime/resilience.ts index 7b44815..22eee4d 100644 --- a/apps/runtime/resilience.ts +++ b/apps/runtime/resilience.ts @@ -92,7 +92,9 @@ export function onShutdown(cb: () => void): void { function gracefulShutdown() { if (shuttingDown) return; shuttingDown = true; - for (const cb of shutdownCallbacks) cb(); + for (const cb of shutdownCallbacks) { + try { cb(); } catch (e) { console.error("[shutdown] callback error", e); } + } const mem = process.memoryUsage(); const heapMb = Math.round(mem.heapUsed / 1024 / 1024); @@ -108,7 +110,7 @@ function gracefulShutdown() { setTimeout(() => { console.warn("[memory] Drain period complete — exiting (code 0)"); process.exit(0); - }, GRACEFUL_DRAIN_MS).unref(); + }, GRACEFUL_DRAIN_MS); } setInterval(() => { diff --git a/apps/runtime/server.ts b/apps/runtime/server.ts index 7a88491..343070d 100644 --- a/apps/runtime/server.ts +++ b/apps/runtime/server.ts @@ -29,6 +29,7 @@ app.use("/*", cors()); app.use("/*", async (c, next) => { if (isShuttingDown()) { c.header("Connection", "close"); + c.header("Retry-After", "5"); return c.text("Service restarting", 503); } return next(); @@ -44,7 +45,8 @@ registerRealtimeSessionRoute(app, { }); const port = Number(process.env.PORT || 4000); -const server = serve({ fetch: app.fetch, port }); -onShutdown(() => server.close()); +let server: ReturnType | undefined; +server = serve({ fetch: app.fetch, port }); +onShutdown(() => server?.close()); console.log(`Runtime server listening at http://localhost:${port}/api/copilotkit`); console.log(`Realtime session endpoint at http://localhost:${port}/realtime/session`);