diff --git a/apps/runtime/resilience.ts b/apps/runtime/resilience.ts index 16dea98..22eee4d 100644 --- a/apps/runtime/resilience.ts +++ b/apps/runtime/resilience.ts @@ -11,8 +11,10 @@ * process stays up for other connections. * - Always exits on uncaughtException (per Node.js guidance) but logs * whether the error was a known network issue or a genuine bug. - * - Periodically logs memory usage and triggers GC when RSS climbs - * toward the container limit (useful with --expose-gc). + * - Periodically logs memory usage, triggers GC when RSS climbs toward + * the container limit, and initiates a **graceful shutdown** when heap + * is dangerously close to the V8 hard limit — preventing the abrupt + * exit-134 OOM crash that kills in-flight requests. */ const IGNORABLE_ERRORS = [ @@ -59,15 +61,68 @@ process.on("uncaughtException", (err: Error) => { process.exit(1); }); -// --- Memory monitoring --- +// --- Memory monitoring with graceful OOM shutdown --- const MEMORY_LOG_INTERVAL_MS = 60_000; const MEMORY_WARN_THRESHOLD_MB = 200; +// Trigger graceful shutdown before V8's hard OOM (--max-old-space-size=256). +// At 235MB heap we're ~20MB from the cliff — enough headroom to drain but +// not so much that we restart unnecessarily. +const GRACEFUL_SHUTDOWN_HEAP_MB = 235; +const GRACEFUL_DRAIN_MS = 5_000; + +let shuttingDown = false; +const shutdownCallbacks: Array<() => void> = []; + +/** + * Exported so server.ts can read it in middleware to reject new requests + * while the process is draining before a graceful restart. + */ +export function isShuttingDown(): boolean { + return shuttingDown; +} + +/** + * Register a callback to run when graceful shutdown begins (e.g. server.close()). + */ +export function onShutdown(cb: () => void): void { + shutdownCallbacks.push(cb); +} + +function gracefulShutdown() { + if (shuttingDown) return; + shuttingDown = true; + for (const cb of shutdownCallbacks) { + try { cb(); } catch (e) { console.error("[shutdown] callback error", e); } + } + + const mem = process.memoryUsage(); + const heapMb = Math.round(mem.heapUsed / 1024 / 1024); + console.warn( + `[memory] Heap at ${heapMb}MB — initiating graceful shutdown to avoid OOM crash`, + ); + console.warn( + `[memory] Draining in-flight requests for ${GRACEFUL_DRAIN_MS}ms before exit…`, + ); + + // Give in-flight requests a few seconds to complete, then exit cleanly. + // Render will restart the instance automatically. + setTimeout(() => { + console.warn("[memory] Drain period complete — exiting (code 0)"); + process.exit(0); + }, GRACEFUL_DRAIN_MS); +} setInterval(() => { const mem = process.memoryUsage(); const rssMb = Math.round(mem.rss / 1024 / 1024); const heapMb = Math.round(mem.heapUsed / 1024 / 1024); + + if (heapMb >= GRACEFUL_SHUTDOWN_HEAP_MB) { + gracefulShutdown(); + return; + } + if (rssMb > MEMORY_WARN_THRESHOLD_MB) { console.warn( `[memory] WARNING RSS=${rssMb}MB heap=${heapMb}MB — approaching limit`, diff --git a/apps/runtime/server.ts b/apps/runtime/server.ts index cefd0ae..343070d 100644 --- a/apps/runtime/server.ts +++ b/apps/runtime/server.ts @@ -4,6 +4,7 @@ import { cors } from "hono/cors"; import { CopilotRuntime, createCopilotEndpointSingleRoute } from "@copilotkit/runtime/v2"; import { LangGraphHttpAgent } from "@copilotkit/runtime/langgraph"; import { registerRealtimeSessionRoute } from "@frenchfryai/runtime"; +import { isShuttingDown, onShutdown } from "./resilience.js"; const agentHost = process.env.LANGGRAPH_DEPLOYMENT_URL || "http://localhost:8123"; const agentUrl = agentHost.startsWith("http") ? agentHost : `http://${agentHost}`; @@ -24,6 +25,16 @@ const copilotApp = createCopilotEndpointSingleRoute({ const app = new Hono(); app.use("/*", cors()); +// Reject new requests during graceful OOM shutdown so in-flight ones can drain +app.use("/*", async (c, next) => { + if (isShuttingDown()) { + c.header("Connection", "close"); + c.header("Retry-After", "5"); + return c.text("Service restarting", 503); + } + return next(); +}); + app.route("/", copilotApp as any); registerRealtimeSessionRoute(app, { @@ -33,10 +44,9 @@ registerRealtimeSessionRoute(app, { }, }); -// Error resilience and memory monitoring — see resilience.ts for details -import "./resilience.js"; - const port = Number(process.env.PORT || 4000); -serve({ fetch: app.fetch, port }); +let server: ReturnType | undefined; +server = serve({ fetch: app.fetch, port }); +onShutdown(() => server?.close()); console.log(`Runtime server listening at http://localhost:${port}/api/copilotkit`); console.log(`Realtime session endpoint at http://localhost:${port}/realtime/session`);