feat: Enable streaming by default for chat completions

Author: ericc-ch

src/lib/cli.ts

@@ -10,7 +10,7 @@ const args = {
   stream: {
     alias: "s",
     type: "boolean",
-    default: false,
+    default: true,
     description: "Enable streaming response for chat completions",
   },
   port: {

src/routes/chat-completions/route.ts

@@ -1,45 +1,21 @@
 import consola from "consola"
 import { Hono } from "hono"
-import { streamSSE } from "hono/streaming"
 import { FetchError } from "ofetch"
 
-import type { ChatCompletionsPayload } from "~/services/copilot/chat-completions/types"
+import { ENV } from "~/config/env"
 
-import { chatCompletions } from "~/services/copilot/chat-completions/service"
+import { handler } from "./handler"
+import { handlerStreaming } from "./handler-streaming"
 
-import { createContentChunk, createFinalChunk, segmentResponse } from "./utils"
+export const completionRoutes = new Hono()
 
-export const chatCompletionsRoutes = new Hono()
-
-chatCompletionsRoutes.post("/chat/completions", async (c) => {
+completionRoutes.post("/chat/completions", (c) => {
   try {
-    const payload = await c.req.json<ChatCompletionsPayload>()
-    payload.stream = false
-
-    consola.info(`Received request: ${JSON.stringify(payload).slice(0, 500)}`)
-
-    const response = await chatCompletions(payload)
-    consola.info(`Response from Copilot: ${JSON.stringify(response)}`)
-
-    const segments = segmentResponse(response.choices[0].message.content)
-    const chunks = segments.map((segment) =>
-      createContentChunk(segment, response, payload.model),
-    )
-
-    chunks.push(createFinalChunk(response, payload.model))
-
-    consola.info(
-      `Streaming response, first chunk: ${JSON.stringify(chunks.at(0))}`,
-    )
-
-    return streamSSE(c, async (stream) => {
-      for (const chunk of chunks) {
-        await stream.writeSSE({
-          data: JSON.stringify(chunk.data),
-        })
-        await stream.sleep(1) // Simulated latency
-      }
-    })
+    if (ENV.ENABLE_STREAMING) {
+      return handlerStreaming(c)
+    } else {
+      return handler(c)
+    }
   } catch (error) {
     if (error instanceof FetchError) {
       consola.error(`Request failed: ${error.message}`, error.response?._data)

src/server.ts

@@ -2,7 +2,7 @@ import { Hono } from "hono"
 import { cors } from "hono/cors"
 import { logger } from "hono/logger"
 
-import { chatCompletionsRoutes } from "./routes/chat-completions/route"
+import { completionRoutes } from "./routes/chat-completions/route"
 import { modelRoutes } from "./routes/models/route"
 
 export const server = new Hono()
@@ -11,5 +11,5 @@ server.use(logger())
 server.use(cors())
 
 server.get("/", (c) => c.text("Server running"))
-server.route("/", chatCompletionsRoutes)
+server.route("/", completionRoutes)
 server.route("/", modelRoutes)