From 8864d1a8e2608e28ec3755f1707e0ef05e831a33 Mon Sep 17 00:00:00 2001
From: babaohuang <babaohuang@gmail.com>
Date: Thu, 14 Dec 2023 20:51:28 +0800
Subject: [PATCH] Revert "Update generate.ts"

This reverts commit 8bbf6e60ad9074a6c71d2044ce76dda56a23c5a5.
---
 src/pages/api/generate.ts | 29 +++++++++++++++++++++++++----
 1 file changed, 25 insertions(+), 4 deletions(-)

diff --git a/src/pages/api/generate.ts b/src/pages/api/generate.ts
index 76549148..390a8025 100644
--- a/src/pages/api/generate.ts
+++ b/src/pages/api/generate.ts
@@ -42,13 +42,34 @@ export const post: APIRoute = async(context) => {
 
     const responseStream = new ReadableStream({
       async start(controller) {
-        const decoder = new TextDecoder('utf-8')
+        const decoder = new TextDecoder('utf-8', { stream: true }) // Use the streaming option
+        let buffer = new Uint8Array()
+
         for await (const chunk of stream) {
-          const value = chunk instanceof Uint8Array ? chunk : new TextEncoder().encode(chunk)
-          const text = decoder.decode(value, { stream: true })
+          const chunkAsText = await chunk.text()
+          const chunkAsUint8Array = new TextEncoder().encode(chunkAsText)
+          // Combine the buffered bytes with the new bytes
+          const combinedChunk = new Uint8Array(buffer.length + chunkAsUint8Array.length)
+          combinedChunk.set(buffer)
+          combinedChunk.set(chunkAsUint8Array, buffer.length)
+
+          // Find the last complete UTF-8 character
+          let end = combinedChunk.length
+          while (end > 0 && (combinedChunk[end - 1] & 0xC0) === 0x80)
+            end--
+
+          // Decode the complete characters, and buffer the rest
+          const text = decoder.decode(combinedChunk.subarray(0, end), { stream: true })
+          buffer = combinedChunk.subarray(end)
           controller.enqueue(new TextEncoder().encode(text))
         }
-        // When the source stream is finished, close our stream.
+
+        // Decode any remaining bytes
+        if (buffer.length > 0) {
+          const text = decoder.decode(buffer, { stream: false }) // Flush the decoder
+          controller.enqueue(new TextEncoder().encode(text))
+        }
+
         controller.close()
       },
     })