From a357a3fd34dd395d8751c544860365ef544b1c4b Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Tue, 12 Dec 2023 16:17:06 -0800 Subject: [PATCH 1/4] patch: implement @lunamidori hotfix for LocalAI streaming chunk overflows resolves #416 --- server/utils/chats/stream.js | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/server/utils/chats/stream.js b/server/utils/chats/stream.js index 643f3d4398e..7f483c67ab9 100644 --- a/server/utils/chats/stream.js +++ b/server/utils/chats/stream.js @@ -253,7 +253,16 @@ function handleStreamResponses(response, stream, responseProps) { } catch {} if (!validJSON) { - chunk += message; + // It can be possible that the chunk decoding is running away + // and the message chunk fails to append due to string length. + // In this case abort the chunk and reset so we can continue. + // ref: https://github.com/Mintplex-Labs/anything-llm/issues/416 + try { + chunk += message; + } catch (e) { + console.log(`Chunk appending error`, e.message); + chunk = ""; + } continue; } else { chunk = ""; From 4f3d064301af325fc15264d9758dc5b3db045293 Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Tue, 12 Dec 2023 16:17:56 -0800 Subject: [PATCH 2/4] change log to error log --- server/utils/chats/stream.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/utils/chats/stream.js b/server/utils/chats/stream.js index 7f483c67ab9..c1911492de6 100644 --- a/server/utils/chats/stream.js +++ b/server/utils/chats/stream.js @@ -260,7 +260,7 @@ function handleStreamResponses(response, stream, responseProps) { try { chunk += message; } catch (e) { - console.log(`Chunk appending error`, e.message); + console.error(`Chunk appending error`, e.message); chunk = ""; } continue; From 6e70bbd1ebd7aa9213ee0ac6956caf33fcd960f7 Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Tue, 12 Dec 2023 16:18:44 -0800 Subject: [PATCH 3/4] log trace --- server/utils/chats/stream.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/utils/chats/stream.js b/server/utils/chats/stream.js index c1911492de6..54300cd8fdd 100644 --- a/server/utils/chats/stream.js +++ b/server/utils/chats/stream.js @@ -250,7 +250,7 @@ function handleStreamResponses(response, stream, responseProps) { try { JSON.parse(message); validJSON = true; - } catch {} + } catch { } if (!validJSON) { // It can be possible that the chunk decoding is running away @@ -260,7 +260,7 @@ function handleStreamResponses(response, stream, responseProps) { try { chunk += message; } catch (e) { - console.error(`Chunk appending error`, e.message); + console.error(`Chunk appending error`, e); chunk = ""; } continue; From 94282d07062baf5afdc804548d9aed71d1f65e8c Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Tue, 12 Dec 2023 16:19:21 -0800 Subject: [PATCH 4/4] lint --- server/utils/chats/stream.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/utils/chats/stream.js b/server/utils/chats/stream.js index 54300cd8fdd..4eb9cf0221a 100644 --- a/server/utils/chats/stream.js +++ b/server/utils/chats/stream.js @@ -250,7 +250,7 @@ function handleStreamResponses(response, stream, responseProps) { try { JSON.parse(message); validJSON = true; - } catch { } + } catch {} if (!validJSON) { // It can be possible that the chunk decoding is running away