From 1e1dc957e9aab3c1c7120816b89b6c609a9c2e8b Mon Sep 17 00:00:00 2001 From: shatfield4 Date: Mon, 30 Jun 2025 18:16:47 -0700 Subject: [PATCH 1/2] add option to disable streaming via env for generic openai provider --- server/utils/AiProviders/genericOpenAi/index.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/utils/AiProviders/genericOpenAi/index.js b/server/utils/AiProviders/genericOpenAi/index.js index 163780b9e11..33fc8b55a2d 100644 --- a/server/utils/AiProviders/genericOpenAi/index.js +++ b/server/utils/AiProviders/genericOpenAi/index.js @@ -35,6 +35,8 @@ class GenericOpenAiLLM { user: this.promptWindowLimit() * 0.7, }; + this.streamingDisabled = + process.env.GENERIC_OPENAI_STREAMING_DISABLED || false; this.embedder = embedder ?? new NativeEmbedder(); this.defaultTemp = 0.7; this.log(`Inference API: ${this.basePath} Model: ${this.model}`); @@ -57,6 +59,7 @@ class GenericOpenAiLLM { } streamingEnabled() { + if (this.streamingDisabled) return false; return "streamGetChatCompletion" in this; } From a78373d455c350b64baaa1fbd26aa2b90f848ede Mon Sep 17 00:00:00 2001 From: shatfield4 Date: Tue, 1 Jul 2025 10:54:12 -0700 Subject: [PATCH 2/2] move env check to streamingEnabled --- server/utils/AiProviders/genericOpenAi/index.js | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/server/utils/AiProviders/genericOpenAi/index.js b/server/utils/AiProviders/genericOpenAi/index.js index 33fc8b55a2d..e7b35c74f93 100644 --- a/server/utils/AiProviders/genericOpenAi/index.js +++ b/server/utils/AiProviders/genericOpenAi/index.js @@ -35,8 +35,6 @@ class GenericOpenAiLLM { user: this.promptWindowLimit() * 0.7, }; - this.streamingDisabled = - process.env.GENERIC_OPENAI_STREAMING_DISABLED || false; this.embedder = embedder ?? new NativeEmbedder(); this.defaultTemp = 0.7; this.log(`Inference API: ${this.basePath} Model: ${this.model}`); @@ -59,7 +57,7 @@ class GenericOpenAiLLM { } streamingEnabled() { - if (this.streamingDisabled) return false; + if (process.env.GENERIC_OPENAI_STREAMING_DISABLED === "true") return false; return "streamGetChatCompletion" in this; }