diff --git a/frontend/src/components/LLMSelection/CometApiLLMOptions/index.jsx b/frontend/src/components/LLMSelection/CometApiLLMOptions/index.jsx index 71fbeec6274..0abcf0acbe6 100644 --- a/frontend/src/components/LLMSelection/CometApiLLMOptions/index.jsx +++ b/frontend/src/components/LLMSelection/CometApiLLMOptions/index.jsx @@ -59,7 +59,7 @@ function AdvancedControls({ settings }) { name="CometApiLLMTimeout" className="border-none bg-theme-settings-input-bg text-theme-text-primary placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5" placeholder="Timeout value between token responses to auto-timeout the stream" - defaultValue={settings?.CometApiLLMTimeout ?? 500} + defaultValue={settings?.CometApiLLMTimeout ?? 3_000} autoComplete="off" onScroll={(e) => e.target.blur()} min={500} diff --git a/frontend/src/components/LLMSelection/NovitaLLMOptions/index.jsx b/frontend/src/components/LLMSelection/NovitaLLMOptions/index.jsx index 0f122b6b5ba..b8318cc0718 100644 --- a/frontend/src/components/LLMSelection/NovitaLLMOptions/index.jsx +++ b/frontend/src/components/LLMSelection/NovitaLLMOptions/index.jsx @@ -59,7 +59,7 @@ function AdvancedControls({ settings }) { name="NovitaLLMTimeout" className="border-none bg-theme-settings-input-bg text-theme-text-primary placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5" placeholder="Timeout value between token responses to auto-timeout the stream" - defaultValue={settings?.NovitaLLMTimeout ?? 500} + defaultValue={settings?.NovitaLLMTimeout ?? 3_000} autoComplete="off" onScroll={(e) => e.target.blur()} min={500} diff --git a/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx b/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx index dc595c84991..9fdd62027a8 100644 --- a/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx +++ b/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx @@ -57,7 +57,7 @@ function AdvancedControls({ settings }) { name="OpenRouterTimeout" className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5" placeholder="Timeout value between token responses to auto-timeout the stream" - defaultValue={settings?.OpenRouterTimeout ?? 500} + defaultValue={settings?.OpenRouterTimeout ?? 3_000} autoComplete="off" onScroll={(e) => e.target.blur()} min={500} diff --git a/server/utils/AiProviders/cometapi/index.js b/server/utils/AiProviders/cometapi/index.js index 82fb7c1bb3e..fca0b0cc50a 100644 --- a/server/utils/AiProviders/cometapi/index.js +++ b/server/utils/AiProviders/cometapi/index.js @@ -19,6 +19,7 @@ const cacheFolder = path.resolve( ); class CometApiLLM { + defaultTimeout = 3_000; constructor(embedder = null, modelPreference = null) { if (!process.env.COMETAPI_LLM_API_KEY) throw new Error("No CometAPI API key was set."); @@ -61,10 +62,14 @@ class CometApiLLM { * CometAPI has various models that never return `finish_reasons` and thus leave the stream open * which causes issues in subsequent messages. This timeout value forces us to close the stream after * x milliseconds. This is a configurable value via the COMETAPI_LLM_TIMEOUT_MS value - * @returns {number} The timeout value in milliseconds (default: 500) + * @returns {number} The timeout value in milliseconds (default: 3_000) */ #parseTimeout() { - if (isNaN(Number(process.env.COMETAPI_LLM_TIMEOUT_MS))) return 500; + this.log( + `CometAPI timeout is set to ${process.env.COMETAPI_LLM_TIMEOUT_MS ?? this.defaultTimeout}ms` + ); + if (isNaN(Number(process.env.COMETAPI_LLM_TIMEOUT_MS))) + return this.defaultTimeout; const setValue = Number(process.env.COMETAPI_LLM_TIMEOUT_MS); if (setValue < 500) return 500; return setValue; diff --git a/server/utils/AiProviders/novita/index.js b/server/utils/AiProviders/novita/index.js index 5bbf49ba7fb..08be9cf83cb 100644 --- a/server/utils/AiProviders/novita/index.js +++ b/server/utils/AiProviders/novita/index.js @@ -18,6 +18,8 @@ const cacheFolder = path.resolve( ); class NovitaLLM { + defaultTimeout = 3_000; + constructor(embedder = null, modelPreference = null) { if (!process.env.NOVITA_LLM_API_KEY) throw new Error("No Novita API key was set."); @@ -62,12 +64,16 @@ class NovitaLLM { * Novita has various models that never return `finish_reasons` and thus leave the stream open * which causes issues in subsequent messages. This timeout value forces us to close the stream after * x milliseconds. This is a configurable value via the NOVITA_LLM_TIMEOUT_MS value - * @returns {number} The timeout value in milliseconds (default: 500) + * @returns {number} The timeout value in milliseconds (default: 3_000) */ #parseTimeout() { - if (isNaN(Number(process.env.NOVITA_LLM_TIMEOUT_MS))) return 500; + this.log( + `Novita timeout is set to ${process.env.NOVITA_LLM_TIMEOUT_MS ?? this.defaultTimeout}ms` + ); + if (isNaN(Number(process.env.NOVITA_LLM_TIMEOUT_MS))) + return this.defaultTimeout; const setValue = Number(process.env.NOVITA_LLM_TIMEOUT_MS); - if (setValue < 500) return 500; + if (setValue < 500) return 500; // 500ms is the minimum timeout return setValue; } @@ -318,7 +324,7 @@ class NovitaLLM { }); } - if (message.finish_reason !== null) { + if (message?.finish_reason !== null) { writeResponseChunk(response, { uuid, sources, diff --git a/server/utils/AiProviders/openRouter/index.js b/server/utils/AiProviders/openRouter/index.js index f456c151414..ea1665c04db 100644 --- a/server/utils/AiProviders/openRouter/index.js +++ b/server/utils/AiProviders/openRouter/index.js @@ -18,6 +18,16 @@ const cacheFolder = path.resolve( ); class OpenRouterLLM { + /** + * Some openrouter models never send a finish_reason and thus leave the stream open in the UI. + * However, because OR is a middleware it can also wait an inordinately long time between chunks so we need + * to ensure that we dont accidentally close the stream too early. If the time between chunks is greater than this timeout + * we will close the stream and assume it to be complete. This is common for free models or slow providers they can + * possibly delegate to during invocation. + * @type {number} + */ + defaultTimeout = 3_000; + constructor(embedder = null, modelPreference = null) { if (!process.env.OPENROUTER_API_KEY) throw new Error("No OpenRouter API key was set."); @@ -85,15 +95,16 @@ class OpenRouterLLM { * OpenRouter has various models that never return `finish_reasons` and thus leave the stream open * which causes issues in subsequent messages. This timeout value forces us to close the stream after * x milliseconds. This is a configurable value via the OPENROUTER_TIMEOUT_MS value - * @returns {number} The timeout value in milliseconds (default: 500) + * @returns {number} The timeout value in milliseconds (default: 3_000) */ #parseTimeout() { this.log( - `OpenRouter timeout is set to ${process.env.OPENROUTER_TIMEOUT_MS ?? 500}ms` + `OpenRouter timeout is set to ${process.env.OPENROUTER_TIMEOUT_MS ?? this.defaultTimeout}ms` ); - if (isNaN(Number(process.env.OPENROUTER_TIMEOUT_MS))) return 500; + if (isNaN(Number(process.env.OPENROUTER_TIMEOUT_MS))) + return this.defaultTimeout; const setValue = Number(process.env.OPENROUTER_TIMEOUT_MS); - if (setValue < 500) return 500; + if (setValue < 500) return 500; // 500ms is the minimum timeout return setValue; }