diff --git a/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx b/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx index 470340d2ff3..2b3e72b3fc1 100644 --- a/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx +++ b/frontend/src/components/LLMSelection/OpenRouterOptions/index.jsx @@ -1,27 +1,72 @@ import System from "@/models/system"; +import { CaretDown, CaretUp } from "@phosphor-icons/react"; import { useState, useEffect } from "react"; export default function OpenRouterOptions({ settings }) { + const [showAdvancedControls, setShowAdvancedControls] = useState(false); + return ( -
-
- - +
+
+
+ + +
+ {!settings?.credentialsOnly && ( + + )} +
+ +
+ ); +} + +function AdvancedControls({ settings }) { + const [showAdvancedControls, setShowAdvancedControls] = useState(false); + + return ( +
+ + - {!settings?.credentialsOnly && ( - - )}
); } diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js index 6f5b4238df6..485837506ab 100644 --- a/server/models/systemSettings.js +++ b/server/models/systemSettings.js @@ -424,6 +424,7 @@ const SystemSettings = { // OpenRouter Keys OpenRouterApiKey: !!process.env.OPENROUTER_API_KEY, OpenRouterModelPref: process.env.OPENROUTER_MODEL_PREF, + OpenRouterTimeout: process.env.OPENROUTER_TIMEOUT_MS, // Mistral AI (API) Keys MistralApiKey: !!process.env.MISTRAL_API_KEY, diff --git a/server/utils/AiProviders/openRouter/index.js b/server/utils/AiProviders/openRouter/index.js index 7d0ff3e3b5c..c7d4dfb0b6c 100644 --- a/server/utils/AiProviders/openRouter/index.js +++ b/server/utils/AiProviders/openRouter/index.js @@ -38,6 +38,7 @@ class OpenRouterLLM { this.embedder = embedder ?? new NativeEmbedder(); this.defaultTemp = 0.7; + this.timeout = this.#parseTimeout(); if (!fs.existsSync(cacheFolder)) fs.mkdirSync(cacheFolder, { recursive: true }); @@ -49,6 +50,22 @@ class OpenRouterLLM { console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args); } + /** + * OpenRouter has various models that never return `finish_reasons` and thus leave the stream open + * which causes issues in subsequent messages. This timeout value forces us to close the stream after + * x milliseconds. This is a configurable value via the OPENROUTER_TIMEOUT_MS value + * @returns {number} The timeout value in milliseconds (default: 500) + */ + #parseTimeout() { + this.log( + `OpenRouter timeout is set to ${process.env.OPENROUTER_TIMEOUT_MS ?? 500}ms` + ); + if (isNaN(Number(process.env.OPENROUTER_TIMEOUT_MS))) return 500; + const setValue = Number(process.env.OPENROUTER_TIMEOUT_MS); + if (setValue < 500) return 500; + return setValue; + } + // This checks if the .cached_at file has a timestamp that is more than 1Week (in millis) // from the current date. If it is, then we will refetch the API so that all the models are up // to date. @@ -161,7 +178,7 @@ class OpenRouterLLM { } handleStream(response, stream, responseProps) { - const timeoutThresholdMs = 500; + const timeoutThresholdMs = this.timeout; const { uuid = uuidv4(), sources = [] } = responseProps; return new Promise(async (resolve) => { diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index d39941ec40f..afacb7279fb 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -365,6 +365,10 @@ const KEY_MAPPING = { envKey: "OPENROUTER_MODEL_PREF", checks: [isNotEmpty], }, + OpenRouterTimeout: { + envKey: "OPENROUTER_TIMEOUT_MS", + checks: [], + }, // Groq Options GroqApiKey: {