diff --git a/server/storage/models/.gitignore b/server/storage/models/.gitignore index f5c78ac877..5e83df7bcc 100644 --- a/server/storage/models/.gitignore +++ b/server/storage/models/.gitignore @@ -9,3 +9,4 @@ gemini togetherAi tesseract ppio +context-windows/* \ No newline at end of file diff --git a/server/utils/AiProviders/anthropic/index.js b/server/utils/AiProviders/anthropic/index.js index c4f53acf63..450b376b3e 100644 --- a/server/utils/AiProviders/anthropic/index.js +++ b/server/utils/AiProviders/anthropic/index.js @@ -45,11 +45,11 @@ class AnthropicLLM { } static promptWindowLimit(modelName) { - return MODEL_MAP.anthropic[modelName] ?? 100_000; + return MODEL_MAP.get("anthropic", modelName) ?? 100_000; } promptWindowLimit() { - return MODEL_MAP.anthropic[this.model] ?? 100_000; + return MODEL_MAP.get("anthropic", this.model) ?? 100_000; } isValidChatCompletionModel(_modelName = "") { diff --git a/server/utils/AiProviders/cohere/index.js b/server/utils/AiProviders/cohere/index.js index 33b65df21c..a6647f5b65 100644 --- a/server/utils/AiProviders/cohere/index.js +++ b/server/utils/AiProviders/cohere/index.js @@ -63,11 +63,11 @@ class CohereLLM { } static promptWindowLimit(modelName) { - return MODEL_MAP.cohere[modelName] ?? 4_096; + return MODEL_MAP.get("cohere", modelName) ?? 4_096; } promptWindowLimit() { - return MODEL_MAP.cohere[this.model] ?? 4_096; + return MODEL_MAP.get("cohere", this.model) ?? 4_096; } async isValidChatCompletionModel(model = "") { diff --git a/server/utils/AiProviders/deepseek/index.js b/server/utils/AiProviders/deepseek/index.js index b91332a84a..4b5b334bfe 100644 --- a/server/utils/AiProviders/deepseek/index.js +++ b/server/utils/AiProviders/deepseek/index.js @@ -29,7 +29,9 @@ class DeepSeekLLM { this.embedder = embedder ?? new NativeEmbedder(); this.defaultTemp = 0.7; - this.log("Initialized with model:", this.model); + this.log( + `Initialized ${this.model} with context window ${this.promptWindowLimit()}` + ); } log(text, ...args) { @@ -53,11 +55,11 @@ class DeepSeekLLM { } static promptWindowLimit(modelName) { - return MODEL_MAP.deepseek[modelName] ?? 8192; + return MODEL_MAP.get("deepseek", modelName) ?? 8192; } promptWindowLimit() { - return MODEL_MAP.deepseek[this.model] ?? 8192; + return MODEL_MAP.get("deepseek", this.model) ?? 8192; } async isValidChatCompletionModel(modelName = "") { diff --git a/server/utils/AiProviders/gemini/defaultModels.js b/server/utils/AiProviders/gemini/defaultModels.js index fde63341d2..4a52dc99c6 100644 --- a/server/utils/AiProviders/gemini/defaultModels.js +++ b/server/utils/AiProviders/gemini/defaultModels.js @@ -1,7 +1,7 @@ const { MODEL_MAP } = require("../modelMap"); const stableModels = [ - // %STABLE_MODELS% - updated 2025-04-07T20:29:49.276Z + // %STABLE_MODELS% - updated 2025-05-13T23:13:58.920Z "gemini-1.5-pro-001", "gemini-1.5-pro-002", "gemini-1.5-pro", @@ -14,6 +14,7 @@ const stableModels = [ "gemini-2.0-flash-001", "gemini-2.0-flash-lite-001", "gemini-2.0-flash-lite", + "gemini-2.0-flash-preview-image-generation", // %EOC_STABLE_MODELS% ]; @@ -22,7 +23,7 @@ const stableModels = [ // generally, v1beta models have `exp` in the name, but not always // so we check for both against a static list as well via API. const v1BetaModels = [ - // %V1BETA_MODELS% - updated 2025-04-07T20:29:49.276Z + // %V1BETA_MODELS% - updated 2025-05-13T23:13:58.920Z "gemini-1.5-pro-latest", "gemini-1.5-flash-latest", "gemini-1.5-flash-8b-latest", @@ -30,6 +31,9 @@ const v1BetaModels = [ "gemini-1.5-flash-8b-exp-0924", "gemini-2.5-pro-exp-03-25", "gemini-2.5-pro-preview-03-25", + "gemini-2.5-flash-preview-04-17", + "gemini-2.5-flash-preview-04-17-thinking", + "gemini-2.5-pro-preview-05-06", "gemini-2.0-flash-exp", "gemini-2.0-flash-exp-image-generation", "gemini-2.0-flash-lite-preview-02-05", @@ -41,6 +45,7 @@ const v1BetaModels = [ "gemini-2.0-flash-thinking-exp", "gemini-2.0-flash-thinking-exp-1219", "learnlm-1.5-pro-experimental", + "learnlm-2.0-flash-experimental", "gemma-3-1b-it", "gemma-3-4b-it", "gemma-3-12b-it", @@ -48,17 +53,17 @@ const v1BetaModels = [ // %EOC_V1BETA_MODELS% ]; -const defaultGeminiModels = [ +const defaultGeminiModels = () => [ ...stableModels.map((model) => ({ id: model, name: model, - contextWindow: MODEL_MAP.gemini[model], + contextWindow: MODEL_MAP.get("gemini", model), experimental: false, })), ...v1BetaModels.map((model) => ({ id: model, name: model, - contextWindow: MODEL_MAP.gemini[model], + contextWindow: MODEL_MAP.get("gemini", model), experimental: true, })), ]; diff --git a/server/utils/AiProviders/gemini/index.js b/server/utils/AiProviders/gemini/index.js index bd2268e517..c7d39bc782 100644 --- a/server/utils/AiProviders/gemini/index.js +++ b/server/utils/AiProviders/gemini/index.js @@ -107,7 +107,7 @@ class GeminiLLM { try { const cacheModelPath = path.resolve(cacheFolder, "models.json"); if (!fs.existsSync(cacheModelPath)) - return MODEL_MAP.gemini[modelName] ?? 30_720; + return MODEL_MAP.get("gemini", modelName) ?? 30_720; const models = safeJsonParse(fs.readFileSync(cacheModelPath)); const model = models.find((model) => model.id === modelName); @@ -118,15 +118,14 @@ class GeminiLLM { return model.contextWindow; } catch (e) { console.error(`GeminiLLM:promptWindowLimit`, e.message); - return MODEL_MAP.gemini[modelName] ?? 30_720; + return MODEL_MAP.get("gemini", modelName) ?? 30_720; } } promptWindowLimit() { try { if (!fs.existsSync(this.cacheModelPath)) - return MODEL_MAP.gemini[this.model] ?? 30_720; - + return MODEL_MAP.get("gemini", this.model) ?? 30_720; const models = safeJsonParse(fs.readFileSync(this.cacheModelPath)); const model = models.find((model) => model.id === this.model); if (!model) @@ -136,7 +135,7 @@ class GeminiLLM { return model.contextWindow; } catch (e) { console.error(`GeminiLLM:promptWindowLimit`, e.message); - return MODEL_MAP.gemini[this.model] ?? 30_720; + return MODEL_MAP.get("gemini", this.model) ?? 30_720; } } @@ -281,7 +280,7 @@ class GeminiLLM { if (allModels.length === 0) { console.error(`Gemini:getGeminiModels - No models found`); - return defaultGeminiModels; + return defaultGeminiModels(); } console.log( diff --git a/server/utils/AiProviders/gemini/syncStaticLists.mjs b/server/utils/AiProviders/gemini/syncStaticLists.mjs index 42382fae38..b276995acd 100644 --- a/server/utils/AiProviders/gemini/syncStaticLists.mjs +++ b/server/utils/AiProviders/gemini/syncStaticLists.mjs @@ -9,7 +9,6 @@ import fs from "fs"; import path from "path"; import dotenv from "dotenv"; -import { MODEL_MAP } from "../modelMap.js"; dotenv.config({ path: `../../../.env.development` }); const existingCachePath = path.resolve('../../../storage/models/gemini') @@ -46,34 +45,4 @@ function updateDefaultModelsFile(models) { fs.writeFileSync(path.join("./defaultModels.js"), defaultModelFileContents); console.log("Updated defaultModels.js. Dont forget to `yarn lint` and commit!"); } - -function updateModelMap(models) { - const existingModelMap = MODEL_MAP; - console.log('Updating modelMap.js `gemini` object...') - console.log(`Removed existing gemini object (${Object.keys(existingModelMap.gemini).length} models) from modelMap.js`); - existingModelMap.gemini = {}; - - for (const model of models) existingModelMap.gemini[model.id] = model.contextWindow; - console.log(`Updated modelMap.js 'gemini' object with ${Object.keys(existingModelMap.gemini).length} models from API`); - - // Update the modelMap.js file - const contents = `/** - * The model name and context window for all know model windows - * that are available through providers which has discrete model options. - * This file is automatically generated by syncStaticLists.mjs - * and should not be edited manually. - * - * Last updated: ${new Date().toISOString()} - */ -const MODEL_MAP = { - ${Object.entries(existingModelMap).map(([key, value]) => `${key}: ${JSON.stringify(value, null, 2)}`).join(',\n')} -}; - -module.exports = { MODEL_MAP }; -`; - fs.writeFileSync(path.resolve("../modelMap.js"), contents); - console.log('Updated modelMap.js `gemini` object. Dont forget to `yarn lint` and commit!'); -} - updateDefaultModelsFile(models); -updateModelMap(models); diff --git a/server/utils/AiProviders/groq/index.js b/server/utils/AiProviders/groq/index.js index 9e7e77fa16..8eddefab0c 100644 --- a/server/utils/AiProviders/groq/index.js +++ b/server/utils/AiProviders/groq/index.js @@ -49,11 +49,11 @@ class GroqLLM { } static promptWindowLimit(modelName) { - return MODEL_MAP.groq[modelName] ?? 8192; + return MODEL_MAP.get("groq", modelName) ?? 8192; } promptWindowLimit() { - return MODEL_MAP.groq[this.model] ?? 8192; + return MODEL_MAP.get("groq", this.model) ?? 8192; } async isValidChatCompletionModel(modelName = "") { diff --git a/server/utils/AiProviders/modelMap/index.js b/server/utils/AiProviders/modelMap/index.js new file mode 100644 index 0000000000..dbf0aff6a6 --- /dev/null +++ b/server/utils/AiProviders/modelMap/index.js @@ -0,0 +1,140 @@ +const path = require("path"); +const fs = require("fs"); +const LEGACY_MODEL_MAP = require("./legacy"); + +class ContextWindowFinder { + static instance = null; + static modelMap = LEGACY_MODEL_MAP; + + /** + * Mapping for AnythingLLM provider <> LiteLLM provider + * @type {Record} + */ + static trackedProviders = { + anthropic: "anthropic", + openai: "openai", + cohere: "cohere_chat", + gemini: "vertex_ai-language-models", + groq: "groq", + xai: "xai", + deepseek: "deepseek", + }; + static expiryMs = 1000 * 60 * 60 * 24 * 3; // 3 days + static remoteUrl = + "https://raw.githubusercontent.com/BerriAI/litellm/main/model_prices_and_context_window.json"; + + cacheLocation = path.resolve( + process.env.STORAGE_DIR + ? path.resolve(process.env.STORAGE_DIR, "models", "context-windows") + : path.resolve(__dirname, `../../../storage/models/context-windows`) + ); + cacheFilePath = path.resolve(this.cacheLocation, "context-windows.json"); + cacheFileExpiryPath = path.resolve(this.cacheLocation, ".cached_at"); + + constructor() { + if (ContextWindowFinder.instance) return ContextWindowFinder.instance; + ContextWindowFinder.instance = this; + if (!fs.existsSync(this.cacheLocation)) + fs.mkdirSync(this.cacheLocation, { recursive: true }); + this.#pullRemoteModelMap(); + } + + log(text, ...args) { + console.log(`\x1b[33m[ContextWindowFinder]\x1b[0m ${text}`, ...args); + } + + /** + * Checks if the cache is stale by checking if the cache file exists and if the cache file is older than the expiry time. + * @returns {boolean} + */ + get isCacheStale() { + if (!fs.existsSync(this.cacheFileExpiryPath)) return true; + const cachedAt = fs.readFileSync(this.cacheFileExpiryPath, "utf8"); + return Date.now() - cachedAt > ContextWindowFinder.expiryMs; + } + + get cache() { + if (!fs.existsSync(this.cacheFileExpiryPath)) return null; + if (!this.isCacheStale) + return JSON.parse( + fs.readFileSync(this.cacheFilePath, { encoding: "utf8" }) + ); + return null; + } + + /** + * Pulls the remote model map from the remote URL, formats it and caches it. + * @returns {Record>} - The formatted model map + */ + async #pullRemoteModelMap() { + const remoteContexWindowMap = await fetch(ContextWindowFinder.remoteUrl) + .then((res) => res.json()) + .then((data) => { + fs.writeFileSync(this.cacheFilePath, JSON.stringify(data, null, 2)); + fs.writeFileSync(this.cacheFileExpiryPath, Date.now().toString()); + this.log("Remote model map synced and cached"); + return data; + }) + .catch((error) => { + this.log("Error syncing remote model map", error); + return null; + }); + if (!remoteContexWindowMap) return null; + + const modelMap = this.#formatModelMap(remoteContexWindowMap); + fs.writeFileSync(this.cacheFilePath, JSON.stringify(modelMap, null, 2)); + fs.writeFileSync(this.cacheFileExpiryPath, Date.now().toString()); + return modelMap; + } + + /** + * Formats the remote model map to a format that is compatible with how we store the model map + * for all providers who use it. + * @param {Record} modelMap - The remote model map + * @returns {Record>} - The formatted model map + */ + #formatModelMap(modelMap = {}) { + const formattedModelMap = {}; + + for (const [provider, liteLLMProviderTag] of Object.entries( + ContextWindowFinder.trackedProviders + )) { + formattedModelMap[provider] = {}; + const matches = Object.entries(modelMap).filter( + ([_key, config]) => config.litellm_provider === liteLLMProviderTag + ); + for (const [key, config] of matches) { + const contextWindow = Number(config.max_input_tokens); + if (isNaN(contextWindow)) continue; + + // Some models have a provider/model-tag format, so we need to get the last part since we dont do paths + // for names with the exception of some router-providers like OpenRouter or Together. + const modelName = key.split("/").pop(); + formattedModelMap[provider][modelName] = contextWindow; + } + } + return formattedModelMap; + } + + /** + * Gets the context window for a given provider and model. + * @param {string} provider - The provider to get the context window for + * @param {string} model - The model to get the context window for + * @returns {number} - The context window for the given provider and model + */ + get(provider = null, model = null) { + if (!provider || !this.cache || !this.cache[provider]) return null; + if (!model) return this.cache[provider]; + const modelContextWindow = this.cache[provider][model]; + if (!modelContextWindow) { + this.log("Invalid access to model context window - not found in cache", { + provider, + model, + }); + return null; + } + return Number(modelContextWindow); + } +} + +module.exports = { MODEL_MAP: new ContextWindowFinder() }; diff --git a/server/utils/AiProviders/modelMap.js b/server/utils/AiProviders/modelMap/legacy.js similarity index 90% rename from server/utils/AiProviders/modelMap.js rename to server/utils/AiProviders/modelMap/legacy.js index a80400c30b..2faf99dc23 100644 --- a/server/utils/AiProviders/modelMap.js +++ b/server/utils/AiProviders/modelMap/legacy.js @@ -1,12 +1,4 @@ -/** - * The model name and context window for all know model windows - * that are available through providers which has discrete model options. - * This file is automatically generated by syncStaticLists.mjs - * and should not be edited manually. - * - * Last updated: 2025-04-07T20:29:49.277Z - */ -const MODEL_MAP = { +const LEGACY_MODEL_MAP = { anthropic: { "claude-instant-1.2": 100000, "claude-2.0": 100000, @@ -117,5 +109,4 @@ const MODEL_MAP = { "grok-beta": 131072, }, }; - -module.exports = { MODEL_MAP }; +module.exports = LEGACY_MODEL_MAP; diff --git a/server/utils/AiProviders/openAi/index.js b/server/utils/AiProviders/openAi/index.js index 2e5996c939..ee33a12bac 100644 --- a/server/utils/AiProviders/openAi/index.js +++ b/server/utils/AiProviders/openAi/index.js @@ -25,6 +25,13 @@ class OpenAiLLM { this.embedder = embedder ?? new NativeEmbedder(); this.defaultTemp = 0.7; + this.log( + `Initialized ${this.model} with context window ${this.promptWindowLimit()}` + ); + } + + log(text, ...args) { + console.log(`\x1b[36m[${this.constructor.name}]\x1b[0m ${text}`, ...args); } /** @@ -54,11 +61,11 @@ class OpenAiLLM { } static promptWindowLimit(modelName) { - return MODEL_MAP.openai[modelName] ?? 4_096; + return MODEL_MAP.get("openai", modelName) ?? 4_096; } promptWindowLimit() { - return MODEL_MAP.openai[this.model] ?? 4_096; + return MODEL_MAP.get("openai", this.model) ?? 4_096; } // Short circuit if name has 'gpt' since we now fetch models from OpenAI API diff --git a/server/utils/AiProviders/xai/index.js b/server/utils/AiProviders/xai/index.js index 2319e72206..f911627094 100644 --- a/server/utils/AiProviders/xai/index.js +++ b/server/utils/AiProviders/xai/index.js @@ -28,7 +28,9 @@ class XAiLLM { this.embedder = embedder ?? new NativeEmbedder(); this.defaultTemp = 0.7; - this.log("Initialized with model:", this.model); + this.log( + `Initialized ${this.model} with context window ${this.promptWindowLimit()}` + ); } log(text, ...args) { @@ -52,11 +54,11 @@ class XAiLLM { } static promptWindowLimit(modelName) { - return MODEL_MAP.xai[modelName] ?? 131_072; + return MODEL_MAP.get("xai", modelName) ?? 131_072; } promptWindowLimit() { - return MODEL_MAP.xai[this.model] ?? 131_072; + return MODEL_MAP.get("xai", this.model) ?? 131_072; } isValidChatCompletionModel(_modelName = "") {