From d71c3097ce8735fc62035b43f09a2fd4506c5816 Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Mon, 21 Oct 2024 14:31:27 -0700 Subject: [PATCH 1/4] Rework failover agent invocation selection for agent and ephemmeral agent --- server/utils/agents/ephemeral.js | 55 +++++++++++++++++---- server/utils/agents/index.js | 82 ++++++++++++++++---------------- 2 files changed, 86 insertions(+), 51 deletions(-) diff --git a/server/utils/agents/ephemeral.js b/server/utils/agents/ephemeral.js index 7b05b4adc83..ef391035598 100644 --- a/server/utils/agents/ephemeral.js +++ b/server/utils/agents/ephemeral.js @@ -99,29 +99,64 @@ class EphemeralAgentHandler extends AgentHandler { } } + /** + * Attempts to find a fallback provider and model to use if the workspace + * does not have an explicit `agentProvider` and `agentModel` set. + * 1. Fallback to the workspace `chatProvider` and `chatModel` if they exist. + * 2. Fallback to the system `LLM_PROVIDER` and try to load the the associated default model via ENV params or a base available model. + * 3. Otherwise, return null - will likely throw an error the user can act on. + * @returns {object|null} - An object with provider and model keys. + */ + #getFallbackProvider() { + // First, fallback to the workspace chat provider and model if they exist + if (this.#workspace.chatProvider && this.#workspace.chatModel) { + return { + provider: this.#workspace.chatProvider, + model: this.#workspace.chatModel, + }; + } + + // If workspace does not have chat provider and model fallback + // to system provider and try to load provider default model + const systemProvider = process.env.LLM_PROVIDER; + const systemModel = this.providerDefault(systemProvider); + if (systemProvider && systemModel) { + return { + provider: systemProvider, + model: systemModel, + }; + } + + return null; + } + /** * Finds or assumes the model preference value to use for API calls. * If multi-model loading is supported, we use their agent model selection of the workspace * If not supported, we attempt to fallback to the system provider value for the LLM preference * and if that fails - we assume a reasonable base model to exist. - * @returns {string} the model preference value to use in API calls + * @returns {string|null} the model preference value to use in API calls */ #fetchModel() { - if (!Object.keys(this.noProviderModelDefault).includes(this.provider)) - return this.#workspace.agentModel || this.providerDefault(); + // Provider was not explicitly set for workspace, so we are going to run our fallback logic + // that will set a provider and model for us to use. + if (!this.provider) { + const fallback = this.#getFallbackProvider(); + if (!fallback) throw new Error("No valid provider found for the agent."); + this.provider = fallback.provider; // re-set the provider to the fallback provider so it is not null. + return fallback.model; // set its defined model based on fallback logic. + } - // Provider has no reliable default (cant load many models) - so we need to look at system - // for the model param. - const sysModelKey = this.noProviderModelDefault[this.provider]; - if (!!sysModelKey) - return process.env[sysModelKey] ?? this.providerDefault(); + // The provider was explicitly set, so check if the workspace has an agent model set. + if (this.#workspace.agentModel) return this.#workspace.agentModel; - // If all else fails - look at the provider default list + // Otherwise, we have no model to use - so guess a default model to use via the provider + // and it's system ENV params and if that fails - we return either a base model or null. return this.providerDefault(); } #providerSetupAndCheck() { - this.provider = this.#workspace.agentProvider; + this.provider = this.#workspace.agentProvider ?? null; this.model = this.#fetchModel(); this.log(`Start ${this.#invocationUUID}::${this.provider}:${this.model}`); this.checkSetup(); diff --git a/server/utils/agents/index.js b/server/utils/agents/index.js index ab86e51ff0e..3413957d8be 100644 --- a/server/utils/agents/index.js +++ b/server/utils/agents/index.js @@ -11,13 +11,6 @@ const ImportedPlugin = require("./imported"); class AgentHandler { #invocationUUID; #funcsToLoad = []; - noProviderModelDefault = { - azure: "OPEN_MODEL_PREF", - lmstudio: "LMSTUDIO_MODEL_PREF", - textgenwebui: null, // does not even use `model` in API req - "generic-openai": "GENERIC_OPEN_AI_MODEL_PREF", - bedrock: "AWS_BEDROCK_LLM_MODEL_PREFERENCE", - }; invocation = null; aibitat = null; channel = null; @@ -184,53 +177,70 @@ class AgentHandler { } } + /** + * Finds the default model for a given provider. If no default model is set for it's associated ENV then + * it will return a reasonable base model for the provider if one exists. + * @param {string} provider - The provider to find the default model for. + * @returns {string|null} The default model for the provider. + */ providerDefault(provider = this.provider) { switch (provider) { case "openai": - return "gpt-4o"; + return process.env.OPEN_MODEL_PREF ?? "gpt-4o"; case "anthropic": - return "claude-3-sonnet-20240229"; + return process.env.ANTHROPIC_MODEL_PREF ?? "claude-3-sonnet-20240229"; case "lmstudio": - return "server-default"; + return process.env.LMSTUDIO_MODEL_PREF ?? "server-default"; case "ollama": - return "llama3:latest"; + return process.env.OLLAMA_MODEL_PREF ?? "llama3:latest"; case "groq": - return "llama3-70b-8192"; + return process.env.GROQ_MODEL_PREF ?? "llama3-70b-8192"; case "togetherai": - return "mistralai/Mixtral-8x7B-Instruct-v0.1"; + return ( + process.env.TOGETHER_AI_MODEL_PREF ?? + "mistralai/Mixtral-8x7B-Instruct-v0.1" + ); case "azure": - return "gpt-3.5-turbo"; - case "koboldcpp": return null; + case "koboldcpp": + return process.env.KOBOLD_CPP_MODEL_PREF ?? null; case "gemini": - return "gemini-pro"; + return process.env.GEMINI_MODEL_PREF ?? "gemini-pro"; case "localai": - return null; + return process.env.LOCAL_AI_MODEL_PREF ?? null; case "openrouter": - return "openrouter/auto"; + return process.env.OPENROUTER_MODEL_PREF ?? "openrouter/auto"; case "mistral": - return "mistral-medium"; + return process.env.MISTRAL_MODEL_PREF ?? "mistral-medium"; case "generic-openai": - return null; + return process.env.GENERIC_OPEN_AI_MODEL_PREF ?? null; case "perplexity": - return "sonar-small-online"; + return process.env.PERPLEXITY_MODEL_PREF ?? "sonar-small-online"; case "textgenwebui": return null; case "bedrock": - return null; + return process.env.AWS_BEDROCK_LLM_MODEL_PREFERENCE ?? null; case "fireworksai": - return null; + return process.env.FIREWORKS_AI_LLM_MODEL_PREF ?? null; case "deepseek": - return "deepseek-chat"; + return process.env.DEEPSEEK_MODEL_PREF ?? "deepseek-chat"; case "litellm": - return null; + return process.env.LITE_LLM_MODEL_PREF ?? null; case "apipie": - return null; + return process.env.APIPIE_LLM_MODEL_PREF ?? null; default: - return "unknown"; + return null; } } + /** + * Attempts to find a fallback provider and model to use if the workspace + * does not have an explicit `agentProvider` and `agentModel` set. + * 1. Fallback to the workspace `chatProvider` and `chatModel` if they exist. + * 2. Fallback to the system `LLM_PROVIDER` and try to load the the associated default model via ENV params or a base available model. + * 3. Otherwise, return null - will likely throw an error the user can act on. + * @returns {object|null} - An object with provider and model keys. + */ #getFallbackProvider() { // First, fallback to the workspace chat provider and model if they exist if ( @@ -262,7 +272,7 @@ class AgentHandler { * If multi-model loading is supported, we use their agent model selection of the workspace * If not supported, we attempt to fallback to the system provider value for the LLM preference * and if that fails - we assume a reasonable base model to exist. - * @returns {string} the model preference value to use in API calls + * @returns {string|null} the model preference value to use in API calls */ #fetchModel() { // Provider was not explicitly set for workspace, so we are going to run our fallback logic @@ -275,21 +285,11 @@ class AgentHandler { } // The provider was explicitly set, so check if the workspace has an agent model set. - if (this.invocation.workspace.agentModel) { + if (this.invocation.workspace.agentModel) return this.invocation.workspace.agentModel; - } - - // If the provider we are using is not supported or does not support multi-model loading - // then we use the default model for the provider. - if (!Object.keys(this.noProviderModelDefault).includes(this.provider)) { - return this.providerDefault(); - } - - // Load the model from the system environment variable for providers with no multi-model loading. - const sysModelKey = this.noProviderModelDefault[this.provider]; - if (sysModelKey) return process.env[sysModelKey] ?? this.providerDefault(); - // Otherwise, we have no model to use - so guess a default model to use. + // Otherwise, we have no model to use - so guess a default model to use via the provider + // and it's system ENV params and if that fails - we return either a base model or null. return this.providerDefault(); } From 5d53cbd5964a1363cd2a967629f70c9e897e7432 Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Mon, 21 Oct 2024 14:36:39 -0700 Subject: [PATCH 2/4] update order of check --- server/utils/agents/ephemeral.js | 9 ++++++--- server/utils/agents/index.js | 9 ++++----- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/server/utils/agents/ephemeral.js b/server/utils/agents/ephemeral.js index ef391035598..443fc4379f6 100644 --- a/server/utils/agents/ephemeral.js +++ b/server/utils/agents/ephemeral.js @@ -138,6 +138,9 @@ class EphemeralAgentHandler extends AgentHandler { * @returns {string|null} the model preference value to use in API calls */ #fetchModel() { + // The provider was explicitly set, so check if the workspace has an agent model set. + if (this.#workspace.agentModel) return this.#workspace.agentModel; + // Provider was not explicitly set for workspace, so we are going to run our fallback logic // that will set a provider and model for us to use. if (!this.provider) { @@ -147,9 +150,6 @@ class EphemeralAgentHandler extends AgentHandler { return fallback.model; // set its defined model based on fallback logic. } - // The provider was explicitly set, so check if the workspace has an agent model set. - if (this.#workspace.agentModel) return this.#workspace.agentModel; - // Otherwise, we have no model to use - so guess a default model to use via the provider // and it's system ENV params and if that fails - we return either a base model or null. return this.providerDefault(); @@ -158,6 +158,9 @@ class EphemeralAgentHandler extends AgentHandler { #providerSetupAndCheck() { this.provider = this.#workspace.agentProvider ?? null; this.model = this.#fetchModel(); + + if (!this.provider) + throw new Error("No valid provider found for the agent."); this.log(`Start ${this.#invocationUUID}::${this.provider}:${this.model}`); this.checkSetup(); } diff --git a/server/utils/agents/index.js b/server/utils/agents/index.js index 3413957d8be..529cd81318b 100644 --- a/server/utils/agents/index.js +++ b/server/utils/agents/index.js @@ -275,6 +275,10 @@ class AgentHandler { * @returns {string|null} the model preference value to use in API calls */ #fetchModel() { + // The provider was explicitly set, so check if the workspace has an agent model set. + if (this.invocation.workspace.agentModel) + return this.invocation.workspace.agentModel; + // Provider was not explicitly set for workspace, so we are going to run our fallback logic // that will set a provider and model for us to use. if (!this.provider) { @@ -284,10 +288,6 @@ class AgentHandler { return fallback.model; // set its defined model based on fallback logic. } - // The provider was explicitly set, so check if the workspace has an agent model set. - if (this.invocation.workspace.agentModel) - return this.invocation.workspace.agentModel; - // Otherwise, we have no model to use - so guess a default model to use via the provider // and it's system ENV params and if that fails - we return either a base model or null. return this.providerDefault(); @@ -299,7 +299,6 @@ class AgentHandler { if (!this.provider) throw new Error("No valid provider found for the agent."); - this.log(`Start ${this.#invocationUUID}::${this.provider}:${this.model}`); this.checkSetup(); } From d5270d62c51e867b81c493d9ca3e4f2c51d201fe Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Mon, 21 Oct 2024 14:38:12 -0700 Subject: [PATCH 3/4] update order of check --- server/utils/agents/ephemeral.js | 6 +++--- server/utils/agents/index.js | 7 +++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/server/utils/agents/ephemeral.js b/server/utils/agents/ephemeral.js index 443fc4379f6..a02472f736e 100644 --- a/server/utils/agents/ephemeral.js +++ b/server/utils/agents/ephemeral.js @@ -138,9 +138,6 @@ class EphemeralAgentHandler extends AgentHandler { * @returns {string|null} the model preference value to use in API calls */ #fetchModel() { - // The provider was explicitly set, so check if the workspace has an agent model set. - if (this.#workspace.agentModel) return this.#workspace.agentModel; - // Provider was not explicitly set for workspace, so we are going to run our fallback logic // that will set a provider and model for us to use. if (!this.provider) { @@ -150,6 +147,9 @@ class EphemeralAgentHandler extends AgentHandler { return fallback.model; // set its defined model based on fallback logic. } + // The provider was explicitly set, so check if the workspace has an agent model set. + if (this.invocation.workspace.agentModel) return this.invocation.workspace.agentModel; + // Otherwise, we have no model to use - so guess a default model to use via the provider // and it's system ENV params and if that fails - we return either a base model or null. return this.providerDefault(); diff --git a/server/utils/agents/index.js b/server/utils/agents/index.js index 529cd81318b..cef6c829fa3 100644 --- a/server/utils/agents/index.js +++ b/server/utils/agents/index.js @@ -275,10 +275,6 @@ class AgentHandler { * @returns {string|null} the model preference value to use in API calls */ #fetchModel() { - // The provider was explicitly set, so check if the workspace has an agent model set. - if (this.invocation.workspace.agentModel) - return this.invocation.workspace.agentModel; - // Provider was not explicitly set for workspace, so we are going to run our fallback logic // that will set a provider and model for us to use. if (!this.provider) { @@ -288,6 +284,9 @@ class AgentHandler { return fallback.model; // set its defined model based on fallback logic. } + // The provider was explicitly set, so check if the workspace has an agent model set. + if (this.invocation.workspace.agentModel) return this.invocation.workspace.agentModel; + // Otherwise, we have no model to use - so guess a default model to use via the provider // and it's system ENV params and if that fails - we return either a base model or null. return this.providerDefault(); From e2546933170f1fbfd7cef46c8403b65a3aab3b0e Mon Sep 17 00:00:00 2001 From: timothycarambat Date: Mon, 21 Oct 2024 14:39:08 -0700 Subject: [PATCH 4/4] lint --- server/utils/agents/ephemeral.js | 3 ++- server/utils/agents/index.js | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/server/utils/agents/ephemeral.js b/server/utils/agents/ephemeral.js index a02472f736e..1ab2a988a2f 100644 --- a/server/utils/agents/ephemeral.js +++ b/server/utils/agents/ephemeral.js @@ -148,7 +148,8 @@ class EphemeralAgentHandler extends AgentHandler { } // The provider was explicitly set, so check if the workspace has an agent model set. - if (this.invocation.workspace.agentModel) return this.invocation.workspace.agentModel; + if (this.invocation.workspace.agentModel) + return this.invocation.workspace.agentModel; // Otherwise, we have no model to use - so guess a default model to use via the provider // and it's system ENV params and if that fails - we return either a base model or null. diff --git a/server/utils/agents/index.js b/server/utils/agents/index.js index cef6c829fa3..98caea5cd37 100644 --- a/server/utils/agents/index.js +++ b/server/utils/agents/index.js @@ -285,7 +285,8 @@ class AgentHandler { } // The provider was explicitly set, so check if the workspace has an agent model set. - if (this.invocation.workspace.agentModel) return this.invocation.workspace.agentModel; + if (this.invocation.workspace.agentModel) + return this.invocation.workspace.agentModel; // Otherwise, we have no model to use - so guess a default model to use via the provider // and it's system ENV params and if that fails - we return either a base model or null.