From dbeb806f0f3eb67d272c5ddf57a586efc8f2378c Mon Sep 17 00:00:00 2001 From: shatfield4 Date: Thu, 16 Nov 2023 11:31:32 -0800 Subject: [PATCH 1/2] allow use of any embedder for any llm/update data handling modal --- .../EmbeddingPreference/index.jsx | 285 +++++++++--------- .../Steps/DataHandling/index.jsx | 56 +++- .../Steps/LLMSelection/index.jsx | 10 +- .../OnboardingFlow/OnboardingModal/index.jsx | 3 +- server/endpoints/admin.js | 12 +- server/endpoints/api/admin/index.js | 12 +- 6 files changed, 202 insertions(+), 176 deletions(-) diff --git a/frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx b/frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx index c81b0e527e3..56c074ae9c8 100644 --- a/frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx +++ b/frontend/src/pages/GeneralSettings/EmbeddingPreference/index.jsx @@ -107,161 +107,148 @@ export default function GeneralEmbeddingPreference() {

- {["openai", "azure"].includes(settings.LLMProvider) ? ( -
-

- Your current LLM preference does not require you to set up - this part of AnythingLLM. -
- Embedding is being automatically managed by AnythingLLM. -

+ <> +
+ Embedding Providers +
+
+ + + +
- ) : ( - <> -
- Embedding Providers -
-
- - - - -
-
- {embeddingChoice === "openai" && ( - <> -
- - -
- - )} +
+ {embeddingChoice === "openai" && ( + <> +
+ + +
+ + )} - {embeddingChoice === "azure" && ( - <> -
- - -
+ {embeddingChoice === "azure" && ( + <> +
+ + +
-
- - -
+
+ + +
-
- - -
- - )} +
+ + +
+ + )} - {embeddingChoice === "localai" && ( - <> -
- - setBasePathValue(e.target.value)} - onBlur={updateBasePath} - required={true} - autoComplete="off" - spellCheck={false} - /> -
- +
+ + setBasePathValue(e.target.value)} + onBlur={updateBasePath} + required={true} + autoComplete="off" + spellCheck={false} /> - - )} -
- - )} +
+ + + )} +
+
diff --git a/frontend/src/pages/OnboardingFlow/OnboardingModal/Steps/DataHandling/index.jsx b/frontend/src/pages/OnboardingFlow/OnboardingModal/Steps/DataHandling/index.jsx index 2f133ff5b7e..dc24fef715d 100644 --- a/frontend/src/pages/OnboardingFlow/OnboardingModal/Steps/DataHandling/index.jsx +++ b/frontend/src/pages/OnboardingFlow/OnboardingModal/Steps/DataHandling/index.jsx @@ -97,16 +97,44 @@ const VECTOR_DB_PRIVACY = { }, }; +const EMBEDDING_ENGINE_PRIVACY = { + openai: { + name: "OpenAI", + description: [ + "Your documents are visible to OpenAI", + "Your documents are not used for training", + ], + logo: OpenAiLogo, + }, + azure: { + name: "Azure OpenAI", + description: [ + "Your documents are not visible to OpenAI or Microsoft", + "Your documents not used for training", + ], + logo: AzureOpenAiLogo, + }, + localai: { + name: "LocalAI", + description: [ + "Your documents are only accessible on the server running LocalAI", + ], + logo: LocalAiLogo, + }, +}; + function DataHandling({ nextStep, prevStep, currentStep }) { const [llmChoice, setLLMChoice] = useState("openai"); const [loading, setLoading] = useState(true); const [vectorDb, setVectorDb] = useState("pinecone"); + const [embeddingEngine, setEmbeddingEngine] = useState("openai"); useEffect(() => { async function fetchKeys() { const _settings = await System.keys(); setLLMChoice(_settings?.LLMProvider); setVectorDb(_settings?.VectorDB); + setEmbeddingEngine(_settings?.EmbeddingEngine); setLoading(false); } @@ -125,7 +153,7 @@ function DataHandling({ nextStep, prevStep, currentStep }) { return (
-
+
LLM Selection
-
+
Vector Database
+ {llmChoice !== embeddingEngine && ( +
+
+ Embedding Engine +
+
+ Vector DB Logo +

+ {EMBEDDING_ENGINE_PRIVACY[embeddingEngine].name} +

+
+
    + {EMBEDDING_ENGINE_PRIVACY[embeddingEngine].description.map( + (desc) => ( +
  • {desc}
    • + ) + )} +
+
+ )}
+
+ + +
)} diff --git a/frontend/src/pages/OnboardingFlow/OnboardingModal/Steps/DataHandling/index.jsx b/frontend/src/pages/OnboardingFlow/OnboardingModal/Steps/DataHandling/index.jsx index dc24fef715d..b81a6795e07 100644 --- a/frontend/src/pages/OnboardingFlow/OnboardingModal/Steps/DataHandling/index.jsx +++ b/frontend/src/pages/OnboardingFlow/OnboardingModal/Steps/DataHandling/index.jsx @@ -152,8 +152,8 @@ function DataHandling({ nextStep, prevStep, currentStep }) { return (
-
-
+
+
LLM Selection
-
+
+
Embedding Engine
+
+ Vector DB Logo +

+ {EMBEDDING_ENGINE_PRIVACY[embeddingEngine].name} +

+
+
    + {EMBEDDING_ENGINE_PRIVACY[embeddingEngine].description.map( + (desc) => ( +
  • {desc}
    • + ) + )} +
+
+ +
Vector Database
- {llmChoice !== embeddingEngine && ( -
-
- Embedding Engine -
-
- Vector DB Logo -

- {EMBEDDING_ENGINE_PRIVACY[embeddingEngine].name} -

-
-
    - {EMBEDDING_ENGINE_PRIVACY[embeddingEngine].description.map( - (desc) => ( -
  • {desc}
    • - ) - )} -
-
- )}
+
+ + +
)} diff --git a/server/utils/AiProviders/azureOpenAi/index.js b/server/utils/AiProviders/azureOpenAi/index.js index a424902b166..82e28204bca 100644 --- a/server/utils/AiProviders/azureOpenAi/index.js +++ b/server/utils/AiProviders/azureOpenAi/index.js @@ -1,9 +1,8 @@ const { AzureOpenAiEmbedder } = require("../../EmbeddingEngines/azureOpenAi"); const { chatPrompt } = require("../../chats"); -class AzureOpenAiLLM extends AzureOpenAiEmbedder { - constructor() { - super(); +class AzureOpenAiLLM { + constructor(embedder = null) { const { OpenAIClient, AzureKeyCredential } = require("@azure/openai"); if (!process.env.AZURE_OPENAI_ENDPOINT) throw new Error("No Azure API endpoint was set."); @@ -20,6 +19,12 @@ class AzureOpenAiLLM extends AzureOpenAiEmbedder { system: this.promptWindowLimit() * 0.15, user: this.promptWindowLimit() * 0.7, }; + + if (!embedder) + console.warn( + "No embedding provider defined for AzureOpenAiLLM - falling back to AzureOpenAiEmbedder for embedding!" + ); + this.embedder = !embedder ? new AzureOpenAiEmbedder() : embedder; } streamingEnabled() { @@ -114,6 +119,14 @@ Context: return data.choices[0].message.content; } + // Simple wrapper for dynamic embedder & normalize interface for all LLM implementations + async embedTextInput(textInput) { + return await this.embedder.embedTextInput(textInput); + } + async embedChunks(textChunks = []) { + return await this.embedder.embedChunks(textChunks); + } + async compressMessages(promptArgs = {}, rawHistory = []) { const { messageArrayCompressor } = require("../../helpers/chat"); const messageArray = this.constructPrompt(promptArgs); diff --git a/server/utils/AiProviders/openAi/index.js b/server/utils/AiProviders/openAi/index.js index 0c5b7116dd2..46464271968 100644 --- a/server/utils/AiProviders/openAi/index.js +++ b/server/utils/AiProviders/openAi/index.js @@ -1,9 +1,8 @@ const { OpenAiEmbedder } = require("../../EmbeddingEngines/openAi"); const { chatPrompt } = require("../../chats"); -class OpenAiLLM extends OpenAiEmbedder { - constructor() { - super(); +class OpenAiLLM { + constructor(embedder = null) { const { Configuration, OpenAIApi } = require("openai"); if (!process.env.OPEN_AI_KEY) throw new Error("No OpenAI API key was set."); @@ -17,6 +16,12 @@ class OpenAiLLM extends OpenAiEmbedder { system: this.promptWindowLimit() * 0.15, user: this.promptWindowLimit() * 0.7, }; + + if (!embedder) + console.warn( + "No embedding provider defined for OpenAiLLM - falling back to OpenAiEmbedder for embedding!" + ); + this.embedder = !embedder ? new OpenAiEmbedder() : embedder; } streamingEnabled() { @@ -203,6 +208,14 @@ Context: return streamRequest; } + // Simple wrapper for dynamic embedder & normalize interface for all LLM implementations + async embedTextInput(textInput) { + return await this.embedder.embedTextInput(textInput); + } + async embedChunks(textChunks = []) { + return await this.embedder.embedChunks(textChunks); + } + async compressMessages(promptArgs = {}, rawHistory = []) { const { messageArrayCompressor } = require("../../helpers/chat"); const messageArray = this.constructPrompt(promptArgs); diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js index c7c61822141..74804b90423 100644 --- a/server/utils/helpers/index.js +++ b/server/utils/helpers/index.js @@ -23,25 +23,22 @@ function getVectorDbClass() { function getLLMProvider() { const vectorSelection = process.env.LLM_PROVIDER || "openai"; - let embedder = null; + const embedder = getEmbeddingEngineSelection(); switch (vectorSelection) { case "openai": const { OpenAiLLM } = require("../AiProviders/openAi"); - return new OpenAiLLM(); + return new OpenAiLLM(embedder); case "azure": const { AzureOpenAiLLM } = require("../AiProviders/azureOpenAi"); - return new AzureOpenAiLLM(); + return new AzureOpenAiLLM(embedder); case "anthropic": const { AnthropicLLM } = require("../AiProviders/anthropic"); - embedder = getEmbeddingEngineSelection(); return new AnthropicLLM(embedder); case "lmstudio": const { LMStudioLLM } = require("../AiProviders/lmStudio"); - embedder = getEmbeddingEngineSelection(); return new LMStudioLLM(embedder); case "localai": const { LocalAiLLM } = require("../AiProviders/localAi"); - embedder = getEmbeddingEngineSelection(); return new LocalAiLLM(embedder); default: throw new Error("ENV: No LLM_PROVIDER value found in environment!");