From 155bc149afb0b27b16aeac8dccdd731649fc4668 Mon Sep 17 00:00:00 2001 From: shatfield4 Date: Tue, 5 Mar 2024 11:54:15 -0800 Subject: [PATCH 1/3] Groq LLM support complete --- .../LLMSelection/GroqAiOptions/index.jsx | 41 ++++ frontend/src/media/llmprovider/groq.png | Bin 0 -> 1450 bytes .../GeneralSettings/LLMPreference/index.jsx | 9 + .../Steps/DataHandling/index.jsx | 9 + .../Steps/LLMPreference/index.jsx | 9 + server/models/systemSettings.js | 13 ++ server/utils/AiProviders/groq/index.js | 211 ++++++++++++++++++ server/utils/helpers/index.js | 3 + server/utils/helpers/updateENV.js | 11 + 9 files changed, 306 insertions(+) create mode 100644 frontend/src/components/LLMSelection/GroqAiOptions/index.jsx create mode 100644 frontend/src/media/llmprovider/groq.png create mode 100644 server/utils/AiProviders/groq/index.js diff --git a/frontend/src/components/LLMSelection/GroqAiOptions/index.jsx b/frontend/src/components/LLMSelection/GroqAiOptions/index.jsx new file mode 100644 index 00000000000..12601c17d5a --- /dev/null +++ b/frontend/src/components/LLMSelection/GroqAiOptions/index.jsx @@ -0,0 +1,41 @@ +export default function GroqAiOptions({ settings }) { + return ( +
+
+ + +
+ +
+ + +
+
+ ); +} diff --git a/frontend/src/media/llmprovider/groq.png b/frontend/src/media/llmprovider/groq.png new file mode 100644 index 0000000000000000000000000000000000000000..31564145e1068131f3cf0a49766efbd68a0b4d54 GIT binary patch literal 1450 zcmb7^dotQT$Rm%*&tNegE6tFx zX~(OSS86M*Ff-*bWJY9AB4p4GJG+1FUwh7;bMO7$d%x%2KR;Pc4mL7Uho!{C#AI+F z)C%l{0cSnPN$C?LbJuzsrmhz*qR>*$4wYt2$y}dOzvyq>& zo)P_{qe&3q`RD6y;l$`#fWt;PNp#!cIN&Y%CK2u8;F{Ovd()hO4f|a*!*N&(xA^jf z^3Rles?r(!-?BNJ5YNSJ!lO)dG|ma>xACQ5QGoxX5DA%2OicJ#s)R4HLuc6_j|jx0 z;O7)!mAT@M)~Uxdo)zTtdcmdtSd+ESw}f(9$38TNdkzn8R2Jn?jgJb>(uZWORTk$= zBT@1`oHOpek&s)%o~ZGd`S&W66sobV?>Q~&3ac{WHPfVvd+D-6T6JiIY@JD{sYekt z1a8zYf4aCts~1A4$todQY7yjB#)0Ku3n(^28dnh_x7q0(SbbeDr+WH%Gf24QMe`Tf z0&qE+M2%%+{!Q(>_o1Ui)#3Puyj%mY-%l_`YDj8d+T?MNwg}}ksa~Bl3zi_Faq-1H zCsmj@+;q$&6NP8gy~gcIGS$7?B})_@n?%ZkBP|B~rO029cEiObRKB%BIg~jPJeRg* zhwo|U-d^e+9db65(}A7RbiR70%yu-hxG>EDN9+!CI?M#qfWG+4as2xaUZ1c6G0$V$ z_f$JL7Rvtt_7jn?;NA9-SbZyNOcYPPzDW2;PZ8#>8azpQoRe;J8nF-N?v-We8LF$d z5BANgPH}gse2M^`FK0t}(%)MzCw~zOY$23fldH%M^97>bh&7e6{Yi5@wh)rLWgl0) zwaj^_oO}BP_x447>E+Va1f)9L%)rsGAWL`fJHxqvNNU^68cX17s(q_iqDYb~s`aIk zy-6=X+sw$8^h-(oB zpyN{1OEZ?{p1C$f+=Jz$nJFp^tpq&-F*S`A(*c0_bS-0GReH@J(5|l1SIk zlb9^Pzwdj?3d_e%yyuAH-dS~rL3s8xW>#4G^h;FN%T~+COa3>s+1$gu=lMy}Z~In) zgfvMol0Eozac0k5=H+rvCe$VM$nN97@TSjkJjO6^Xxq&X&W%wpX4a{4@#2EiH)IGy z=?bw?>p-%^>lIPY2 zV!qI2!{>Svz3FRiQ!m@XE9yUU+OA$XWVV4b@3fRzumHCa+Y}Ccj!O>4$HFRXE zZBT5<`L3Jtb;7_M%;xnN?IxvM7wTO4e%X)XntK<_Sft!Kd*c>k?Fnkx1MJU?E*j@2 z!r6lkL4oz0ZG%_E1mdkp--dRAVrP;+MKg{R5h*Lseb|K=B&j4 literal 0 HcmV?d00001 diff --git a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx index ac4c1e3d1e4..37c15634d52 100644 --- a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx +++ b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx @@ -16,6 +16,7 @@ import MistralLogo from "@/media/llmprovider/mistral.jpeg"; import HuggingFaceLogo from "@/media/llmprovider/huggingface.png"; import PerplexityLogo from "@/media/llmprovider/perplexity.png"; import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg"; +import GroqLogo from "@/media/llmprovider/groq.png"; import PreLoader from "@/components/Preloader"; import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions"; import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions"; @@ -33,6 +34,7 @@ import LLMItem from "@/components/LLMSelection/LLMItem"; import { MagnifyingGlass } from "@phosphor-icons/react"; import PerplexityOptions from "@/components/LLMSelection/PerplexityOptions"; import OpenRouterOptions from "@/components/LLMSelection/OpenRouterOptions"; +import GroqAiOptions from "@/components/LLMSelection/GroqAiOptions"; export default function GeneralLLMPreference() { const [saving, setSaving] = useState(false); @@ -173,6 +175,13 @@ export default function GeneralLLMPreference() { options: , description: "A unified interface for LLMs.", }, + { + name: "Groq", + value: "groq", + logo: GroqLogo, + options: , + description: "Run high performance LLMs.", + }, { name: "Native", value: "native", diff --git a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx index 5beec3c1764..af3b3a9d03b 100644 --- a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx +++ b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx @@ -13,6 +13,7 @@ import MistralLogo from "@/media/llmprovider/mistral.jpeg"; import HuggingFaceLogo from "@/media/llmprovider/huggingface.png"; import PerplexityLogo from "@/media/llmprovider/perplexity.png"; import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg"; +import GroqLogo from "@/media/llmprovider/groq.png"; import ZillizLogo from "@/media/vectordbs/zilliz.png"; import AstraDBLogo from "@/media/vectordbs/astraDB.png"; import ChromaLogo from "@/media/vectordbs/chroma.png"; @@ -127,6 +128,14 @@ const LLM_SELECTION_PRIVACY = { ], logo: OpenRouterLogo, }, + groq: { + name: "Groq", + description: [ + "Your chats will not be used for training", + "Your prompts and document text used in response creation are visible to Groq", + ], + logo: GroqLogo, + }, }; const VECTOR_DB_PRIVACY = { diff --git a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx index 433914ae66d..16a3c959139 100644 --- a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx +++ b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx @@ -13,6 +13,7 @@ import MistralLogo from "@/media/llmprovider/mistral.jpeg"; import HuggingFaceLogo from "@/media/llmprovider/huggingface.png"; import PerplexityLogo from "@/media/llmprovider/perplexity.png"; import OpenRouterLogo from "@/media/llmprovider/openrouter.jpeg"; +import GroqLogo from "@/media/llmprovider/groq.png"; import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions"; import AzureAiOptions from "@/components/LLMSelection/AzureAiOptions"; import AnthropicAiOptions from "@/components/LLMSelection/AnthropicAiOptions"; @@ -31,6 +32,7 @@ import paths from "@/utils/paths"; import showToast from "@/utils/toast"; import { useNavigate } from "react-router-dom"; import OpenRouterOptions from "@/components/LLMSelection/OpenRouterOptions"; +import GroqAiOptions from "@/components/LLMSelection/GroqAiOptions"; const TITLE = "LLM Preference"; const DESCRIPTION = @@ -147,6 +149,13 @@ export default function LLMPreference({ options: , description: "A unified interface for LLMs.", }, + { + name: "Groq", + value: "groq", + logo: GroqLogo, + options: , + description: "Run high performance LLMs.", + }, { name: "Native", value: "native", diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js index 31d5c59a8d5..ba19fd112ef 100644 --- a/server/models/systemSettings.js +++ b/server/models/systemSettings.js @@ -212,6 +212,19 @@ const SystemSettings = { MistralApiKey: !!process.env.MISTRAL_API_KEY, MistralModelPref: process.env.MISTRAL_MODEL_PREF, + // For embedding credentials when mistral is selected. + OpenAiKey: !!process.env.OPEN_AI_KEY, + AzureOpenAiEndpoint: process.env.AZURE_OPENAI_ENDPOINT, + AzureOpenAiKey: !!process.env.AZURE_OPENAI_KEY, + AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF, + } + : {}), + + ...(llmProvider === "groq" + ? { + GroqApiKey: !!process.env.GROQ_API_KEY, + GroqModelPref: process.env.GROQ_MODEL_PREF, + // For embedding credentials when mistral is selected. OpenAiKey: !!process.env.OPEN_AI_KEY, AzureOpenAiEndpoint: process.env.AZURE_OPENAI_ENDPOINT, diff --git a/server/utils/AiProviders/groq/index.js b/server/utils/AiProviders/groq/index.js new file mode 100644 index 00000000000..a28a95c8088 --- /dev/null +++ b/server/utils/AiProviders/groq/index.js @@ -0,0 +1,211 @@ +const { OpenAiEmbedder } = require("../../EmbeddingEngines/openAi"); +const { chatPrompt } = require("../../chats"); +const { handleDefaultStreamResponse } = require("../../helpers/chat/responses"); + +class GroqLLM { + constructor(embedder = null, modelPreference = null) { + const { Configuration, OpenAIApi } = require("openai"); + if (!process.env.GROQ_API_KEY) throw new Error("No Groq API key was set."); + + const config = new Configuration({ + basePath: "https://api.groq.com/openai/v1", + apiKey: process.env.GROQ_API_KEY, + }); + + this.openai = new OpenAIApi(config); + this.model = + modelPreference || process.env.GROQ_MODEL_PREF || "llama2-70b-4096"; + this.limits = { + history: this.promptWindowLimit() * 0.15, + system: this.promptWindowLimit() * 0.15, + user: this.promptWindowLimit() * 0.7, + }; + + if (!embedder) + console.warn( + "No embedding provider defined for GroqLLM - falling back to OpenAiEmbedder for embedding!" + ); + this.embedder = !embedder ? new OpenAiEmbedder() : embedder; + this.defaultTemp = 0.7; + } + + #appendContext(contextTexts = []) { + if (!contextTexts || !contextTexts.length) return ""; + return ( + "\nContext:\n" + + contextTexts + .map((text, i) => { + return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`; + }) + .join("") + ); + } + + streamingEnabled() { + return "streamChat" in this && "streamGetChatCompletion" in this; + } + + promptWindowLimit() { + switch (this.model) { + case "llama2-70b-4096": + return 4096; + case "mixtral-8x7b-32768": + return 32_768; + default: + return 4096; + } + } + + async isValidChatCompletionModel(modelName = "") { + const validModels = ["llama2-70b-4096", "mixtral-8x7b-32768"]; + const isPreset = validModels.some((model) => modelName === model); + if (isPreset) return true; + + const model = await this.openai + .retrieveModel(modelName) + .then((res) => res.data) + .catch(() => null); + return !!model; + } + + constructPrompt({ + systemPrompt = "", + contextTexts = [], + chatHistory = [], + userPrompt = "", + }) { + const prompt = { + role: "system", + content: `${systemPrompt}${this.#appendContext(contextTexts)}`, + }; + return [prompt, ...chatHistory, { role: "user", content: userPrompt }]; + } + + async isSafe(_input = "") { + // Not implemented so must be stubbed + return { safe: true, reasons: [] }; + } + + async sendChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `Groq chat: ${this.model} is not valid for chat completion!` + ); + + const textResponse = await this.openai + .createChatCompletion({ + model: this.model, + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), + n: 1, + messages: await this.compressMessages( + { + systemPrompt: chatPrompt(workspace), + userPrompt: prompt, + chatHistory, + }, + rawHistory + ), + }) + .then((json) => { + const res = json.data; + if (!res.hasOwnProperty("choices")) + throw new Error("OpenAI chat: No results!"); + if (res.choices.length === 0) + throw new Error("OpenAI chat: No results length!"); + return res.choices[0].message.content; + }) + .catch((error) => { + throw new Error( + `Groq::createChatCompletion failed with: ${error.message}` + ); + }); + + return textResponse; + } + + async streamChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `Groq chat: ${this.model} is not valid for chat completion!` + ); + + const streamRequest = await this.openai.createChatCompletion( + { + model: this.model, + stream: true, + temperature: Number(workspace?.openAiTemp ?? this.defaultTemp), + n: 1, + messages: await this.compressMessages( + { + systemPrompt: chatPrompt(workspace), + userPrompt: prompt, + chatHistory, + }, + rawHistory + ), + }, + { responseType: "stream" } + ); + return streamRequest; + } + + async getChatCompletion(messages = null, { temperature = 0.7 }) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `Groq chat: ${this.model} is not valid for chat completion!` + ); + + const { data } = await this.openai + .createChatCompletion({ + model: this.model, + messages, + temperature, + }) + .catch((e) => { + throw new Error(e.response.data.error.message); + }); + + if (!data.hasOwnProperty("choices")) return null; + return data.choices[0].message.content; + } + + async streamGetChatCompletion(messages = null, { temperature = 0.7 }) { + if (!(await this.isValidChatCompletionModel(this.model))) + throw new Error( + `Groq chat: ${this.model} is not valid for chat completion!` + ); + + const streamRequest = await this.openai.createChatCompletion( + { + model: this.model, + stream: true, + messages, + temperature, + }, + { responseType: "stream" } + ); + return streamRequest; + } + + handleStream(response, stream, responseProps) { + return handleDefaultStreamResponse(response, stream, responseProps); + } + + // Simple wrapper for dynamic embedder & normalize interface for all LLM implementations + async embedTextInput(textInput) { + return await this.embedder.embedTextInput(textInput); + } + async embedChunks(textChunks = []) { + return await this.embedder.embedChunks(textChunks); + } + + async compressMessages(promptArgs = {}, rawHistory = []) { + const { messageArrayCompressor } = require("../../helpers/chat"); + const messageArray = this.constructPrompt(promptArgs); + return await messageArrayCompressor(this, messageArray, rawHistory); + } +} + +module.exports = { + GroqLLM, +}; diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js index a31a3e4f99b..7836097256c 100644 --- a/server/utils/helpers/index.js +++ b/server/utils/helpers/index.js @@ -73,6 +73,9 @@ function getLLMProvider(modelPreference = null) { case "huggingface": const { HuggingFaceLLM } = require("../AiProviders/huggingface"); return new HuggingFaceLLM(embedder, modelPreference); + case "groq": + const { GroqLLM } = require("../AiProviders/groq"); + return new GroqLLM(embedder, modelPreference); default: throw new Error("ENV: No LLM_PROVIDER value found in environment!"); } diff --git a/server/utils/helpers/updateENV.js b/server/utils/helpers/updateENV.js index 1ca9368204f..87a601e0f87 100644 --- a/server/utils/helpers/updateENV.js +++ b/server/utils/helpers/updateENV.js @@ -259,6 +259,16 @@ const KEY_MAPPING = { checks: [isNotEmpty], }, + // OpenRouter Options + GroqApiKey: { + envKey: "GROQ_API_KEY", + checks: [isNotEmpty], + }, + GroqModelPref: { + envKey: "GROQ_MODEL_PREF", + checks: [isNotEmpty], + }, + // System Settings AuthToken: { envKey: "AUTH_TOKEN", @@ -336,6 +346,7 @@ function supportedLLM(input = "") { "huggingface", "perplexity", "openrouter", + "groq", ].includes(input); return validSelection ? null : `${input} is not a valid LLM provider.`; } From 01bee6d1a0fa8c01067912d5ce59b53576a12e2b Mon Sep 17 00:00:00 2001 From: shatfield4 Date: Tue, 5 Mar 2024 12:15:27 -0800 Subject: [PATCH 2/3] update useGetProvidersModels for groq models --- frontend/src/components/LLMSelection/GroqAiOptions/index.jsx | 2 +- frontend/src/hooks/useGetProvidersModels.js | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/frontend/src/components/LLMSelection/GroqAiOptions/index.jsx b/frontend/src/components/LLMSelection/GroqAiOptions/index.jsx index 12601c17d5a..cc6fbbcc061 100644 --- a/frontend/src/components/LLMSelection/GroqAiOptions/index.jsx +++ b/frontend/src/components/LLMSelection/GroqAiOptions/index.jsx @@ -23,7 +23,7 @@ export default function GroqAiOptions({ settings }) {