Mintplex-Labs · timothycarambat · Nov 9, 2023 · Nov 6, 2023 · Nov 6, 2023 · Nov 6, 2023
diff --git a/README.md b/README.md
@@ -52,9 +52,10 @@ Some cool features of AnythingLLM
 
 ### Supported LLMs and Vector Databases
 **Supported LLMs:**
-- OpenAI
-- Azure OpenAI
-- Anthropic ClaudeV2
+- [OpenAI](https://openai.com)
+- [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service)
+- [Anthropic ClaudeV2](https://www.anthropic.com/)
+- [LM Studio (all models)](https://lmstudio.ai)
 
 **Supported Vector Databases:**
 - [LanceDB](https://github.com/lancedb/lancedb) (default)
@@ -73,7 +74,7 @@ This monorepo consists of three main sections:
 ### Requirements
 - `yarn` and `node` on your machine
 - `python` 3.9+ for running scripts in `collector/`.
-- access to an LLM like `GPT-3.5`, `GPT-4`, etc.
+- access to an LLM service like `GPT-3.5`, `GPT-4`, `Mistral`, `LLama`, etc.
 - (optional) a vector database like Pinecone, qDrant, Weaviate, or Chroma*.
 *AnythingLLM by default uses a built-in vector db called LanceDB.
 

diff --git a/docker/.env.example b/docker/.env.example
@@ -19,6 +19,10 @@ CACHE_VECTORS="true"
 # ANTHROPIC_API_KEY=sk-ant-xxxx
 # ANTHROPIC_MODEL_PREF='claude-2'
 
+# LLM_PROVIDER='lmstudio'
+# LMSTUDIO_BASE_PATH='http://your-server:1234/v1'
+# LMSTUDIO_MODEL_TOKEN_LIMIT=4096
+
 ###########################################
 ######## Embedding API SElECTION ##########
 ###########################################

diff --git a/frontend/src/components/LLMSelection/LMStudioOptions/index.jsx b/frontend/src/components/LLMSelection/LMStudioOptions/index.jsx
@@ -0,0 +1,59 @@
+import { Info } from "@phosphor-icons/react";
+import paths from "../../../utils/paths";
+
+export default function LMStudioOptions({ settings, showAlert = false }) {
+  return (
+    <div className="w-full flex flex-col">
+      {showAlert && (
+        <div className="flex flex-col md:flex-row md:items-center gap-x-2 text-white mb-6 bg-blue-800/30 w-fit rounded-lg px-4 py-2">
+          <div className="gap-x-2 flex items-center">
+            <Info size={12} className="hidden md:visible" />
+            <p className="text-sm md:text-base">
+              LMStudio as your LLM requires you to set an embedding service to
+              use.
+            </p>
+          </div>
+          <a
+            href={paths.general.embeddingPreference()}
+            className="text-sm md:text-base my-2 underline"
+          >
+            Manage embedding &rarr;
+          </a>
+        </div>
+      )}
+      <div className="w-full flex items-center gap-4">
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-4">
+            LMStudio Base URL
+          </label>
+          <input
+            type="url"
+            name="LMStudioBasePath"
+            className="bg-zinc-900 text-white placeholder-white placeholder-opacity-60 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            placeholder="http://localhost:1234/v1"
+            defaultValue={settings?.LMStudioBasePath}
+            required={true}
+            autoComplete="off"
+            spellCheck={false}
+          />
+        </div>
+        <div className="flex flex-col w-60">
+          <label className="text-white text-sm font-semibold block mb-4">
+            Token context window
+          </label>
+          <input
+            type="number"
+            name="LMStudioTokenLimit"
+            className="bg-zinc-900 text-white placeholder-white placeholder-opacity-60 text-sm rounded-lg focus:border-white block w-full p-2.5"
+            placeholder="4096"
+            min={1}
+            onScroll={(e) => e.target.blur()}
+            defaultValue={settings?.LMStudioTokenLimit}
+            required={true}
+            autoComplete="off"
+          />
+        </div>
+      </div>
+    </div>
+  );
+}
diff --git a/frontend/src/media/llmprovider/lmstudio.png b/frontend/src/media/llmprovider/lmstudio.png
diff --git a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx
@@ -8,11 +8,13 @@ import showToast from "../../../utils/toast";
 import OpenAiLogo from "../../../media/llmprovider/openai.png";
 import AzureOpenAiLogo from "../../../media/llmprovider/azure.png";
 import AnthropicLogo from "../../../media/llmprovider/anthropic.png";
+import LMStudioLogo from "../../../media/llmprovider/LMStudio.png";
 import PreLoader from "../../../components/Preloader";
 import LLMProviderOption from "../../../components/LLMSelection/LLMProviderOption";
 import OpenAiOptions from "../../../components/LLMSelection/OpenAiOptions";
 import AzureAiOptions from "../../../components/LLMSelection/AzureAiOptions";
 import AnthropicAiOptions from "../../../components/LLMSelection/AnthropicAiOptions";
+import LMStudioOptions from "../../../components/LLMSelection/LMStudioOptions";
 
 export default function GeneralLLMPreference() {
   const [saving, setSaving] = useState(false);
@@ -130,6 +132,15 @@ export default function GeneralLLMPreference() {
                   image={AnthropicLogo}
                   onClick={updateLLMChoice}
                 />
+                <LLMProviderOption
+                  name="LM Studio"
+                  value="lmstudio"
+                  link="lmstudio.ai"
+                  description="Discover, download, and run thousands of cutting edge LLMs in a few clicks."
+                  checked={llmChoice === "lmstudio"}
+                  image={LMStudioLogo}
+                  onClick={updateLLMChoice}
+                />
               </div>
               <div className="mt-10 flex flex-wrap gap-4 max-w-[800px]">
                 {llmChoice === "openai" && (
@@ -141,6 +152,9 @@ export default function GeneralLLMPreference() {
                 {llmChoice === "anthropic" && (
                   <AnthropicAiOptions settings={settings} showAlert={true} />
                 )}
+                {llmChoice === "lmstudio" && (
+                  <LMStudioOptions settings={settings} showAlert={true} />
+                )}
               </div>
             </div>
           </form>

diff --git a/frontend/src/pages/OnboardingFlow/OnboardingModal/Steps/LLMSelection/index.jsx b/frontend/src/pages/OnboardingFlow/OnboardingModal/Steps/LLMSelection/index.jsx
@@ -2,12 +2,14 @@ import React, { memo, useEffect, useState } from "react";
 import OpenAiLogo from "../../../../../media/llmprovider/openai.png";
 import AzureOpenAiLogo from "../../../../../media/llmprovider/azure.png";
 import AnthropicLogo from "../../../../../media/llmprovider/anthropic.png";
+import LMStudioLogo from "../../../../../media/llmprovider/lmstudio.png";
 import System from "../../../../../models/system";
 import PreLoader from "../../../../../components/Preloader";
 import LLMProviderOption from "../../../../../components/LLMSelection/LLMProviderOption";
 import OpenAiOptions from "../../../../../components/LLMSelection/OpenAiOptions";
 import AzureAiOptions from "../../../../../components/LLMSelection/AzureAiOptions";
 import AnthropicAiOptions from "../../../../../components/LLMSelection/AnthropicAiOptions";
+import LMStudioOptions from "../../../../../components/LLMSelection/LMStudioOptions";
 
 function LLMSelection({ nextStep, prevStep, currentStep }) {
   const [llmChoice, setLLMChoice] = useState("openai");
@@ -46,6 +48,8 @@ function LLMSelection({ nextStep, prevStep, currentStep }) {
     switch (data.LLMProvider) {
       case "anthropic":
         return nextStep("embedding_preferences");
+      case "lmstudio":
+        return nextStep("embedding_preferences");
       default:
         return nextStep("vector_database");
     }
@@ -94,13 +98,25 @@ function LLMSelection({ nextStep, prevStep, currentStep }) {
               image={AnthropicLogo}
               onClick={updateLLMChoice}
             />
+            <LLMProviderOption
+              name="LM Studio"
+              value="lmstudio"
+              link="lmstudio.ai"
+              description="Discover, download, and run thousands of cutting edge LLMs in a few clicks."
+              checked={llmChoice === "lmstudio"}
+              image={LMStudioLogo}
+              onClick={updateLLMChoice}
+            />
           </div>
           <div className="mt-10 flex flex-wrap gap-4 max-w-[800px]">
             {llmChoice === "openai" && <OpenAiOptions settings={settings} />}
             {llmChoice === "azure" && <AzureAiOptions settings={settings} />}
             {llmChoice === "anthropic" && (
               <AnthropicAiOptions settings={settings} />
             )}
+            {llmChoice === "lmstudio" && (
+              <LMStudioOptions settings={settings} />
+            )}
           </div>
         </div>
         <div className="flex w-full justify-between items-center p-6 space-x-2 border-t rounded-b border-gray-500/50">

diff --git a/server/.env.example b/server/.env.example
@@ -19,6 +19,10 @@ JWT_SECRET="my-random-string-for-seeding" # Please generate random string at lea
 # ANTHROPIC_API_KEY=sk-ant-xxxx
 # ANTHROPIC_MODEL_PREF='claude-2'
 
+# LLM_PROVIDER='lmstudio'
+# LMSTUDIO_BASE_PATH='http://your-server:1234/v1'
+# LMSTUDIO_MODEL_TOKEN_LIMIT=4096
+
 ###########################################
 ######## Embedding API SElECTION ##########
 ###########################################
@@ -58,4 +62,4 @@ VECTOR_DB="lancedb"
 # CLOUD DEPLOYMENT VARIRABLES ONLY
 # AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting.
 # STORAGE_DIR= # absolute filesystem path with no trailing slash
-# NO_DEBUG="true"
+# NO_DEBUG="true"
diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js
@@ -81,6 +81,19 @@ const SystemSettings = {
             AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF,
           }
         : {}),
+
+      ...(llmProvider === "lmstudio"
+        ? {
+            LMStudioBasePath: process.env.LMSTUDIO_BASE_PATH,
+            LMStudioTokenLimit: process.env.LMSTUDIO_MODEL_TOKEN_LIMIT,
+
+            // For embedding credentials when lmstudio is selected.
+            OpenAiKey: !!process.env.OPEN_AI_KEY,
+            AzureOpenAiEndpoint: process.env.AZURE_OPENAI_ENDPOINT,
+            AzureOpenAiKey: !!process.env.AZURE_OPENAI_KEY,
+            AzureOpenAiEmbeddingModelPref: process.env.EMBEDDING_MODEL_PREF,
+          }
+        : {}),
     };
   },
 

diff --git a/server/utils/AiProviders/lmStudio/index.js b/server/utils/AiProviders/lmStudio/index.js
@@ -0,0 +1,139 @@
+const { chatPrompt } = require("../../chats");
+
+//  hybrid of openAi LLM chat completion for LMStudio
+class LMStudioLLM {
+  constructor(embedder = null) {
+    if (!process.env.LMSTUDIO_BASE_PATH)
+      throw new Error("No LMStudio API Base Path was set.");
+
+    const { Configuration, OpenAIApi } = require("openai");
+    const config = new Configuration({
+      basePath: process.env.LMSTUDIO_BASE_PATH?.replace(/\/+$/, ""), // here is the URL to your LMStudio instance
+    });
+    this.lmstudio = new OpenAIApi(config);
+    // When using LMStudios inference server - the model param is not required so
+    // we can stub it here.
+    this.model = "model-placeholder";
+    this.limits = {
+      history: this.promptWindowLimit() * 0.15,
+      system: this.promptWindowLimit() * 0.15,
+      user: this.promptWindowLimit() * 0.7,
+    };
+
+    if (!embedder)
+      throw new Error(
+        "INVALID LM STUDIO SETUP. No embedding engine has been set. Go to instance settings and set up an embedding interface to use LMStudio as your LLM."
+      );
+    this.embedder = embedder;
+  }
+
+  // Ensure the user set a value for the token limit
+  // and if undefined - assume 4096 window.
+  promptWindowLimit() {
+    const limit = process.env.LMSTUDIO_MODEL_TOKEN_LIMIT || 4096;
+    if (!limit || isNaN(Number(limit)))
+      throw new Error("No LMStudio token context limit was set.");
+    return Number(limit);
+  }
+
+  async isValidChatCompletionModel(_ = "") {
+    // LMStudio may be anything. The user must do it correctly.
+    // See comment about this.model declaration in constructor
+    return true;
+  }
+
+  constructPrompt({
+    systemPrompt = "",
+    contextTexts = [],
+    chatHistory = [],
+    userPrompt = "",
+  }) {
+    const prompt = {
+      role: "system",
+      content: `${systemPrompt}
+Context:
+    ${contextTexts
+      .map((text, i) => {
+        return `[CONTEXT ${i}]:\n${text}\n[END CONTEXT ${i}]\n\n`;
+      })
+      .join("")}`,
+    };
+    return [prompt, ...chatHistory, { role: "user", content: userPrompt }];
+  }
+
+  async isSafe(_input = "") {
+    // Not implemented so must be stubbed
+    return { safe: true, reasons: [] };
+  }
+
+  async sendChat(chatHistory = [], prompt, workspace = {}, rawHistory = []) {
+    if (!this.model)
+      throw new Error(
+        `LMStudio chat: ${model} is not valid or defined for chat completion!`
+      );
+
+    const textResponse = await this.lmstudio
+      .createChatCompletion({
+        model: this.model,
+        temperature: Number(workspace?.openAiTemp ?? 0.7),
+        n: 1,
+        messages: await this.compressMessages(
+          {
+            systemPrompt: chatPrompt(workspace),
+            userPrompt: prompt,
+            chatHistory,
+          },
+          rawHistory
+        ),
+      })
+      .then((json) => {
+        const res = json.data;
+        if (!res.hasOwnProperty("choices"))
+          throw new Error("LMStudio chat: No results!");
+        if (res.choices.length === 0)
+          throw new Error("LMStudio chat: No results length!");
+        return res.choices[0].message.content;
+      })
+      .catch((error) => {
+        throw new Error(
+          `LMStudio::createChatCompletion failed with: ${error.message}`
+        );
+      });
+
+    return textResponse;
+  }
+
+  async getChatCompletion(messages = null, { temperature = 0.7 }) {
+    if (!this.model)
+      throw new Error(
+        `LMStudio chat: ${this.model} is not valid or defined model for chat completion!`
+      );
+
+    const { data } = await this.lmstudio.createChatCompletion({
+      model: this.model,
+      messages,
+      temperature,
+    });
+
+    if (!data.hasOwnProperty("choices")) return null;
+    return data.choices[0].message.content;
+  }
+
+  // Simple wrapper for dynamic embedder & normalize interface for all LLM implementations
+  async embedTextInput(textInput) {
+    return await this.embedder.embedTextInput(textInput);
+  }
+  async embedChunks(textChunks = []) {
+    return await this.embedder.embedChunks(textChunks);
+  }
+
+  async compressMessages(promptArgs = {}, rawHistory = []) {
+    const { messageArrayCompressor } = require("../../helpers/chat");
+    const messageArray = this.constructPrompt(promptArgs);
+    return await messageArrayCompressor(this, messageArray, rawHistory);
+  }
+}
+
+module.exports = {
+  LMStudioLLM,
+};
diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js
@@ -23,6 +23,7 @@ function getVectorDbClass() {
 
 function getLLMProvider() {
   const vectorSelection = process.env.LLM_PROVIDER || "openai";
+  let embedder = null;
   switch (vectorSelection) {
     case "openai":
       const { OpenAiLLM } = require("../AiProviders/openAi");
@@ -32,8 +33,12 @@ function getLLMProvider() {
       return new AzureOpenAiLLM();
     case "anthropic":
       const { AnthropicLLM } = require("../AiProviders/anthropic");
-      const embedder = getEmbeddingEngineSelection();
+      embedder = getEmbeddingEngineSelection();
       return new AnthropicLLM(embedder);
+    case "lmstudio":
+      const { LMStudioLLM } = require("../AiProviders/lmStudio");
+      embedder = getEmbeddingEngineSelection();
+      return new LMStudioLLM(embedder);
     default:
       throw new Error("ENV: No LLM_PROVIDER value found in environment!");
   }