Mintplex-Labs · timothycarambat · Dec 5, 2024 · Dec 5, 2024 · Dec 5, 2024 · Dec 5, 2024
diff --git a/README.md b/README.md
@@ -78,6 +78,7 @@ AnythingLLM divides your documents into objects called `workspaces`. A Workspace
 - [Azure OpenAI](https://azure.microsoft.com/en-us/products/ai-services/openai-service)
 - [AWS Bedrock](https://aws.amazon.com/bedrock/)
 - [Anthropic](https://www.anthropic.com/)
+- [NVIDIA NIM (chat models)](https://build.nvidia.com/explore/discover)
 - [Google Gemini Pro](https://ai.google.dev/)
 - [Hugging Face (chat models)](https://huggingface.co/)
 - [Ollama (chat models)](https://ollama.ai/)

diff --git a/docker/.env.example b/docker/.env.example
@@ -117,6 +117,10 @@ GID='1000'
 # XAI_LLM_API_KEY='xai-your-api-key-here'
 # XAI_LLM_MODEL_PREF='grok-beta'
 
+# LLM_PROVIDER='nvidia-nim'
+# NVIDIA_NIM_LLM_BASE_PATH='http://127.0.0.1:8000'
+# NVIDIA_NIM_LLM_MODEL_PREF='meta/llama-3.2-3b-instruct'
+
 ###########################################
 ######## Embedding API SElECTION ##########
 ###########################################

diff --git a/frontend/src/components/LLMSelection/NvidiaNimOptions/index.jsx b/frontend/src/components/LLMSelection/NvidiaNimOptions/index.jsx
@@ -0,0 +1,11 @@
+import RemoteNvidiaNimOptions from "./remote";
+import ManagedNvidiaNimOptions from "./managed";
+
+export default function NvidiaNimOptions({ settings }) {
+  const version = "remote"; // static to "remote" when in docker version.
+  return version === "remote" ? (
+    <RemoteNvidiaNimOptions settings={settings} />
+  ) : (
+    <ManagedNvidiaNimOptions settings={settings} />
+  );
+}
diff --git a/frontend/src/components/LLMSelection/NvidiaNimOptions/managed.jsx b/frontend/src/components/LLMSelection/NvidiaNimOptions/managed.jsx
@@ -0,0 +1,7 @@
+/**
+ * This component is used to select, start, and manage NVIDIA NIM
+ * containers and images via docker management tools.
+ */
+export default function ManagedNvidiaNimOptions({ settings }) {
+  return null;
+}
diff --git a/frontend/src/components/LLMSelection/NvidiaNimOptions/remote.jsx b/frontend/src/components/LLMSelection/NvidiaNimOptions/remote.jsx
@@ -0,0 +1,130 @@
+import PreLoader from "@/components/Preloader";
+import useProviderEndpointAutoDiscovery from "@/hooks/useProviderEndpointAutoDiscovery";
+import System from "@/models/system";
+import { NVIDIA_NIM_COMMON_URLS } from "@/utils/constants";
+import { useState, useEffect } from "react";
+
+/**
+ * This component is used to select a remote Nvidia NIM model endpoint
+ * This is the default component and way to connect to NVIDIA NIM
+ * as the "managed" provider can only work in the Desktop context.
+ */
+export default function RemoteNvidiaNimOptions({ settings }) {
+  const {
+    autoDetecting: loading,
+    basePath,
+    basePathValue,
+    handleAutoDetectClick,
+  } = useProviderEndpointAutoDiscovery({
+    provider: "nvidia-nim",
+    initialBasePath: settings?.NvidiaNimLLMBasePath,
+    ENDPOINTS: NVIDIA_NIM_COMMON_URLS,
+  });
+
+  return (
+    <div className="flex gap-[36px] mt-1.5">
+      <div className="flex flex-col w-60">
+        <div className="flex justify-between items-center mb-2">
+          <label className="text-white text-sm font-semibold">
+            Nvidia Nim Base URL
+          </label>
+          {loading ? (
+            <PreLoader size="6" />
+          ) : (
+            <>
+              {!basePathValue.value && (
+                <button
+                  onClick={handleAutoDetectClick}
+                  className="bg-primary-button text-xs font-medium px-2 py-1 rounded-lg hover:bg-secondary hover:text-white shadow-[0_4px_14px_rgba(0,0,0,0.25)]"
+                >
+                  Auto-Detect
+                </button>
+              )}
+            </>
+          )}
+        </div>
+        <input
+          type="url"
+          name="NvidiaNimLLMBasePath"
+          className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+          placeholder="http://localhost:8000/v1"
+          value={basePathValue.value}
+          required={true}
+          autoComplete="off"
+          spellCheck={false}
+          onChange={basePath.onChange}
+          onBlur={basePath.onBlur}
+        />
+        <p className="text-xs leading-[18px] font-base text-white text-opacity-60 mt-2">
+          Enter the URL where Nvidia NIM is running.
+        </p>
+      </div>
+      {!settings?.credentialsOnly && (
+        <NvidiaNimModelSelection
+          settings={settings}
+          basePath={basePath.value}
+        />
+      )}
+    </div>
+  );
+}
+function NvidiaNimModelSelection({ settings, basePath }) {
+  const [models, setModels] = useState([]);
+  const [loading, setLoading] = useState(true);
+
+  useEffect(() => {
+    async function findCustomModels() {
+      setLoading(true);
+      const { models } = await System.customModels(
+        "nvidia-nim",
+        null,
+        basePath
+      );
+      setModels(models);
+      setLoading(false);
+    }
+    findCustomModels();
+  }, [basePath]);
+
+  if (loading || models.length === 0) {
+    return (
+      <div className="flex flex-col w-60">
+        <label className="text-white text-sm font-semibold block mb-3">
+          Chat Model Selection
+        </label>
+        <select
+          name="NvidiaNimLLMModelPref"
+          disabled={true}
+          className="border-none bg-theme-settings-input-bg border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+        >
+          <option disabled={true} selected={true}>
+            -- loading available models --
+          </option>
+        </select>
+      </div>
+    );
+  }
+
+  return (
+    <div className="flex flex-col w-60">
+      <label className="text-white text-sm font-semibold block mb-3">
+        Chat Model Selection
+      </label>
+      <select
+        name="NvidiaNimLLMModelPref"
+        required={true}
+        className="border-none bg-theme-settings-input-bg border-gray-500 text-white text-sm rounded-lg block w-full p-2.5"
+      >
+        {models.map((model) => (
+          <option
+            key={model.id}
+            value={model.id}
+            selected={settings?.NvidiaNimLLMModelPref === model.id}
+          >
+            {model.name}
+          </option>
+        ))}
+      </select>
+    </div>
+  );
+}
diff --git a/frontend/src/hooks/useGetProvidersModels.js b/frontend/src/hooks/useGetProvidersModels.js
@@ -43,6 +43,7 @@ const PROVIDER_DEFAULT_MODELS = {
   ollama: [],
   togetherai: [],
   fireworksai: [],
+  "nvidia-nim": [],
   groq: [],
   native: [],
   cohere: [

diff --git a/frontend/src/media/llmprovider/nvidia-nim.png b/frontend/src/media/llmprovider/nvidia-nim.png
diff --git a/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx b/frontend/src/pages/GeneralSettings/LLMPreference/index.jsx
@@ -29,6 +29,7 @@ import AWSBedrockLogo from "@/media/llmprovider/bedrock.png";
 import DeepSeekLogo from "@/media/llmprovider/deepseek.png";
 import APIPieLogo from "@/media/llmprovider/apipie.png";
 import XAILogo from "@/media/llmprovider/xai.png";
+import NvidiaNimLogo from "@/media/llmprovider/nvidia-nim.png";
 
 import PreLoader from "@/components/Preloader";
 import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
@@ -56,6 +57,7 @@ import AWSBedrockLLMOptions from "@/components/LLMSelection/AwsBedrockLLMOptions
 import DeepSeekOptions from "@/components/LLMSelection/DeepSeekOptions";
 import ApiPieLLMOptions from "@/components/LLMSelection/ApiPieOptions";
 import XAILLMOptions from "@/components/LLMSelection/XAiLLMOptions";
+import NvidiaNimOptions from "@/components/LLMSelection/NvidiaNimOptions";
 
 import LLMItem from "@/components/LLMSelection/LLMItem";
 import { CaretUpDown, MagnifyingGlass, X } from "@phosphor-icons/react";
@@ -94,6 +96,15 @@ export const AVAILABLE_LLM_PROVIDERS = [
     description: "Google's largest and most capable AI model",
     requiredConfig: ["GeminiLLMApiKey"],
   },
+  {
+    name: "Nvidia NIM",
+    value: "nvidia-nim",
+    logo: NvidiaNimLogo,
+    options: (settings) => <NvidiaNimOptions settings={settings} />,
+    description:
+      "Run full parameter LLMs directly on your GPU using Nvidia's inference microservice via Docker.",
+    requiredConfig: ["NvidiaNimLLMBasePath"],
+  },
   {
     name: "HuggingFace",
     value: "huggingface",

diff --git a/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/DataHandling/index.jsx
@@ -9,6 +9,7 @@ import GeminiLogo from "@/media/llmprovider/gemini.png";
 import OllamaLogo from "@/media/llmprovider/ollama.png";
 import TogetherAILogo from "@/media/llmprovider/togetherai.png";
 import FireworksAILogo from "@/media/llmprovider/fireworksai.jpeg";
+import NvidiaNimLogo from "@/media/llmprovider/nvidia-nim.png";
 import LMStudioLogo from "@/media/llmprovider/lmstudio.png";
 import LocalAiLogo from "@/media/llmprovider/localai.png";
 import MistralLogo from "@/media/llmprovider/mistral.jpeg";
@@ -76,6 +77,13 @@ export const LLM_SELECTION_PRIVACY = {
     ],
     logo: GeminiLogo,
   },
+  "nvidia-nim": {
+    name: "Nvidia NIM",
+    description: [
+      "Your model and chats are only accessible on the machine running the Nvidia NIM service",
+    ],
+    logo: NvidiaNimLogo,
+  },
   lmstudio: {
     name: "LMStudio",
     description: [

diff --git a/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx b/frontend/src/pages/OnboardingFlow/Steps/LLMPreference/index.jsx
@@ -24,7 +24,7 @@ import DeepSeekLogo from "@/media/llmprovider/deepseek.png";
 import APIPieLogo from "@/media/llmprovider/apipie.png";
 import NovitaLogo from "@/media/llmprovider/novita.png";
 import XAILogo from "@/media/llmprovider/xai.png";
-
+import NvidiaNimLogo from "@/media/llmprovider/nvidia-nim.png";
 import CohereLogo from "@/media/llmprovider/cohere.png";
 import OpenAiOptions from "@/components/LLMSelection/OpenAiOptions";
 import GenericOpenAiOptions from "@/components/LLMSelection/GenericOpenAiOptions";
@@ -51,6 +51,7 @@ import DeepSeekOptions from "@/components/LLMSelection/DeepSeekOptions";
 import ApiPieLLMOptions from "@/components/LLMSelection/ApiPieOptions";
 import NovitaLLMOptions from "@/components/LLMSelection/NovitaLLMOptions";
 import XAILLMOptions from "@/components/LLMSelection/XAiLLMOptions";
+import NvidiaNimOptions from "@/components/LLMSelection/NvidiaNimOptions";
 
 import LLMItem from "@/components/LLMSelection/LLMItem";
 import System from "@/models/system";
@@ -91,6 +92,14 @@ const LLMS = [
     options: (settings) => <GeminiLLMOptions settings={settings} />,
     description: "Google's largest and most capable AI model",
   },
+  {
+    name: "Nvidia NIM",
+    value: "nvidia-nim",
+    logo: NvidiaNimLogo,
+    options: (settings) => <NvidiaNimOptions settings={settings} />,
+    description:
+      "Run full parameter LLMs directly on your GPU using Nvidia's inference microservice via Docker.",
+  },
   {
     name: "HuggingFace",
     value: "huggingface",

diff --git a/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx b/frontend/src/pages/WorkspaceSettings/AgentConfig/AgentLLMSelection/index.jsx
@@ -28,6 +28,7 @@ const ENABLED_PROVIDERS = [
   "litellm",
   "apipie",
   "xai",
+  "nvidia-nim",
   // TODO: More agent support.
   // "cohere",         // Has tool calling and will need to build explicit support
   // "huggingface"     // Can be done but already has issues with no-chat templated. Needs to be tested.

diff --git a/frontend/src/utils/constants.js b/frontend/src/utils/constants.js
@@ -37,6 +37,13 @@ export const LOCALAI_COMMON_URLS = [
   "http://172.17.0.1:8080/v1",
 ];
 
+export const NVIDIA_NIM_COMMON_URLS = [
+  "http://127.0.0.1:8000/v1/version",
+  "http://localhost:8000/v1/version",
+  "http://host.docker.internal:8000/v1/version",
+  "http://172.17.0.1:8000/v1/version",
+];
+
 export function fullApiUrl() {
   if (API_BASE !== "/api") return API_BASE;
   return `${window.location.origin}/api`;

diff --git a/server/.env.example b/server/.env.example
@@ -107,6 +107,10 @@ SIG_SALT='salt' # Please generate random string at least 32 chars long.
 # XAI_LLM_API_KEY='xai-your-api-key-here'
 # XAI_LLM_MODEL_PREF='grok-beta'
 
+# LLM_PROVIDER='nvidia-nim'
+# NVIDIA_NIM_LLM_BASE_PATH='http://127.0.0.1:8000'
+# NVIDIA_NIM_LLM_MODEL_PREF='meta/llama-3.2-3b-instruct'
+
 ###########################################
 ######## Embedding API SElECTION ##########
 ###########################################

diff --git a/server/models/systemSettings.js b/server/models/systemSettings.js
@@ -547,6 +547,11 @@ const SystemSettings = {
       // xAI LLM API Keys
       XAIApiKey: !!process.env.XAI_LLM_API_KEY,
       XAIModelPref: process.env.XAI_LLM_MODEL_PREF,
+
+      // Nvidia NIM Keys
+      NvidiaNimLLMBasePath: process.env.NVIDIA_NIM_LLM_BASE_PATH,
+      NvidiaNimLLMModelPref: process.env.NVIDIA_NIM_LLM_MODEL_PREF,
+      NvidiaNimLLMTokenLimit: process.env.NVIDIA_NIM_LLM_MODEL_TOKEN_LIMIT,
     };
   },