Mintplex-Labs · timothycarambat · Jun 9, 2023 · Jun 8, 2023 · Jun 8, 2023 · Jun 9, 2023
diff --git a/README.md b/README.md
@@ -2,7 +2,8 @@
 
 [![Twitter](https://img.shields.io/twitter/url/https/twitter.com/tim.svg?style=social&label=Follow%20%40Timothy%20Carambat)](https://twitter.com/tcarambat) [![](https://dcbadge.vercel.app/api/server/6UyHPeGZAC?compact=true&style=flat)](https://discord.gg/6UyHPeGZAC)
 
-A full-stack application and tool suite that enables you to turn any document, resource, or piece of content into a piece of data that any LLM can use as reference during chatting. This application runs with very minimal overhead as by default the LLM and vectorDB are hosted remotely, but can be swapped for local instances. Currently this project supports Pinecone & ChromaDB for vector storage and OpenAI for chatting.
+A full-stack application and tool suite that enables you to turn any document, resource, or piece of content into a piece of data that any LLM can use as reference during chatting. This application runs with very minimal overhead as by default the LLM and vectorDB are hosted remotely, but can be swapped for local instances. Currently this project supports [Pinecone](https://pinecone.io), [ChromaDB](https://trychroma.com) & more for vector storage and [OpenAI](https://openai.com) for LLM/chatting.
+
 
 ![Chatting](/images/screenshots/chat.png)
 [view more screenshots](/images/screenshots/SCREENSHOTS.md)
@@ -38,8 +39,8 @@ This monorepo consists of three main sections:
 - `yarn` and `node` on your machine
 - `python` 3.8+ for running scripts in `collector/`.
 - access to an LLM like `GPT-3.5`, `GPT-4`*.
-- a [Pinecone.io](https://pinecone.io) free account* **or** Local Chroma instance running.
-*you can use drop in replacements for these. This is just the easiest to get up and running fast.
+- a [Pinecone.io](https://pinecone.io) free account*.
+*you can use drop in replacements for these. This is just the easiest to get up and running fast. We support multiple vector database providers.
 
 ### How to get started
 - `yarn setup` from the project root directory.

diff --git a/frontend/src/components/Modals/ManageWorkspace.jsx b/frontend/src/components/Modals/ManageWorkspace.jsx
@@ -14,6 +14,7 @@ import { nFormatter } from "../../utils/numbers";
 import { dollarFormat } from "../../utils/numbers";
 import paths from "../../utils/paths";
 import { useParams } from "react-router-dom";
+import { titleCase } from "text-case";
 
 const noop = () => false;
 export default function ManageWorkspace({ hideModal = noop, workspace }) {
@@ -24,15 +25,19 @@ export default function ManageWorkspace({ hideModal = noop, workspace }) {
   const [directories, setDirectories] = useState(null);
   const [originalDocuments, setOriginalDocuments] = useState([]);
   const [selectedFiles, setSelectFiles] = useState([]);
+  const [vectordb, setVectorDB] = useState(null);
+  const [showingNoRemovalModal, setShowingNoRemovalModal] = useState(false);
 
   useEffect(() => {
     async function fetchKeys() {
       const _workspace = await Workspace.bySlug(workspace.slug);
       const localFiles = await System.localFiles();
+      const settings = await System.keys();
       const originalDocs = _workspace.documents.map((doc) => doc.docpath) || [];
       setDirectories(localFiles);
       setOriginalDocuments([...originalDocs]);
       setSelectFiles([...originalDocs]);
+      setVectorDB(settings?.VectorDB);
       setLoading(false);
     }
     fetchKeys();
@@ -97,11 +102,25 @@ export default function ManageWorkspace({ hideModal = noop, workspace }) {
       : selectedFiles.some((doc) => doc.includes(filepath));
   };
 
+  const isOriginalDoc = (filepath) => {
+    const isFolder = !filepath.includes("/");
+    return isFolder
+      ? originalDocuments.some((doc) => doc.includes(filepath.split("/")[0]))
+      : originalDocuments.some((doc) => doc.includes(filepath));
+  };
+
   const toggleSelection = (filepath) => {
     const isFolder = !filepath.includes("/");
     const parent = isFolder ? filepath : filepath.split("/")[0];
 
     if (isSelected(filepath)) {
+      // Certain vector DBs do not contain the ability to delete vectors
+      // so we cannot remove from these. The user will have to clear the entire workspace.
+      if (["lancedb"].includes(vectordb) && isOriginalDoc(filepath)) {
+        setShowingNoRemovalModal(true);
+        return false;
+      }
+
       const updatedDocs = isFolder
         ? selectedFiles.filter((doc) => !doc.includes(parent))
         : selectedFiles.filter((doc) => !doc.includes(filepath));
@@ -168,6 +187,12 @@ export default function ManageWorkspace({ hideModal = noop, workspace }) {
           updateWorkspace={updateWorkspace}
         />
       )}
+      {showingNoRemovalModal && (
+        <CannotRemoveModal
+          hideModal={() => setShowingNoRemovalModal(false)}
+          vectordb={vectordb}
+        />
+      )}
       <div className="fixed top-0 left-0 right-0 z-50 w-full p-4 overflow-x-hidden overflow-y-auto md:inset-0 h-[calc(100%-1rem)] h-full bg-black bg-opacity-50 flex items-center justify-center">
         <div
           className="flex fixed top-0 left-0 right-0 w-full h-full"
@@ -463,6 +488,42 @@ function ConfirmationModal({
   );
 }
 
+function CannotRemoveModal({ hideModal, vectordb }) {
+  return (
+    <dialog
+      open={true}
+      style={{ zIndex: 100 }}
+      className="fixed top-0 flex bg-black bg-opacity-50 w-[100vw] h-full items-center justify-center "
+    >
+      <div className="px-10 p-4 w-1/2 rounded-lg bg-white shadow dark:bg-stone-700 text-black dark:text-slate-200">
+        <div className="flex flex-col w-full">
+          <p className="text-lg font-semibold text-red-500">
+            You cannot remove this document!
+          </p>
+
+          <div className="flex flex-col gap-y-1">
+            <p className="text-base mt-4">
+              {titleCase(vectordb)} does not support atomic removal of
+              documents.
+              <br />
+              Unfortunately, you will have to delete the entire workspace to
+              remove this document from being referenced.
+            </p>
+          </div>
+          <div className="flex w-full justify-center items-center mt-4">
+            <button
+              onClick={hideModal}
+              className="text-gray-800 hover:bg-gray-100 px-4 py-1 rounded-lg dark:text-slate-200 dark:hover:bg-stone-900"
+            >
+              I Understand
+            </button>
+          </div>
+        </div>
+      </div>
+    </dialog>
+  );
+}
+
 export function useManageWorkspaceModal() {
   const [showing, setShowing] = useState(false);
   const showModal = () => {

diff --git a/server/.env.example b/server/.env.example
@@ -13,6 +13,9 @@ PINECONE_ENVIRONMENT=
 PINECONE_API_KEY=
 PINECONE_INDEX=
 
+# Enable all below if you are using vector database: LanceDB.
+# VECTOR_DB="lancedb"
+
 # CLOUD DEPLOYMENT VARIRABLES ONLY
 # AUTH_TOKEN="hunter2" # This is the password to your application if remote hosting.
 # STORAGE_DIR= # absolute filesystem path with no trailing slash
diff --git a/server/.gitignore b/server/.gitignore
@@ -4,4 +4,5 @@ documents/*
 vector-cache/*.json
 !documents/DOCUMENTS.md
 logs/server.log
-*.db
+*.db
+lancedb
diff --git a/server/package.json b/server/package.json
@@ -29,7 +29,8 @@
     "slugify": "^1.6.6",
     "sqlite": "^4.2.1",
     "sqlite3": "^5.1.6",
-    "uuid": "^9.0.0"
+    "uuid": "^9.0.0",
+    "vectordb": "0.1.5-beta"
   },
   "devDependencies": {
     "nodemon": "^2.0.22",

diff --git a/server/utils/chroma/index.js b/server/utils/chroma/index.js
@@ -65,14 +65,6 @@ const Chroma = {
       modelName: model,
     });
   },
-  chatLLM: function () {
-    const model = process.env.OPEN_MODEL_PREF || "gpt-3.5-turbo";
-    return new ChatOpenAI({
-      openAIApiKey: process.env.OPEN_AI_KEY,
-      temperature: 0.7,
-      modelName: model,
-    });
-  },
   embedChunk: async function (openai, textChunk) {
     const {
       data: { data },
@@ -274,16 +266,6 @@ const Chroma = {
       };
     }
 
-    // const collection = await client.getCollection({ name: namespace, embeddingFunction: this.embeddingFunc() })
-    // const results = await collection.get({
-    //   where: {
-    //     description: 'a custom file uploaded by the user.'
-    //   },
-    //   includes: ['ids']
-    // })
-    // console.log(results)
-    // return { response: null, sources: [], }
-
     const vectorStore = await ChromaStore.fromExistingCollection(
       this.embedder(),
       { collectionName: namespace, url: process.env.CHROMA_ENDPOINT }

diff --git a/server/utils/helpers/index.js b/server/utils/helpers/index.js
@@ -1,13 +1,16 @@
 function getVectorDbClass() {
   const { Pinecone } = require("../pinecone");
   const { Chroma } = require("../chroma");
+  const { LanceDb } = require("../lancedb");
 
   const vectorSelection = process.env.VECTOR_DB || "pinecone";
   switch (vectorSelection) {
     case "pinecone":
       return Pinecone;
     case "chroma":
       return Chroma;
+    case "lancedb":
+      return LanceDb;
     default:
       throw new Error("ENV: No VECTOR_DB value found in environment!");
   }