Mintplex-Labs · timothycarambat · Jun 27, 2024 · Jun 27, 2024 · Jun 27, 2024
diff --git a/server/package.json b/server/package.json
@@ -26,6 +26,7 @@
     "@google/generative-ai": "^0.7.1",
     "@googleapis/youtube": "^9.0.0",
     "@ladjs/graceful": "^3.2.2",
+    "@lancedb/lancedb": "0.5.2",
     "@langchain/anthropic": "0.1.16",
     "@langchain/community": "0.0.53",
     "@langchain/core": "0.1.61",
@@ -81,7 +82,6 @@
     "url-pattern": "^1.0.3",
     "uuid": "^9.0.0",
     "uuid-apikey": "^1.5.3",
-    "vectordb": "0.4.11",
     "weaviate-ts-client": "^1.4.0",
     "winston": "^3.13.0"
   },

diff --git a/server/utils/vectorDbProviders/lance/index.js b/server/utils/vectorDbProviders/lance/index.js
@@ -1,16 +1,23 @@
-const lancedb = require("vectordb");
+const lancedb = require("@lancedb/lancedb");
 const { toChunks, getEmbeddingEngineSelection } = require("../../helpers");
 const { TextSplitter } = require("../../TextSplitter");
 const { SystemSettings } = require("../../../models/systemSettings");
 const { storeVectorResult, cachedVectorInformation } = require("../../files");
 const { v4: uuidv4 } = require("uuid");
 const { sourceIdentifier } = require("../../chats");
 
+/**
+ * LancedDB Client connection object
+ * @typedef {import('@lancedb/lancedb').Connection} LanceClient
+ */
+
 const LanceDb = {
   uri: `${
     !!process.env.STORAGE_DIR ? `${process.env.STORAGE_DIR}/` : "./storage/"
   }lancedb`,
   name: "LanceDb",
+
+  /** @returns {Promise<{client: LanceClient}>} */
   connect: async function () {
     if (process.env.VECTOR_DB !== "lancedb")
       throw new Error("LanceDB::Invalid ENV settings");
@@ -29,14 +36,12 @@ const LanceDb = {
     return { heartbeat: Number(new Date()) };
   },
   tables: async function () {
-    const fs = require("fs");
     const { client } = await this.connect();
-    const dirs = fs.readdirSync(client.uri);
-    return dirs.map((folder) => folder.replace(".lance", ""));
+    return await client.tableNames();
   },
   totalVectors: async function () {
     const { client } = await this.connect();
-    const tables = await this.tables();
+    const tables = await client.tableNames();
     let count = 0;
     for (const tableName of tables) {
       const table = await client.openTable(tableName);
@@ -52,6 +57,16 @@ const LanceDb = {
     const table = await client.openTable(_namespace);
     return (await table.countRows()) || 0;
   },
+  /**
+   * Performs a SimilaritySearch on a give LanceDB namespace.
+   * @param {LanceClient} client
+   * @param {string} namespace
+   * @param {number[]} queryVector
+   * @param {number} similarityThreshold
+   * @param {number} topN
+   * @param {string[]} filterIdentifiers
+   * @returns
+   */
   similarityResponse: async function (
     client,
     namespace,
@@ -68,10 +83,10 @@ const LanceDb = {
     };
 
     const response = await collection
-      .search(queryVector)
-      .metricType("cosine")
+      .vectorSearch(queryVector)
+      .distanceType("cosine")
       .limit(topN)
-      .execute();
+      .toArray();
 
     response.forEach((item) => {
       if (this.distanceToSimilarity(item._distance) < similarityThreshold)
@@ -94,6 +109,12 @@ const LanceDb = {
 
     return result;
   },
+  /**
+   *
+   * @param {LanceClient} client
+   * @param {string} namespace
+   * @returns
+   */
   namespace: async function (client, namespace = null) {
     if (!namespace) throw new Error("No namespace value provided.");
     const collection = await client.openTable(namespace).catch(() => false);
@@ -103,6 +124,13 @@ const LanceDb = {
       ...collection,
     };
   },
+  /**
+   *
+   * @param {LanceClient} client
+   * @param {number[]} data
+   * @param {string} namespace
+   * @returns
+   */
   updateOrCreateCollection: async function (client, data = [], namespace) {
     const hasNamespace = await this.hasNamespace(namespace);
     if (hasNamespace) {
@@ -120,14 +148,25 @@ const LanceDb = {
     const exists = await this.namespaceExists(client, namespace);
     return exists;
   },
-  namespaceExists: async function (_client, namespace = null) {
+  /**
+   *
+   * @param {LanceClient} client
+   * @param {string} namespace
+   * @returns
+   */
+  namespaceExists: async function (client, namespace = null) {
     if (!namespace) throw new Error("No namespace value provided.");
-    const collections = await this.tables();
+    const collections = await client.tableNames();
     return collections.includes(namespace);
   },
+  /**
+   *
+   * @param {LanceClient} client
+   * @param {string} namespace
+   * @returns
+   */
   deleteVectorsInNamespace: async function (client, namespace = null) {
-    const fs = require("fs");
-    fs.rm(`${client.uri}/${namespace}.lance`, { recursive: true }, () => null);
+    await client.dropTable(namespace);
     return true;
   },
   deleteDocumentFromNamespace: async function (namespace, docId) {