这是indexloc提供的服务，不要输入任何密码
Skip to content

LanceDB upgrade and improved functionality. #155

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Jul 20, 2023
Merged

LanceDB upgrade and improved functionality. #155

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 0 additions & 38 deletions frontend/src/components/Modals/MangeWorkspace/Documents/CannotRemoveModal/index.jsx
View file
Open in desktop

This file was deleted.

18 changes: 0 additions & 18 deletions frontend/src/components/Modals/MangeWorkspace/Documents/index.jsx
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import paths from "../../../../utils/paths";
import { useParams } from "react-router-dom";
import Directory from "./Directory";
import ConfirmationModal from "./ConfirmationModal";
import CannotRemoveModal from "./CannotRemoveModal";
import { AlertTriangle } from "react-feather";

export default function DocumentSettings({ workspace }) {
Expand All @@ -16,22 +15,18 @@ export default function DocumentSettings({ workspace }) {
const [directories, setDirectories] = useState(null);
const [originalDocuments, setOriginalDocuments] = useState([]);
const [selectedFiles, setSelectFiles] = useState([]);
const [vectordb, setVectorDB] = useState(null);
const [showingNoRemovalModal, setShowingNoRemovalModal] = useState(false);
const [hasFiles, setHasFiles] = useState(true);

useEffect(() => {
async function fetchKeys() {
const localFiles = await System.localFiles();
const settings = await System.keys();
const originalDocs = workspace.documents.map((doc) => doc.docpath) || [];
const hasAnyFiles = localFiles.items.some(
(folder) => folder?.items?.length > 0
);
setDirectories(localFiles);
setOriginalDocuments([...originalDocs]);
setSelectFiles([...originalDocs]);
setVectorDB(settings?.VectorDB);
setHasFiles(hasAnyFiles);
setLoading(false);
}
Expand Down Expand Up @@ -109,13 +104,6 @@ export default function DocumentSettings({ workspace }) {
const parent = isFolder ? filepath : filepath.split("/")[0];

if (isSelected(filepath)) {
// Certain vector DBs do not contain the ability to delete vectors
// so we cannot remove from these. The user will have to clear the entire workspace.
if (["lancedb"].includes(vectordb) && isOriginalDoc(filepath)) {
setShowingNoRemovalModal(true);
return false;
}

const updatedDocs = isFolder
? selectedFiles.filter((doc) => !doc.includes(parent))
: selectedFiles.filter((doc) => !doc.includes(filepath));
Expand Down Expand Up @@ -160,12 +148,6 @@ export default function DocumentSettings({ workspace }) {
updateWorkspace={updateWorkspace}
/>
)}
{showingNoRemovalModal && (
<CannotRemoveModal
hideModal={() => setShowingNoRemovalModal(false)}
vectordb={vectordb}
/>
)}
<div className="p-6 flex h-full w-full max-h-[80vh] overflow-y-scroll">
<div className="flex flex-col gap-y-1 w-full">
{!hasFiles && (
Expand Down
4 changes: 2 additions & 2 deletions server/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,10 +35,10 @@
"sqlite": "^4.2.1",
"sqlite3": "^5.1.6",
"uuid": "^9.0.0",
"vectordb": "0.1.5"
"vectordb": "0.1.12"
},
"devDependencies": {
"nodemon": "^2.0.22",
"prettier": "^2.4.1"
}
}
}
43 changes: 35 additions & 8 deletions server/utils/vectorDbProviders/lance/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,21 @@ const LanceDb = {
await this.connect();
return { heartbeat: Number(new Date()) };
},
tables: async function () {
const fs = require("fs");
const { client } = await this.connect();
const dirs = fs.readdirSync(client.uri);
return dirs.map((folder) => folder.replace(".lance", ""));
},
Comment on lines +45 to +50
Copy link
Member Author

@timothycarambat timothycarambat Jul 20, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The native tableNames() function does not work so we roll our own.

totalIndicies: async function () {
return 0; // Unsupported for LanceDB - so always zero
const { client } = await this.connect();
const tables = await this.tables();
let count = 0;
for (const tableName of tables) {
const table = await client.openTable(tableName);
count += await table.countRows();
}
return count;
},
embeddingFunc: function () {
return new lancedb.OpenAIEmbeddingFunction(
Expand Down Expand Up @@ -121,7 +134,8 @@ const LanceDb = {
};
},
updateOrCreateCollection: async function (client, data = [], namespace) {
if (await this.hasNamespace(namespace)) {
const hasNamespace = await this.hasNamespace(namespace);
if (hasNamespace) {
const collection = await client.openTable(namespace);
await collection.add(data);
return true;
Expand All @@ -136,21 +150,34 @@ const LanceDb = {
const exists = await this.namespaceExists(client, namespace);
return exists;
},
namespaceExists: async function (client, namespace = null) {
namespaceExists: async function (_client, namespace = null) {
if (!namespace) throw new Error("No namespace value provided.");
const collections = await client.tableNames();
const collections = await this.tables();
return collections.includes(namespace);
},
deleteVectorsInNamespace: async function (client, namespace = null) {
const fs = require("fs");
fs.rm(`${client.uri}/${namespace}.lance`, { recursive: true }, () => null);
return true;
},
deleteDocumentFromNamespace: async function (_namespace, _docId) {
console.error(
`LanceDB:deleteDocumentFromNamespace - unsupported operation. No changes made to vector db.`
deleteDocumentFromNamespace: async function (namespace, docId) {
const { client } = await this.connect();
const exists = await this.namespaceExists(client, namespace);
if (!exists) {
console.error(
`LanceDB:deleteDocumentFromNamespace - namespace ${namespace} does not exist.`
);
return;
}

const { DocumentVectors } = require("../../../models/vectors");
const table = await client.openTable(namespace);
const vectorIds = (await DocumentVectors.where(`docId = '${docId}'`)).map(
(record) => record.vectorId
);
return false;

await table.delete(`id IN (${vectorIds.map((v) => `'${v}'`).join(",")})`);
return true;
},
addDocumentToNamespace: async function (
namespace,
Expand Down
Loading