Mintplex-Labs · timothycarambat · May 12, 2025 · May 9, 2025 · May 9, 2025 · May 10, 2025
diff --git a/collector/extensions/index.js b/collector/extensions/index.js
@@ -4,6 +4,7 @@ const { resolveRepoLoader, resolveRepoLoaderFunction } = require("../utils/exten
 const { reqBody } = require("../utils/http");
 const { validURL } = require("../utils/url");
 const RESYNC_METHODS = require("./resync");
+const { loadObsidianVault } = require("../utils/extensions/ObsidianVault");
 
 function extensions(app) {
   if (!app) return;
@@ -180,6 +181,27 @@ function extensions(app) {
       return;
     }
   );
+
+  app.post(
+    "/ext/obsidian/vault",
+    [verifyPayloadIntegrity, setDataSigner],
+    async function (request, response) {
+      try {
+        const { files } = reqBody(request);
+        const result = await loadObsidianVault({ files });
+        response.status(200).json(result);
+      } catch (e) {
+        console.error(e);
+        response.status(400).json({
+          success: false,
+          reason: e.message,
+          data: null,
+        });
+      }
+      return;
+    }
+  );
 }
 
-module.exports = extensions;
+
+module.exports = extensions;
diff --git a/collector/utils/extensions/ObsidianVault/index.js b/collector/utils/extensions/ObsidianVault/index.js
@@ -0,0 +1,91 @@
+const { v4 } = require("uuid");
+const { default: slugify } = require("slugify");
+const path = require("path");
+const fs = require("fs");
+const {
+  writeToServerDocuments,
+  sanitizeFileName,
+  documentsFolder,
+} = require("../../files");
+
+function parseObsidianVaultPath(files = []) {
+  const possiblePaths = new Set();
+  files.forEach(
+    (file) => file?.path && possiblePaths.add(file.path.split("/")[0])
+  );
+
+  switch (possiblePaths.size) {
+    case 0:
+      return null;
+    case 1:
+      // The user specified a vault properly - so all files are in the same folder.
+      return possiblePaths.values().next().value;
+    default:
+      return null;
+  }
+}
+
+async function loadObsidianVault({ files = [] }) {
+  if (!files || files?.length === 0)
+    return { success: false, error: "No files provided" };
+  const vaultName = parseObsidianVaultPath(files);
+  const folderUUId = v4().slice(0, 4);
+  const outFolder = vaultName
+    ? slugify(`obsidian-vault-${vaultName}-${folderUUId}`).toLowerCase()
+    : slugify(`obsidian-${folderUUId}`).toLowerCase();
+  const outFolderPath = path.resolve(documentsFolder, outFolder);
+  if (!fs.existsSync(outFolderPath))
+    fs.mkdirSync(outFolderPath, { recursive: true });
+
+  console.log(
+    `Processing ${files.length} files from Obsidian Vault ${
+      vaultName ? `"${vaultName}"` : ""
+    }`
+  );
+  const results = [];
+  for (const file of files) {
+    try {
+      const fullPageContent = file?.content;
+      // If the file has no content or is just whitespace, skip it.
+      if (!fullPageContent || fullPageContent.trim() === "") continue;
+
+      const data = {
+        id: v4(),
+        url: `obsidian://${file.path}`,
+        title: file.name,
+        docAuthor: "Obsidian Vault",
+        description: file.name,
+        docSource: "Obsidian Vault",
+        chunkSource: `obsidian://${file.path}`,
+        published: new Date().toLocaleString(),
+        wordCount: fullPageContent.split(" ").length,
+        pageContent: fullPageContent,
+        token_count_estimate: fullPageContent.length / 4, // rough estimate
+      };
+
+      const targetFileName = sanitizeFileName(
+        `${slugify(file.name)}-${data.id}`
+      );
+      writeToServerDocuments(data, targetFileName, outFolderPath);
+      results.push({ file: file.path, status: "success" });
+    } catch (e) {
+      console.error(`Failed to process ${file.path}:`, e);
+      results.push({ file: file.path, status: "failed", reason: e.message });
+    }
+  }
+
+  return {
+    success: true,
+    data: {
+      processed: results.filter((r) => r.status === "success").length,
+      failed: results.filter((r) => r.status === "failed").length,
+      total: files.length,
+      results,
+      destination: path.basename(outFolderPath),
+    },
+  };
+}
+
+module.exports = {
+  loadObsidianVault,
+};
diff --git a/collector/utils/files/index.js b/collector/utils/files/index.js
@@ -2,6 +2,15 @@ const fs = require("fs");
 const path = require("path");
 const { MimeDetector } = require("./mime");
 
+/**
+ * The folder where documents are stored to be stored when
+ * processed by the collector.
+ */
+const documentsFolder =
+  process.env.NODE_ENV === "development"
+    ? path.resolve(__dirname, `../../../server/storage/documents`)
+    : path.resolve(process.env.STORAGE_DIR, `documents`);
+
 /**
  * Checks if a file is text by checking the mime type and then falling back to buffer inspection.
  * This way we can capture all the cases where the mime type is not known but still parseable as text
@@ -189,4 +198,5 @@ module.exports = {
   normalizePath,
   isWithin,
   sanitizeFileName,
+  documentsFolder,
 };
diff --git a/frontend/src/components/DataConnectorOption/media/index.js b/frontend/src/components/DataConnectorOption/media/index.js
@@ -4,6 +4,7 @@ import YouTube from "./youtube.svg";
 import Link from "./link.svg";
 import Confluence from "./confluence.jpeg";
 import DrupalWiki from "./drupalwiki.jpg";
+import Obsidian from "./obsidian.png";
 
 const ConnectorImages = {
   github: GitHub,
@@ -12,6 +13,7 @@ const ConnectorImages = {
   websiteDepth: Link,
   confluence: Confluence,
   drupalwiki: DrupalWiki,
+  obsidian: Obsidian,
 };
 
 export default ConnectorImages;
diff --git a/frontend/src/components/DataConnectorOption/media/obsidian.png b/frontend/src/components/DataConnectorOption/media/obsidian.png
diff --git a/frontend/src/components/Modals/ManageWorkspace/DataConnectors/Connectors/Obsidian/index.jsx b/frontend/src/components/Modals/ManageWorkspace/DataConnectors/Connectors/Obsidian/index.jsx
@@ -0,0 +1,175 @@
+import { useState } from "react";
+import { useTranslation } from "react-i18next";
+import { FolderOpen, Info } from "@phosphor-icons/react";
+import System from "@/models/system";
+import showToast from "@/utils/toast";
+
+export default function ObsidianOptions() {
+  const { t } = useTranslation();
+  const [loading, setLoading] = useState(false);
+  const [vaultPath, setVaultPath] = useState("");
+  const [selectedFiles, setSelectedFiles] = useState([]);
+
+  const handleFolderPick = async (e) => {
+    const files = Array.from(e.target.files);
+    if (files.length === 0) return;
+
+    // Filter for .md files only
+    const markdownFiles = files.filter((file) => file.name.endsWith(".md"));
+    setSelectedFiles(markdownFiles);
+
+    // Set the folder path from the first file
+    if (markdownFiles.length > 0) {
+      const path = markdownFiles[0].webkitRelativePath.split("/")[0];
+      setVaultPath(path);
+    }
+  };
+
+  const handleSubmit = async (e) => {
+    e.preventDefault();
+    if (selectedFiles.length === 0) return;
+
+    try {
+      setLoading(true);
+      showToast("Importing Obsidian vault - this may take a while.", "info", {
+        clear: true,
+        autoClose: false,
+      });
+
+      // Read all files and prepare them for submission
+      const fileContents = await Promise.all(
+        selectedFiles.map(async (file) => {
+          const content = await file.text();
+          return {
+            name: file.name,
+            path: file.webkitRelativePath,
+            content: content,
+          };
+        })
+      );
+
+      const { data, error } = await System.dataConnectors.obsidian.collect({
+        files: fileContents,
+      });
+
+      if (!!error) {
+        showToast(error, "error", { clear: true });
+        setLoading(false);
+        setSelectedFiles([]);
+        setVaultPath("");
+        return;
+      }
+
+      // Show results
+      const successCount = data.processed;
+      const failCount = data.failed;
+      const totalCount = data.total;
+
+      if (successCount === totalCount) {
+        showToast(
+          `Successfully imported ${successCount} files from your vault!`,
+          "success",
+          { clear: true }
+        );
+      } else {
+        showToast(
+          `Imported ${successCount} files, ${failCount} failed`,
+          "warning",
+          { clear: true }
+        );
+      }
+
+      setLoading(false);
+    } catch (e) {
+      console.error(e);
+      showToast(e.message, "error", { clear: true });
+      setLoading(false);
+    }
+  };
+
+  return (
+    <div className="flex w-full">
+      <div className="flex flex-col w-full px-1 md:pb-6 pb-16">
+        <form className="w-full" onSubmit={handleSubmit}>
+          <div className="w-full flex flex-col py-2">
+            <div className="w-full flex flex-col gap-4">
+              <div className="flex flex-col md:flex-row md:items-center gap-x-2 text-white mb-4 bg-blue-800/30 w-fit rounded-lg px-4 py-2">
+                <div className="gap-x-2 flex items-center">
+                  <Info className="shrink-0" size={25} />
+                  <p className="text-sm">
+                    {t("connectors.obsidian.vault_warning")}
+                  </p>
+                </div>
+              </div>
+
+              <div className="flex flex-col">
+                <div className="flex flex-col gap-y-1 mb-4">
+                  <label className="text-white text-sm font-bold">
+                    {t("connectors.obsidian.vault_location")}
+                  </label>
+                  <p className="text-xs font-normal text-theme-text-secondary">
+                    {t("connectors.obsidian.vault_description")}
+                  </p>
+                </div>
+                <div className="flex gap-x-2">
+                  <input
+                    type="text"
+                    value={vaultPath}
+                    onChange={(e) => setVaultPath(e.target.value)}
+                    placeholder="/path/to/your/vault"
+                    className="border-none bg-theme-settings-input-bg text-white placeholder:text-theme-settings-input-placeholder text-sm rounded-lg focus:outline-primary-button active:outline-primary-button outline-none block w-full p-2.5"
+                    required={true}
+                    autoComplete="off"
+                    spellCheck={false}
+                    readOnly
+                  />
+                  <label className="px-3 py-2 bg-theme-settings-input-bg border border-none rounded-lg text-white hover:bg-theme-settings-input-bg/80 cursor-pointer">
+                    <FolderOpen size={20} />
+                    <input
+                      type="file"
+                      webkitdirectory=""
+                      onChange={handleFolderPick}
+                      className="hidden"
+                    />
+                  </label>
+                </div>
+                {selectedFiles.length > 0 && (
+                  <>
+                    <p className="text-xs text-white mt-2 font-bold">
+                      {t("connectors.obsidian.selected_files", {
+                        count: selectedFiles.length,
+                      })}
+                    </p>
+
+                    {selectedFiles.map((file, i) => (
+                      <p key={i} className="text-xs text-white mt-2">
+                        {file.webkitRelativePath}
+                      </p>
+                    ))}
+                  </>
+                )}
+              </div>
+            </div>
+          </div>
+
+          <div className="flex flex-col gap-y-2 w-full pr-10">
+            <button
+              type="submit"
+              disabled={loading || selectedFiles.length === 0}
+              className="border-none mt-2 w-full justify-center px-4 py-2 rounded-lg text-dark-text light:text-white text-sm font-bold items-center flex gap-x-2 bg-theme-home-button-primary hover:bg-theme-home-button-primary-hover disabled:bg-theme-home-button-primary-hover disabled:cursor-not-allowed"
+            >
+              {loading
+                ? t("connectors.obsidian.importing")
+                : t("connectors.obsidian.import_vault")}
+            </button>
+            {loading && (
+              <p className="text-xs text-white/50">
+                {t("connectors.obsidian.processing_time")}
+              </p>
+            )}
+          </div>
+        </form>
+      </div>
+    </div>
+  );
+}
diff --git a/frontend/src/components/Modals/ManageWorkspace/DataConnectors/index.jsx b/frontend/src/components/Modals/ManageWorkspace/DataConnectors/index.jsx
@@ -9,6 +9,7 @@ import DrupalWikiOptions from "./Connectors/DrupalWiki";
 import { useState } from "react";
 import ConnectorOption from "./ConnectorOption";
 import WebsiteDepthOptions from "./Connectors/WebsiteDepth";
+import ObsidianOptions from "./Connectors/Obsidian";
 
 export const getDataConnectors = (t) => ({
   github: {
@@ -47,6 +48,12 @@ export const getDataConnectors = (t) => ({
     description: "Import Drupal Wiki spaces in a single click.",
     options: <DrupalWikiOptions />,
   },
+  obsidian: {
+    name: "Obsidian",
+    image: ConnectorImages.obsidian,
+    description: "Import Obsidian vault in a single click.",
+    options: <ObsidianOptions />,
+  },
 });
 
 export default function DataConnectors() {

diff --git a/frontend/src/locales/ar/common.js b/frontend/src/locales/ar/common.js
@@ -641,6 +641,17 @@ const TRANSLATIONS = {
       watch_explained_block3_end: null,
       accept: null,
     },
+    obsidian: {
+      name: null,
+      description: null,
+      vault_location: null,
+      vault_description: null,
+      selected_files: null,
+      importing: null,
+      import_vault: null,
+      processing_time: null,
+      vault_warning: null,
+    },
   },
   chat_window: {
     welcome: null,